使用 Ceph RBD 的完整示例
原文链接
该测试在外部集群及rook集群中都已经测试：通过
k8s集群
[root@kmaster ceph]# kubectl get node -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
kmaster Ready control-plane,master 4h58m v1.21.0 192.168.31.10 <none> Rocky Linux 8 4.18.0-240.22.1.el8.x86_64 docker://20.10.6
knode01 Ready <none> 4h57m v1.21.0 192.168.31.11 <none> Rocky Linux 8 4.18.0-240.22.1.el8.x86_64 docker://20.10.6
knode02 Ready <none> 4h57m v1.21.0 192.168.31.12 <none> Rocky Linux 8 4.18.0-240.22.1.el8.x86_64 docker://20.10.6
[root@kmaster ceph]#
CEPH 集群
[root@kmaster ceph]# kubectl get pod -nrook-ceph
NAME READY STATUS RESTARTS AGE
csi-cephfsplugin-4wr2d 3/3 Running 0 4h58m
csi-cephfsplugin-nztms 3/3 Running 0 4h57m
csi-cephfsplugin-provisioner-6f75644874-jxb6n 6/6 Running 0 4h54m
csi-cephfsplugin-provisioner-6f75644874-shszs 6/6 Running 4 4h58m
csi-cephfsplugin-rbvfd 3/3 Running 0 4h58m
csi-rbdplugin-6h8nb 3/3 Running 0 4h58m
csi-rbdplugin-htpnp 3/3 Running 0 4h57m
csi-rbdplugin-jzf6n 3/3 Running 0 4h58m
csi-rbdplugin-provisioner-67fb987799-fxj68 6/6 Running 4 4h58m
csi-rbdplugin-provisioner-67fb987799-zzm2x 6/6 Running 0 4h54m
rook-ceph-crashcollector-kmaster-7596c6f695-z784j 1/1 Running 0 4h56m
rook-ceph-crashcollector-knode01-5c75d4cbc8-z45lb 1/1 Running 0 4h57m
rook-ceph-crashcollector-knode02-67d58f7c55-m8rvj 1/1 Running 0 4h57m
rook-ceph-mgr-a-cfdb8d4b8-kxxr9 1/1 Running 0 4h56m
rook-ceph-mon-a-77dbfbb9b6-wlzbw 1/1 Running 0 4h57m
rook-ceph-mon-b-65d59f4667-hvdbt 1/1 Running 0 4h57m
rook-ceph-mon-c-9c8b69b9c-4x86g 1/1 Running 0 4h57m
rook-ceph-operator-6459f5dc4b-pq8gc 1/1 Running 0 4h58m
rook-ceph-osd-0-6dd858b9b5-xqlc8 1/1 Running 0 4h56m
rook-ceph-osd-1-8596cc946c-cldp5 1/1 Running 0 4h56m
rook-ceph-osd-2-76c758bd77-gpwx9 1/1 Running 0 4h56m
rook-ceph-osd-prepare-kmaster-4rh6n 0/1 Completed 0 32m
rook-ceph-osd-prepare-knode01-lxm4c 0/1 Completed 0 32m
rook-ceph-osd-prepare-knode02-cwfhm 0/1 Completed 0 32m
rook-ceph-tools-7467d8bf8-zqqlj 1/1 Running 0 104m
[root@kmaster ceph]#
创建pool
[root@rook-ceph-tools-7467d8bf8-x7scq /]# ceph osd pool create k8s 128 128
pool 'k8s' created
[root@rook-ceph-tools-7467d8bf8-x7scq /]# ceph osd pool application enable k8s rbd
enabled application 'rbd' on pool 'k8s'
[root@rook-ceph-tools-7467d8bf8-x7scq /]#
[root@rook-ceph-tools-7467d8bf8-x7scq /]#
[root@kmaster kubernetes]# k exec -it -n rook-ceph rook-ceph-tools-7467d8bf8-zqqlj /bin/bash -- ceph -s
cluster:
id: 0ad47b5f-e055-4448-b8b6-5ab5ccd57799
health: HEALTH_WARN
1 pool(s) do not have an application enabled
services:
mon: 3 daemons, quorum a,b,c (age 34m)
mgr: a(active, since 2d)
osd: 3 osds: 3 up (since 88m), 3 in (since 2d)
data:
pools: 2 pools, 33 pgs
objects: 12 objects, 3.2 MiB
usage: 3.2 GiB used, 132 GiB / 135 GiB avail
pgs: 33 active+clean
[root@kmaster kubernetes]#
[root@kmaster kubernetes]# k exec -it -n rook-ceph rook-ceph-tools-7467d8bf8-zqqlj /bin/bash -- ceph osd pool application enable k8s rbd
enabled application 'rbd' on pool 'k8s'
[root@kmaster kubernetes]#
创建rbd
[root@master ~]# rbd create ceph-image1 --pool k8s --size=1G
[root@master ~]# rbd ls --pool k8s
ceph-image
ceph-image1
[root@master ~]#
查询secret
[root@rook-ceph-tools-7467d8bf8-x7scq /]# ceph auth get-key client.admin | base64
QVFCTjJDQmgwU2FKTXhBQXBPUklUYU5QZTJjaklVaG9TbXBTYnc9PQ==
[root@rook-ceph-tools-7467d8bf8-x7scq /]#
创建secret、pvc、pc
ceph-pv.yaml
apiVersion: v1
kind: Secret
metadata:
name: ceph-secret
data:
key: QVFCTjJDQmgwU2FKTXhBQXBPUklUYU5QZTJjaklVaG9TbXBTYnc9PQ==
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: ceph-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteOnce
rbd:
monitors:
- 10.111.94.249:6789
- 10.106.67.38:6789
- 10.99.183.92:6789
pool: k8s
image: ceph-image
user: admin
secretRef:
name: ceph-secret
fsType: ext4
readOnly: false
persistentVolumeReclaimPolicy: Recycle
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: ceph-claim
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
创建pod.yaml
apiVersion: v1
kind: Pod
metadata:
name: ceph-pod1
spec:
containers:
- name: ceph-busybox
image: busybox:latest
command: ["sleep", "60000"]
volumeMounts:
- name: ceph-vo1
mountPath: /usr/share/busybox
readOnly: false
volumes:
- name: ceph-vo1
persistentVolumeClaim:
claimName: ceph-claim
外部ceph集群测试
[root@master ~]# kubectl get pod,pvc,pv
NAME READY STATUS RESTARTS AGE
pod/ceph-pod1 1/1 Running 0 7m46s
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/ceph-claim Bound ceph-pv 2Gi RWO 35m
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
persistentvolume/ceph-pv 2Gi RWO Recycle Bound default/ceph-claim 35m
[root@master ~]#
Rook集群测试
[root@kmaster ceph]# kg pod,pvc,pv -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/ceph-pod1 1/1 Running 0 58s 10.244.2.26 knode02 <none> <none>
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE VOLUMEMODE
persistentvolumeclaim/ceph-claim Bound ceph-pv 2Gi RWO 14m Filesystem
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE VOLUMEMODE
persistentvolume/ceph-pv 2Gi RWO Recycle Bound default/ceph-claim 14m Filesystem
[root@kmaster ceph]#