环境需求
#请参考https://www.shuzihan.com/2022/09/29/1244/,先部署搭建k8s集群
#节点信息
10.0.0.160 easzlab-k8s-ceph-01 4c 8g 200G
10.0.0.161 easzlab-k8s-ceph-02 4c 8g 200G
10.0.0.162 easzlab-k8s-ceph-03 4c 8g 200Ggit下载包
[root@easzlab-deploy-01 ceph]#wget https://github.com/rook/rook/archive/refs/tags/v1.10.2.tar.gz
[root@easzlab-deploy-01 ceph]#tar xf v1.10.2.tar.gz
[root@easzlab-deploy-01 ceph]#cd rook-1.10.2/deploy/examples/
[root@easzlab-deploy-01 examples]#kubectl create -f crds.yaml -f common.yaml -f operator.yaml
customresourcedefinition.apiextensions.k8s.io/cephblockpoolradosnamespaces.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephblockpools.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephbucketnotifications.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephbuckettopics.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephclients.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephclusters.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephfilesystemmirrors.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephfilesystems.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephfilesystemsubvolumegroups.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephnfses.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectrealms.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectstores.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectstoreusers.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectzonegroups.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectzones.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephrbdmirrors.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/objectbucketclaims.objectbucket.io created
customresourcedefinition.apiextensions.k8s.io/objectbuckets.objectbucket.io created
namespace/rook-ceph created
clusterrole.rbac.authorization.k8s.io/cephfs-csi-nodeplugin created
clusterrole.rbac.authorization.k8s.io/cephfs-external-provisioner-runner created
clusterrole.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
clusterrole.rbac.authorization.k8s.io/rbd-external-provisioner-runner created
clusterrole.rbac.authorization.k8s.io/rook-ceph-cluster-mgmt created
clusterrole.rbac.authorization.k8s.io/rook-ceph-global created
clusterrole.rbac.authorization.k8s.io/rook-ceph-mgr-cluster created
clusterrole.rbac.authorization.k8s.io/rook-ceph-mgr-system created
clusterrole.rbac.authorization.k8s.io/rook-ceph-object-bucket created
clusterrole.rbac.authorization.k8s.io/rook-ceph-osd created
clusterrole.rbac.authorization.k8s.io/rook-ceph-system created
clusterrolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role created
clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-global created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-cluster created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-object-bucket created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-osd created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-system created
role.rbac.authorization.k8s.io/cephfs-external-provisioner-cfg created
role.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
role.rbac.authorization.k8s.io/rbd-external-provisioner-cfg created
role.rbac.authorization.k8s.io/rook-ceph-cmd-reporter created
role.rbac.authorization.k8s.io/rook-ceph-mgr created
role.rbac.authorization.k8s.io/rook-ceph-osd created
role.rbac.authorization.k8s.io/rook-ceph-purge-osd created
role.rbac.authorization.k8s.io/rook-ceph-rgw created
role.rbac.authorization.k8s.io/rook-ceph-system created
rolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role-cfg created
rolebinding.rbac.authorization.k8s.io/rbd-csi-nodeplugin-role-cfg created
rolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role-cfg created
rolebinding.rbac.authorization.k8s.io/rook-ceph-cluster-mgmt created
rolebinding.rbac.authorization.k8s.io/rook-ceph-cmd-reporter created
rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr created
rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-system created
rolebinding.rbac.authorization.k8s.io/rook-ceph-osd created
rolebinding.rbac.authorization.k8s.io/rook-ceph-purge-osd created
rolebinding.rbac.authorization.k8s.io/rook-ceph-rgw created
rolebinding.rbac.authorization.k8s.io/rook-ceph-system created
serviceaccount/rook-ceph-cmd-reporter created
serviceaccount/rook-ceph-mgr created
serviceaccount/rook-ceph-osd created
serviceaccount/rook-ceph-purge-osd created
serviceaccount/rook-ceph-rgw created
serviceaccount/rook-ceph-system created
serviceaccount/rook-csi-cephfs-plugin-sa created
serviceaccount/rook-csi-cephfs-provisioner-sa created
serviceaccount/rook-csi-rbd-plugin-sa created
serviceaccount/rook-csi-rbd-provisioner-sa created
configmap/rook-ceph-operator-config created
deployment.apps/rook-ceph-operator created
[root@easzlab-deploy-01 examples]#kubectl create -f cluster.yaml
cephcluster.ceph.rook.io/rook-ceph created
[root@easzlab-deploy-01 examples]#kubectl apply -f toolbox.yaml #安装ceph管理端
deployment.apps/rook-ceph-tools created
[root@easzlab-deploy-01 examples]#kubectl apply -f dashboard-external-https.yaml #通过svc映射端口到本地
service/rook-ceph-mgr-dashboard-external-https created
由于部分镜像需要外网环境下载所以需要单独下载后上传
[root@easzlab-deploy-01 ansible]#vim root_ceph_images.yaml
---
- name: install ceph images
hosts: master,node,ceph
tasks:
- name: create dir1
file:
path: "{{ item }}"
state: directory
loop:
- /data
- /data/ceph
- name: unarchive
unarchive:
src: rook-ceph.tar.gz
dest: /data/ceph
- name: shell
shell: "nerdctl -n k8s.io load -i /data/ceph/rook-ceph/{{ item }}"
loop:
- csi-attacher.tar
- csi-node-driver-registrar.tar
- csi-provisioner.tar
- csi-resizer.tar
- csi-snapshotter.tar
[root@easzlab-deploy-01 ansible]#ansible-playbook root_ceph_images.yaml
PLAY [install ceph images] ******************************************************************************************************************************************************************
TASK [Gathering Facts] **********************************************************************************************************************************************************************
ok: [easzlab-k8s-master-03]
ok: [easzlab-k8s-master-01]
ok: [easzlab-k8s-node-01]
ok: [easzlab-k8s-master-02]
ok: [easzlab-k8s-node-02]
ok: [easzlab-k8s-node-03]
ok: [easzlab-k8s-ceph-01]
ok: [easzlab-k8s-ceph-02]
ok: [easzlab-k8s-ceph-03]
TASK [create dir1] **************************************************************************************************************************************************************************
ok: [easzlab-k8s-master-03] => (item=/data)
ok: [easzlab-k8s-master-01] => (item=/data)
ok: [easzlab-k8s-node-01] => (item=/data)
ok: [easzlab-k8s-node-02] => (item=/data)
ok: [easzlab-k8s-master-02] => (item=/data)
ok: [easzlab-k8s-master-03] => (item=/data/ceph)
ok: [easzlab-k8s-master-01] => (item=/data/ceph)
ok: [easzlab-k8s-node-01] => (item=/data/ceph)
ok: [easzlab-k8s-node-02] => (item=/data/ceph)
ok: [easzlab-k8s-node-03] => (item=/data)
ok: [easzlab-k8s-master-02] => (item=/data/ceph)
ok: [easzlab-k8s-ceph-02] => (item=/data)
ok: [easzlab-k8s-ceph-01] => (item=/data)
ok: [easzlab-k8s-node-03] => (item=/data/ceph)
ok: [easzlab-k8s-ceph-03] => (item=/data)
ok: [easzlab-k8s-ceph-01] => (item=/data/ceph)
ok: [easzlab-k8s-ceph-02] => (item=/data/ceph)
ok: [easzlab-k8s-ceph-03] => (item=/data/ceph)
TASK [unarchive] ****************************************************************************************************************************************************************************
ok: [easzlab-k8s-master-03]
ok: [easzlab-k8s-master-01]
ok: [easzlab-k8s-master-02]
ok: [easzlab-k8s-node-02]
ok: [easzlab-k8s-node-01]
ok: [easzlab-k8s-node-03]
ok: [easzlab-k8s-ceph-01]
ok: [easzlab-k8s-ceph-03]
ok: [easzlab-k8s-ceph-02]
TASK [shell] ********************************************************************************************************************************************************************************
changed: [easzlab-k8s-node-01] => (item=csi-attacher.tar)
changed: [easzlab-k8s-node-02] => (item=csi-attacher.tar)
changed: [easzlab-k8s-master-03] => (item=csi-attacher.tar)
changed: [easzlab-k8s-master-01] => (item=csi-attacher.tar)
changed: [easzlab-k8s-node-01] => (item=csi-node-driver-registrar.tar)
changed: [easzlab-k8s-master-02] => (item=csi-attacher.tar)
changed: [easzlab-k8s-master-01] => (item=csi-node-driver-registrar.tar)
changed: [easzlab-k8s-node-02] => (item=csi-node-driver-registrar.tar)
changed: [easzlab-k8s-master-03] => (item=csi-node-driver-registrar.tar)
changed: [easzlab-k8s-master-02] => (item=csi-node-driver-registrar.tar)
changed: [easzlab-k8s-node-02] => (item=csi-provisioner.tar)
changed: [easzlab-k8s-node-01] => (item=csi-provisioner.tar)
changed: [easzlab-k8s-master-03] => (item=csi-provisioner.tar)
changed: [easzlab-k8s-master-01] => (item=csi-provisioner.tar)
changed: [easzlab-k8s-master-02] => (item=csi-provisioner.tar)
changed: [easzlab-k8s-node-02] => (item=csi-resizer.tar)
changed: [easzlab-k8s-master-01] => (item=csi-resizer.tar)
changed: [easzlab-k8s-master-03] => (item=csi-resizer.tar)
changed: [easzlab-k8s-node-01] => (item=csi-resizer.tar)
changed: [easzlab-k8s-node-01] => (item=csi-snapshotter.tar)
changed: [easzlab-k8s-master-03] => (item=csi-snapshotter.tar)
changed: [easzlab-k8s-node-02] => (item=csi-snapshotter.tar)
changed: [easzlab-k8s-master-01] => (item=csi-snapshotter.tar)
changed: [easzlab-k8s-master-02] => (item=csi-resizer.tar)
changed: [easzlab-k8s-node-03] => (item=csi-attacher.tar)
changed: [easzlab-k8s-master-02] => (item=csi-snapshotter.tar)
changed: [easzlab-k8s-ceph-03] => (item=csi-attacher.tar)
changed: [easzlab-k8s-ceph-02] => (item=csi-attacher.tar)
changed: [easzlab-k8s-ceph-01] => (item=csi-attacher.tar)
changed: [easzlab-k8s-node-03] => (item=csi-node-driver-registrar.tar)
changed: [easzlab-k8s-ceph-03] => (item=csi-node-driver-registrar.tar)
changed: [easzlab-k8s-ceph-02] => (item=csi-node-driver-registrar.tar)
changed: [easzlab-k8s-ceph-01] => (item=csi-node-driver-registrar.tar)
changed: [easzlab-k8s-ceph-01] => (item=csi-provisioner.tar)
changed: [easzlab-k8s-ceph-03] => (item=csi-provisioner.tar)
changed: [easzlab-k8s-ceph-02] => (item=csi-provisioner.tar)
changed: [easzlab-k8s-node-03] => (item=csi-provisioner.tar)
changed: [easzlab-k8s-ceph-01] => (item=csi-resizer.tar)
changed: [easzlab-k8s-ceph-03] => (item=csi-resizer.tar)
changed: [easzlab-k8s-ceph-02] => (item=csi-resizer.tar)
changed: [easzlab-k8s-node-03] => (item=csi-resizer.tar)
changed: [easzlab-k8s-ceph-03] => (item=csi-snapshotter.tar)
changed: [easzlab-k8s-ceph-01] => (item=csi-snapshotter.tar)
changed: [easzlab-k8s-ceph-02] => (item=csi-snapshotter.tar)
changed: [easzlab-k8s-node-03] => (item=csi-snapshotter.tar)
PLAY RECAP **********************************************************************************************************************************************************************************
easzlab-k8s-ceph-01 : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
easzlab-k8s-ceph-02 : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
easzlab-k8s-ceph-03 : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
easzlab-k8s-master-01 : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
easzlab-k8s-master-02 : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
easzlab-k8s-master-03 : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
easzlab-k8s-node-01 : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
easzlab-k8s-node-02 : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
easzlab-k8s-node-03 : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
访问ceph dashboard
#默认创建的用户为admin
#此处获取密码
[root@easzlab-deploy-01 ansible]#kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo
;2JD;!GfO>sBAH>z'5Y^
扩容ceph节点
#关闭节点机器
#添加硬盘设备
#观察ceph状态
[root@easzlab-deploy-01 ansible]#kubectl exec -it -n rook-ceph rook-ceph-tools-7564bb9799-qndhf sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
sh-4.4$ ceph osd df tree
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME
-1 0.68358 - 700 GiB 67 MiB 2.2 MiB 1 KiB 65 MiB 700 GiB 0.01 1.00 - root default
-5 0.29298 - 300 GiB 25 MiB 884 KiB 1 KiB 25 MiB 300 GiB 0.01 0.89 - host 10-0-0-160
1 hdd 0.19530 1.00000 200 GiB 5.9 MiB 668 KiB 1 KiB 5.2 MiB 200 GiB 0.00 0.31 1 up osd.1
3 hdd 0.09769 1.00000 100 GiB 20 MiB 216 KiB 0 B 19 MiB 100 GiB 0.02 2.04 0 up osd.3
-3 0.19530 - 200 GiB 21 MiB 652 KiB 0 B 20 MiB 200 GiB 0.01 1.08 - host 10-0-0-161
0 hdd 0.19530 1.00000 200 GiB 21 MiB 652 KiB 0 B 20 MiB 200 GiB 0.01 1.08 0 down osd.0
-7 0.19530 - 200 GiB 21 MiB 668 KiB 0 B 20 MiB 200 GiB 0.01 1.08 - host 10-0-0-162
2 hdd 0.19530 1.00000 200 GiB 21 MiB 668 KiB 0 B 20 MiB 200 GiB 0.01 1.08 1 up osd.2
TOTAL 700 GiB 67 MiB 2.2 MiB 1.1 KiB 65 MiB 700 GiB 0.01
MIN/MAX VAR: 0.31/2.04 STDDEV: 0.01
#添加完后启动节点
#注意可能启动后无法自动创建新的osd需要手动删除pod后重新自动加载pod才能扩容成功
创建ceph 存储类，为k8s集群提供动态存储
[root@easzlab-deploy-01 ceph]#vim hdd-sc.yaml
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: hdd-rook-ceph-block
namespace: rook-ceph
spec:
failureDomain: host
replicated:
size: 2
deviceClass: hdd
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: hdd-rook-ceph-block
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
clusterID: rook-ceph
pool: hdd-rook-ceph-block
imageFormat: "2"
imageFeatures: layering
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
csi.storage.k8s.io/fstype: xfs
reclaimPolicy: Delete
[root@easzlab-deploy-01 ceph]#kubectl apply -f hdd-sc.yaml
cephblockpool.ceph.rook.io/hdd-rook-ceph-block created
storageclass.storage.k8s.io/hdd-rook-ceph-block created
[root@easzlab-deploy-01 ceph]#kubectl get sc -A
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
hdd-rook-ceph-block rook-ceph.rbd.csi.ceph.com Delete Immediate false 13s
#指定为集群的默认存储类
[root@easzlab-deploy-01 ceph]#kubectl patch storageclass hdd-rook-ceph-block -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
storageclass.storage.k8s.io/hdd-rook-ceph-block patched
[root@easzlab-deploy-01 ceph]#kubectl get sc -A
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
hdd-rook-ceph-block (default) rook-ceph.rbd.csi.ceph.com Delete Immediate false 2m15s
