{"id":580,"date":"2025-07-04T16:32:04","date_gmt":"2025-07-04T07:32:04","guid":{"rendered":"http:\/\/www.iwgh.my\/?p=580"},"modified":"2025-07-09T11:36:46","modified_gmt":"2025-07-09T02:36:46","slug":"ceph-reef-%eb%b2%84%ec%a0%84-%ec%84%a4%ec%b9%98%ed%95%98%ea%b8%b0-with-rocky-linux-9","status":"publish","type":"post","link":"https:\/\/iwgh.duckdns.org\/?p=580","title":{"rendered":"Ceph reef \ubc84\uc804 \uc124\uce58\ud558\uae30.(with Rocky Linux 9)"},"content":{"rendered":"\n
\"\"<\/figure>\n\n\n\n

!! ceph reef \ubc84\uc804\ubd80\ud130\ub294 cephadm\uc744 \uc81c\uacf5\ud558\uc5ec \uad00\ub9ac\uba74\uc5d0\uc11c \ud3b8\uc758\uc131\uc744 \uc81c\uacf5.
\ubc30\ud3ec\ub294 \ubaa8\ub450 Container\ub97c \ud1b5\ud574 \ubc30\ud3ec\ub418\uba70 \uc791\ub3d9 \uc5ed\uc2dc \ub9c8\ucc2c\uac00\uc9c0\ub85c \ucee8\ud14c\uc774\ub108\uae30\ubc18.<\/p>\n\n\n\n

1. \uc0ac\uc804\ud655\uc778\uc0ac\ud56d
1.1. dnf update.(OS current \ubc84\uc804 \uc720\uc9c0 \ubc0f node\uac04 \ubc84\uc804 \uc77c\uce58.)
1.2. \uc0ac\uc804 package \uc124\uce58
# dnf install epel-release
# dnf install centos-release-ceph-reef.noarch
# rpm –import ‘https:\/\/download.ceph.com\/keys\/release.asc’
# systemctl disable firewalld.service
# systemctl stop firewalld.service
# getenforce ;;selinux \ud574\uc81c.
# vi \/etc\/selinux\/config
SELINUX=disable
# dnf install podman ;; ceph\ub294 \uc8fc\ub85c podman \uc0ac\uc6a9, docker\ub3c4 \uac00\ub2a5.
# dnf install python3
1.3 \uac01 node \uc2dc\uac04\uc77c\uce58 \ubc0f ntp\/chronyd \uc124\uc815<\/p>\n\n\n\n

2. cephadm bootstrep node \uc124\uc815 \ubc0f \ubc30\ud3ec.
2.1. cephadm \uc124\uce58
# dnf install -y cephadm<\/p>\n\n\n\n

\n
# dnf install -y cephadm\n\n\ub9c8\uc9c0\ub9c9 \uba54\ud0c0\uc790\ub8cc \ub9cc\ub8cc\ud655\uc778(0:03:56 \uc774\uc804): 2025\ub144 07\uc6d4 03\uc77c (\ubaa9) \uc624\ud6c4 03\uc2dc 43\ubd84 58\ucd08.\n\uc885\uc18d\uc131\uc774 \ud574\uacb0\ub418\uc5c8\uc2b5\ub2c8\ub2e4.\n================================================================================================================ \uafb8\ub7ec\ubbf8                     \uad6c\uc870                      \ubc84\uc804                               \uc800\uc7a5\uc18c                               \ud06c\uae30\n===================================================================================================================================\uc124\uce58 \uc911:\ncephadm                    noarch                    2:18.2.7-2.el9s                    centos-ceph-reef                    228 k\n\n\uc5f0\uacb0 \uc694\uc57d\n===================================================================================================================================\uc124\uce58  1 \uafb8\ub7ec\ubbf8\n\n\uc804\uccb4 \ub0b4\ub824\ubc1b\uae30 \ud06c\uae30: 228 k\n\uc124\uce58\ub41c \ud06c\uae30 : 218 k\n\uafb8\ub7ec\ubbf8 \ub0b4\ub824\ubc1b\uae30 \uc911:\ncephadm-18.2.7-2.el9s.noarch.rpm                                                                   316 kB\/s | 228 kB     00:00    \n-----------------------------------------------------------------------------------------------------------------------------------\ud569\uacc4                                                                                               166 kB\/s | 228 kB     00:01     \n\uc5f0\uacb0 \ud655\uc778 \uc2e4\ud589 \uc911\n\uc5f0\uacb0 \ud655\uc778\uc5d0 \uc131\uacf5\ud588\uc2b5\ub2c8\ub2e4.\n\uc5f0\uacb0 \uc2dc\ud5d8 \uc2e4\ud589 \uc911\n\uc5f0\uacb0 \uc2dc\ud5d8\uc5d0 \uc131\uacf5\ud588\uc2b5\ub2c8\ub2e4.\n\uc5f0\uacb0 \uc2e4\ud589 \uc911\n\uc900\ube44 \uc911     :                                                                                                                1\/1 \n\uad6c\ud604 \uc911     : cephadm-2:18.2.7-2.el9s.noarch                                                                                 1\/1 \n\uc124\uce58 \uc911     : cephadm-2:18.2.7-2.el9s.noarch                                                                                 1\/1 \n\uad6c\ud604 \uc911     : cephadm-2:18.2.7-2.el9s.noarch                                                                                 1\/1 \n\ud655\uc778 \uc911     : cephadm-2:18.2.7-2.el9s.noarch                                                                                 1\/1 \n\n\uc124\uce58\ub418\uc5c8\uc2b5\ub2c8\ub2e4:\ncephadm-2:18.2.7-2.el9s.noarch                                                                                                   \n\n\uc644\ub8cc\ub418\uc5c8\uc2b5\ub2c8\ub2e4!<\/code><\/pre>\n<\/blockquote>\n\n\n\n
  2.2. bootstrap \ubc30\ud3ec (mon-ip ; \ubc30\ud3ec\ud558\ub294 node ip, cluster-ip\uac00 \uc0c1\uc774\ud55c \uacbd\uc6b0 –cluster-network \uc0ac\uc6a9.
   # cephadm bootstrap –mon-ip 10.10.10.191<\/p>\n\n\n\n
\n
[root@cephtest01 ~]# cephadm bootstrap –mon-ip 10.10.10.191
Creating directory \/etc\/ceph for ceph.conf
Verifying podman|docker is present\u2026
Verifying lvm2 is present\u2026
Verifying time synchronization is in place\u2026
Unit chronyd.service is enabled and running
Repeating the final host check\u2026
podman (\/usr\/bin\/podman) version 5.4.0 is present
systemctl is present
lvcreate is present
Unit chronyd.service is enabled and running
Host looks OK
Cluster fsid: 00468a4c-57da-11f0-8985-506b8db0fed8
Verifying IP 10.10.10.191 port 3300 \u2026
Verifying IP 10.10.10.191 port 6789 \u2026
Mon IP 10.10.10.191<\/code> is in CIDR network 10.10.10.0\/24<\/code>
Mon IP 10.10.10.191<\/code> is in CIDR network 10.10.10.0\/24<\/code>
Internal network (–cluster-network) has not been provided, OSD replication will default to the public_network
Pulling container image quay.io\/ceph\/ceph:v18\u2026
Ceph version: ceph version 18.2.7 (6b0e988052ec84cf2d4a54ff9bbbc5e720b621ad) reef (stable)
Extracting ceph user uid\/gid from container image\u2026
Creating initial keys\u2026
Creating initial monmap\u2026
Creating mon\u2026
Waiting for mon to start\u2026
Waiting for mon\u2026
mon is available
Assimilating anything we can from ceph.conf\u2026
Generating new minimal ceph.conf\u2026
Restarting the monitor\u2026
Setting public_network to 10.10.10.0\/24 in global config section
Wrote config to \/etc\/ceph\/ceph.conf
Wrote keyring to \/etc\/ceph\/ceph.client.admin.keyring
Creating mgr\u2026
Verifying port 0.0.0.0:9283 \u2026
Verifying port 0.0.0.0:8765 \u2026
Verifying port 0.0.0.0:8443 \u2026
Waiting for mgr to start\u2026
Waiting for mgr\u2026
mgr not available, waiting (1\/15)\u2026
mgr not available, waiting (2\/15)\u2026
mgr not available, waiting (3\/15)\u2026
mgr not available, waiting (4\/15)\u2026
mgr not available, waiting (5\/15)\u2026
mgr not available, waiting (6\/15)\u2026
mgr not available, waiting (7\/15)\u2026
mgr not available, waiting (8\/15)\u2026
mgr not available, waiting (9\/15)\u2026
mgr is available
Enabling cephadm module\u2026
Waiting for the mgr to restart\u2026
Waiting for mgr epoch 5\u2026
mgr epoch 5 is available
Setting orchestrator backend to cephadm\u2026
Generating ssh key\u2026
Wrote public SSH key to \/etc\/ceph\/ceph.pub
Adding key to root@localhost authorized_keys\u2026
Adding host cephtest01\u2026
Deploying mon service with default placement\u2026
Deploying mgr service with default placement\u2026
Deploying crash service with default placement\u2026
Deploying ceph-exporter service with default placement\u2026
Deploying prometheus service with default placement\u2026
Deploying grafana service with default placement\u2026
Deploying node-exporter service with default placement\u2026
Deploying alertmanager service with default placement\u2026
Enabling the dashboard module\u2026
Waiting for the mgr to restart\u2026
Waiting for mgr epoch 9\u2026
mgr epoch 9 is available
Generating a dashboard self-signed certificate\u2026
Creating initial admin user\u2026
Fetching dashboard port number\u2026
Ceph Dashboard is now available at:<\/p>\n\n\n\n
            URL: https:\/\/cephtest01:8443\/\n            User: admin\n        Password: jba2v2zr5h\n\nEnabling client.admin keyring and conf on hosts with \"admin\" label\nSaving cluster configuration to \/var\/lib\/ceph\/00468a4c-57da-11f0-8985-506b8db0fed8\/config directory\nEnabling autotune for osd_memory_target\nYou can access the Ceph CLI as following in case of multi-cluster or non-default config:\n\n        sudo \/usr\/sbin\/cephadm shell --fsid 00468a4c-57da-11f0-8985-506b8db0fed8 -c \/etc\/ceph\/ceph.conf -k \/etc\/ceph\/ceph.client.admin.keyring\n\nOr, if you are only running a single cluster on this host:\n\n        sudo \/usr\/sbin\/cephadm shell \n\nPlease consider enabling telemetry to help improve Ceph:\n\n        ceph telemetry on\n\nFor more information see:\n\n        https://docs.ceph.com\/en\/latest\/mgr\/telemetry\/\n\nBootstrap complete.<\/code><\/pre>\n<\/blockquote>\n\n\n\n
– \uc124\uce58 \ud6c4, \ucee8\ud14c\uc774\ub108 \uc0c1\ud0dc. \uc544\uc9c1 bootstrap \uc11c\ubc84\uc5d0\uc11c\ub9cc \ud655\uc778\uac00\ub2a5.
 \ubc30\ud3ec\ub418\uace0 \ub098\uba74 \/etc\/ceph\/\uc544\ub798 \ud30c\uc77c\uc774 \uc0dd\uc131\ub428. \/etc\/ceph\/ceph.conf, \/etc\/ceph\/ceph.pub, \/etc\/ceph\/ceph.client.admin.keyring \ub4f1.
ceph -s \uba85\ub839\uc5b4\ub85c \uc0c1\ud0dc\ud655\uc778. (cephadm shell\uc5d0\uc11c \uac00\ub2a5)
<\/p>\n\n\n\n
\"\"<\/figure>\n\n\n\n
3. \ub2e4\ub978 node \ucd94\uac00, ceph\uc758 \uacbd\uc6b0 replication set\uc774 \ucd5c\uc18c 3\uc744 \uad8c\uc7a5\ud558\ubbc0\ub85c 3\uac1c\uc774\uc0c1\uc758 node\uc5d0 \ubc30\ud3ec\ud558\ub294 \uac83\uc744 \uad8c\uc7a5.
  3.1. \ubc30\ud3ec\ub178\ub4dc\uc5d0\uc11c bootstrap\ub85c \uc0dd\uc131\ub41c pub key\ub97c \uac01 node\uc5d0 \ubcf5\uc0ac
   # ssh-copy-id -f -i \/etc\/ceph\/ceph.pub root@10.10.10.192 ;; \ubc30\ud3ec\ud560 node IP<\/p>\n\n\n\n
\n
[root@cephtest01 ~]# ssh-copy-id -f -i \/etc\/ceph\/ceph.pub root@10.10.10.192\n\/usr\/bin\/ssh-copy-id: INFO: Source of key(s) to be installed: \"\/etc\/ceph\/ceph.pub\"\nThe authenticity of host '10.10.10.192 (10.10.10.192)' can't be established.\nED25519 key fingerprint is SHA256:4RR1aWJ25aMcAv609P6YAA5\/fsBRB+4dnINZCHziaeY.\nThis key is not known by any other names\nAre you sure you want to continue connecting (yes\/no\/[fingerprint])? yes\nroot@10.10.10.192's password: \nPermission denied, please try again.\nroot@10.10.10.192's password: \n\nNumber of key(s) added: 1\n\nNow try logging into the machine, with:   \"ssh 'root@10.10.10.192'\"\nand check to make sure that only the key(s) you wanted were added.\n\n[root@cephtest01 ~]# ssh-copy-id -f -i \/etc\/ceph\/ceph.pub root@10.10.10.193\n\/usr\/bin\/ssh-copy-id: INFO: Source of key(s) to be installed: \"\/etc\/ceph\/ceph.pub\"\nThe authenticity of host '10.10.10.193 (10.10.10.193)' can't be established.\nED25519 key fingerprint is SHA256:4RR1aWJ25aMcAv609P6YAA5\/fsBRB+4dnINZCHziaeY.\nThis host key is known by the following other names\/addresses:\n    ~\/.ssh\/known_hosts:1: 10.10.10.192\nAre you sure you want to continue connecting (yes\/no\/[fingerprint])? yes\nroot@10.10.10.193's password: \n\nNumber of key(s) added: 1\n\nNow try logging into the machine, with:   \"ssh 'root@10.10.10.193'\"\nand check to make sure that only the key(s) you wanted were added.<\/code><\/pre>\n<\/blockquote>\n\n\n\n
  3.2. ceph node \ucd94\uac00.  !! ceph cluster \uad00\ub828\ub41c \uba85\ub839\uc5b4\ub294 \ubc30\ud3ec\ub41c ceph cluster\uc5d0\uc11c \ud574\uc57c\ub428.<\/strong>
   # cephadm shell –fsid ceph-cluster-fsid -c \/etc\/ceph\/ceph.conf -k \/etc\/ceph\/ceph.client.admin.keyring
   -> cephadm shell \ub610\ub294 \uc0c1\uae30 fsid, config, keyring \ud30c\uc77c \uc9c0\uc815\ud558\uc5ec ceph shell \uc9c4\uc785\ud558\uc5ec \uc2e4\ud589
   (fsid\ub294 ceph -s \ub610\ub294 ceph.conf \uc5d0\uc11c \ud655\uc778)
   [ceph: <\/strong>root@cephtest01 \/]# -> cephadm shell \uc2e4\ud589\uc2dc \ud504\ub86c\ud504\ud130 \uc55e\uc5d0 ceph: \uc774 \ubd99\uc74c.
 
  – ceph orch host add \uba85\ub839\uc5b4\ub85c host add, add\ub418\uba74 \ucee8\ud14c\uc774\ub108\ub97c \ud1b5\ud574 ceph cluster \uad6c\uc131\uc694\uc18c\ub97c \ubc30\ud3ec.
   [ceph: root@cephtest01 \/]# ceph orch host add cephtest02 10.10.10.192
   Added host ‘cephtest02’ with addr ‘10.10.10.192’
   [ceph: root@cephtest01 \/]# ceph orch host add cephtest03 10.10.10.193
   Added host ‘cephtest03’ with addr ‘10.10.10.193’<\/p>\n\n\n\n
4. Disk\ub97c OSD\ub85c \ucd94\uac00 \ubc0f cephfs \uc0dd\uc131, mount
  – physical disk\ub97c OSD \ub85c \ucd94\uac00, data pool, metadata pool \uc0dd\uc131, ceph fs \uc0dd\uc131, mount
  4.1. OSD\ub85c \uc0ac\uc6a9\uac00\ub2a5\ud55c disk \ud655\uc778
   cephadm shell\uc5d0\uc11c # ceph orch device ls (\uc774\ud6c4 cephadm shell\uc5d0\uc11c \uc2e4\ud589\uae30\uc900.)<\/mark><\/strong>
   \ub610\ub294 
   # cephadm shell — ceph orch device ls
  !! OSD \uc0dd\uc131\uc774\ud6c4 ls\ud574\uc11c AVAILABLE\uc5d0 yes\ub77c\uace0 \ub098\uc624\uc9c0 \uc54a\uc74c. OSD \uc0ac\uc6a9\uac00\ub2a5\ud558\uba74 AVAILABLE “Yes”\ub85c \ud45c\uae30.<\/p>\n\n\n\n
\"\"<\/figure>\n\n\n\n
  4.2. reef, cephadm\uc758 \uacbd\uc6b0 OSD\ub97c \uac1c\ubcc4\uc801\uc73c\ub85c \ub123\uc9c0 \uc54a\uace0 \uc815\ucc45\uc73c\ub85c OSD \uc0dd\uc131
   # ceph orch apply osd –all-available-devices
   ;; \uc0ac\uc6a9\uac00\ub2a5\ud55c \ubaa8\ub4e0 \ub514\uc2a4\ud06c\ub97c OSD\ub85c \uc790\ub3d9 \uc0ac\uc6a9.
  !!! ceph orch apply osd cephtest02:\/dev\/sdb -> device ls\uc5d0\uc11c \ub098\uc628 host\/path\ub97c \uc774\uc6a9\ud574 \uac1c\ubcc4\uc9c0\uc815.
   \ub610\ub294 –format \ub97c \uc0ac\uc6a9\ud558\uc5ec \ud2b9\uc815 \ub178\ub4dc, \ud2b9\uc815 path, \ud2b9\uc815 type\ub3c4 \uac00\ub2a5.

   ceph osd df \/ ceph osd tree \ub4f1\uc73c\ub85c OSD \uc0c1\ud0dc \ud655\uc778 \uac00\ub2a5.<\/p>\n\n\n\n
\"\"<\/figure>\n\n\n\n
5. cephfs \uc0dd\uc131 \ubc0f \ub9c8\uc6b4\ud2b8
  5.1. bootstrap node\uc5d0\uc11c cephadm shell\uc5d0\uc11c data pool, metadata pool \uc0dd\uc131
   # ceph osd pool create cephfs_data 3
   (‘cephfs_data’ \ud480, \ubcf5\uc81c\ubcf8 3\uac1c, DATA\ub294 \ubcf5\uc81c\ubcf8 2\uac1c \uc774\uc0c1)
   # ceph osd pool create cephfs_metadata 3
   (‘cephfs_metadata’ \ud480, \ubcf5\uc81c\ubcf8 3\uac1c, metadata\ub294 \ubcf5\uc81c\ubcf8 3\uac1c \uc774\uc0c1)<\/p>\n\n\n\n

  5.2. ceph fs \uc0dd\uc131, \uc2e4\uc81c \ub9c8\uc6b4\ud2b8\ub418\ub294 fs
   # ceph fs new cephfs cephfs_metadata cephfs_data  
   (ceph fs new <fs_name> <metadata_pool_name> <data_pool_name>  <\/p>\n\n\n\n
\n
# ceph fs new cephfs cephfs_metadata cephfs_data<\/p>\n\n\n\n
Pool ‘cephfs_data’ (id ‘2’) has pg autoscale mode ‘on’ but is not marked as bulk.
Consider setting the flag by running
    # ceph osd pool set cephfs_data bulk true
new fs with metadata pool 3 and data pool 2<\/p>\n<\/blockquote>\n\n\n\n
  5.3. mds (Metadata Server) \uc0dd\uc131 
  (\ud30c\uc77c \uc774\ub984, \ub514\ub809\ud130\ub9ac \ud2b8\ub9ac, \uc811\uadfc \uad8c\ud55c, \uc2ec\ubcfc\ub9ad \ub9c1\ud06c \ub4f1\uc758 \uba54\ud0c0\ub370\uc774\ud130\ub97c \ub2f4\ub2f9<\/strong>\ud558\ub294 \uc11c\ubc84, 
  \ud30c\uc77c\uc774 \uc5b4\ub514\uc5d0 \uc800\uc7a5\ub418\ub294\uc9c0 \uc54c\ub824\uc90c.)
   # ceph orch apply mds cephfs
   Scheduled mds.cephfs update…
<\/p>\n\n\n\n
  !! mds\uc758 \uac1c\uc218\ub294 max_mds \uac12\uacfc mds \ubc30\ud3ec \uba85\ub839\uc5b4\ub85c \uc870\uc815.
  max_mds \uac12\uc740 active mds\uc758 \uac12. Default\ub85c \uc704\uc640\uac19\uc774 \ubc30\ud3ec\ud558\uba74 1 active + 1 standby, max_mds \uac12\uc740 1.
  mds\ub97c \ub354 \ub123\uc5b4\uc11c mds \ubd84\uc0b0\uc744 \ud558\uace0 \uc2f6\ub2e4\uba74 max_mds\ub97c \uc218\uc815\ud558\uace0 \ucd94\uac00\ub85c \ubc30\ud3ec \ud544\uc694.
  \ub2e8. standby mds\uac00 1\uac1c\ub3c4 \uc5c6\uc73c\uba74 \uacbd\uace0\ub098\uc634.(max 2\uac1c \ubc30\ud3ec\ub3c4 2\uac1c\uba74 active\ub9cc 2\uac1c \ub418\ub108 \uacbd\uace0.)<\/p>\n\n\n\n
\n
max_mds \uc218\uc815 
(mds 3\uac1c \ubc30\ud3ec\ud558\uace0 max_mds \uac00 1\uc774\uba74 active 1\/standby 2\uac1c\ub85c \uc790\ub3d9 \uc870\uc815\ub428. max_mds 2\uc774\uba74 active 2\/standby 1\uac1c\ub85c \uc790\ub3d9 \uc870\uc815)
  [ceph: root@cephtest01 \/]# ceph fs set cephfs max_mds 1
<\/p>\n\n\n\n
mds \ubc30\ud3ec (\uac1c\uc218 \ub610\ub294 node)
  [ceph: root@cephtest01 \/]# ceph orch apply mds cephfs –placement=”cephtest01 cephtest02 cephtest03″
  Scheduled mds.cephfs update\u2026
\ub610\ub294
  [ceph: root@cephtest01 \/]# ceph orch apply mds cephfs –placement=”3″
  Scheduled mds.cephfs update\u2026<\/p>\n<\/blockquote>\n\n\n\n
  \uc804\uccb4 \ub0b4\uc6a9 \ud655\uc778 ceph -s \ub85c \ud655\uc778 \uac00\ub2a5. <\/p>\n\n\n\n
\n
[ceph: root@cephtest01 \/]# ceph -s
cluster:
id: 00468a4c-57da-11f0-8985-506b8db0fed8
health: HEALTH_OK
services:
    mon: 3 daemons, quorum cephtest01,cephtest02,cephtest03 (age 23m)
    mgr: cephtest01.qwcvel(active, since 49m), standbys: cephtest02.jgcctu
    mds: 1\/1 daemons up, 1 standby
    osd: 6 osds: 6 up (since 23m), 6 in (since 24m)

data:
    volumes: 1\/1 healthy
    pools:   3 pools, 65 pgs
    objects: 24 objects, 579 KiB
    usage:   197 MiB used, 600 GiB \/ 600 GiB avail
    pgs:     65 active+clean

[ceph: root@cephtest01 \/]# ceph fs ls
name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]

[ceph: root@cephtest01 \/]# ceph fs status
cephfs – 0 clients
======
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active cephfs.cephtest03.wkggmh Reqs: 0 \/s 10 13 12 0
POOL TYPE USED AVAIL
cephfs_metadata metadata 96.0k 189G
cephfs_data data 0 189G
STANDBY MDS
cephfs.cephtest01.anylgd
MDS version: ceph version 18.2.7 (6b0e988052ec84cf2d4a54ff9bbbc5e720b621ad) reef (stable)

!! mds \ubc30\ud3ec\uac1c\uc218 2\uac1c\ub85c \ub298\ub9ac\uae30
[ceph: root@cephtest01 \/]# ceph fs set cephfs max_mds 2
[ceph: root@cephtest01 \/]# ceph -s
cluster:
id: 00468a4c-57da-11f0-8985-506b8db0fed8
health: HEALTH_WARN
insufficient standby MDS daemons available<\/p>\n\n\n\n
services:
mon: 3 daemons, quorum cephtest01,cephtest02,cephtest03 (age 18h)
mgr: cephtest01.qwcvel(active, since 19h), standbys: cephtest02.jgcctu
mds: 2\/2 daemons up
osd: 6 osds: 6 up (since 18h), 6 in (since 18h)<\/p>\n\n\n\n
data:
volumes: 1\/1 healthy
pools: 3 pools, 65 pgs
objects: 24 objects, 579 KiB
usage: 251 MiB used, 600 GiB \/ 600 GiB avail
pgs: 65 active+clean<\/p>\n<\/blockquote>\n\n\n\n
6. \ud30c\uc77c\uc2dc\uc2a4\ud15c \ub9c8\uc6b4\ud2b8
  6.1. \ub9c8\uc6b4\ud2b8\ub97c \uc704\ud574\uc11c\ub294 “ceph-common” \uad6c\uc131\uc694\uc18c\uac00 \ud544\uc694\ud558\ub2e4. (mount \ud544\uc694\ud55c \ubaa8\ub4e0 node.)
  Rocky9 \ud658\uacbd\uc5d0\uc11c \uc758\uc874\uc131 \ubb38\uc81c\ub85c –nobest \uc635\uc158\uc73c\ub85c \uc124\uce58\ud568. (\uc791\ub3d9\ud14c\uc2a4\ud2b8 \uc815\uc0c1)
   [root@cephtest02 ~]# dnf install ceph-common –nobest

  6.2. ceph.conf \ud30c\uc77c \ubc0f \uc778\uc99d keyring \ud30c\uc77c clients\ub85c \ubcf5\uc0ac
  \uc778\uc99d keyring\uc758 \uacbd\uc6b0 \uae30\ubcf8 admin keyring\ub97c \uc0ac\uc6a9\ud560 \uc218\ub3c4 \uc788\uc73c\ub098, ceph cluster\uc758 \ubaa8\ub4e0 \uad8c\ud55c\uc744 \uac00\uc9c0\ubbc0\ub85c
  \uac00\ub2a5\ud558\uba74 \ubcc4\ub3c4 \uad8c\ud55c\uc81c\ud55c\ud0a4\ub97c \uc0ac\uc6a9\ud558\uc5ec \ub9c8\uc6b4\ud2b8.
  \ud0a4\uc0dd\uc131
   # ceph auth get-or-create client.cephfsuser <\/strong>mon ‘allow r’ mds ‘allow r, allow rw path=\/’ osd ‘allow rw pool=cephfs_data<\/strong>, allow rwx pool=cephfs_metadata<\/strong>‘ > ceph.client.cephfsuser<\/strong>.keyring
   ;; cephfsuser (\uc720\uc800\uc9c0\uc815), data pool\uc740 rw, metadata pool\uc740 rwx, \ud0a4 \uc774\ub984\uc740 ceph.client.<username>.keyring\ub85c \uac01 clients\uc758 \/etc\/ceph\/ \uc544\ub798 \uc704\uce58.
   # chmod 600 ceph.client.cephfsuser.keyring
   # mv ceph.client.cephfsuser.keyring \/etc\/ceph\/ceph.client.cephfsuser.keyring
   \ud0a4 \ubc0f \uc124\uc815 \ubcf5\uc0ac
   # scp \/etc\/ceph\/ceph.conf 10.10.10.192:\/etc\/ceph\/ceph.conf
   # scp \/etc\/ceph\/ceph.client.cephfsuser.keyring 10.10.10.192:\/etc\/ceph\/ceph.client.cephfsuser.keyring

  6.3. mount.ceph\ub97c \ud1b5\ud574 \ub9c8\uc6b4\ud2b8 (mount -t ceph \ub3c4 \uac00\ub2a5\ud558\ub098 \uc635\uc158\uc774 \uc0c1\uc774\ud568)
   # mount.ceph cephfsuser@.cephfs=\/ \/cephfs
   ;; mount.ceph <username>@.<ceph_fs_name>=<dir> <mount_point>
   ;; <dir>\uc744 “\/”\ub85c \ud558\uba74 \uc804\uccb4 cephfs \ub9c8\uc6b4\ud2b8, \ud558\uc704\ub514\ub809\ud1a0\ub9ac \ubcc4\ub3c4 \ub9c8\uc6b4\ud2b8\ub3c4 \uac00\ub2a5.<\/p>\n\n\n\n
\n
[root@cephtest01 ~]# mount.ceph cephfsuser@.cephfs=\/ \/cephfs
[root@cephtest01 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 4.0M 0 4.0M 0% \/dev
tmpfs 7.7G 100K 7.7G 1% \/dev\/shm
tmpfs 3.1G 178M 2.9G 6% \/run
\/dev\/mapper\/rl-root 100G 8.1G 92G 9% \/
\/dev\/sda2 436M 413M 24M 95% \/boot
tmpfs 1.6G 52K 1.6G 1% \/run\/user\/42
tmpfs 1.6G 36K 1.6G 1% \/run\/user\/0
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/8aecd5bb54c78897521030679dd692b88aa74c39e783a6c8897e2d77fc2dc33a\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/458f5f9b878af138d8e5ead078b316bb6ae8aaa92e75f5c320d5fdf031923b90\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/fd6a1c01f8f3e83b0f0fb4145c38fc52f3c85ba847a0084ec51be32655f31ae7\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/6176bc236bb42b08b63be48041ae7332fc247b2dd49420c3a380d34509504f19\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/b96fd2e91ec678f7a34c24cc6d7c51617938a69fd735dff7deafbbd194849338\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/79977ae8d4b6cc8c958a8445366e056ef039ac5cc048d0adec3e3a6b9a553198\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/db4ac1084218bd2c0fd54a8382fd4d446a6c0c7fa0ca1efb84c3b56909f15522\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/eef64fb45b4f7750bc1004c99ef750217e8d055eeb9e9d8329251d4eee25a753\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/ed87435c471c4a3b686566d6b58176ddd60668ca80f09128812092fadb744e03\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/06aef07d5d0f259e1485b8ee9580ea4a9c9a66187ea79a75fe45af3c31f5159c\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/73efe72defdd883e5361446d4f36772bb9bef1eaa829e23fbfe0a1d7436d572d\/merged
overlay 100G 8.1G 92G 9% \/var\/lib\/containers\/storage\/overlay\/6a56c658b5a9929399ba8a91234f13e1e783f2e76b5d108d57a8368d3018f737\/merged
cephfsuser@00468a4c-57da-11f0-8985-506b8db0fed8.cephfs=\/ 190G 0 190G 0% \/cephfs<\/p>\n<\/blockquote>\n\n\n\n
   !! \/etc\/fstab\uc5d0 \ud3ec\ud568\ud558\uc5ec \ubd80\ud305\uc2dc \ub9c8\uc6b4\ud2b8
     # vi \/etc\/fstab
     # ceph fs mount
     cephfsuser@00468a4c-57da-11f0-8985-506b8db0fed8.cephfs=\/ \/cephfs ceph name=cephfsuser,_netdev 0 2
    \ub610\ub294
    10.10.10.191:6789:\/ \/cephfs ceph name=cephfsuser,_netdev 0 2
    !! \ub450\uac00\uc9c0 \ubaa8\ub450 \ub9c8\uc6b4\ud2b8 \ud655\uc778, FSID\ub97c \uc774\uc6a9\ud558\ub294 \uac83\uc774 \ub354 \uc548\uc815\uc801\uc73c\ub85c \ubcf4\uc784.
    \ubcc0\uacbd \ud6c4, systemctl daemon-reload \ub85c fstab \uc124\uc815 \uc77d\uae30 \ud544\uc694. 
<\/p>\n\n\n\n
** \uacb0\ub860 !!
\uac04\ub2e8\ud55c \uc131\ub2a5
NFS\ub85c \ub9c8\uc6b4\ud2b8\ub41c \ub514\uc2a4\ud06c\uc640 \ube44\uc2b7\ud55c \uc131\ub2a5. (\ub124\ud2b8\uc6cc\ud06c 1G\uc758 \ud55c\uacc4?)
\ubcf5\uc0ac\uc911\uc5d0 CPU \uc0ac\uc6a9\ub7c9 10~20%\uc99d\uac00, \uba54\ubaa8\ub9ac \uc0ac\uc6a9\ub7c9\uc740 ceph \uc124\uc815\uc644\ub8cc \ud6c4 \ud070 \ubcc0\ud654\uc5c6\uc74c.
\ub85c\uceec \ub514\uc2a4\ud06c \ubcf5\uc0ac \uc131\ub2a5\uacfc \ube44\uad50\ud558\uace0 \uc2f6\uc5c8\uc73c\ub098, HCI\ub97c \uc0ac\uc6a9\ud574 VM\uc73c\ub85c \ud14c\uc2a4\ud2b8\ud55c \uad00\uacc4\ub85c \ub85c\uceec \ub514\uc2a4\ud06c \ubcf5\uc0ac\ub294 \ub85c\ub4dc\uac00 \uac70\uc758 \uac78\ub9ac\uc9c0 \uc54a\uc74c. HCI dedup\uc758 \uc601\ud5a5\uc73c\ub85c \ubcf4\uc784.<\/p>\n\n\n\n
\"\"<\/figure>\n\n\n\n
<\/p>\n\n\n\n
!! 1\uac1c node rebooting \uc2dc \ud558\uae30\uc640 \uac19\uc740 \uc0c1\ud0dc.
  \ud2b9\uc774\uc0ac\ud56d\uc73c\ub85c\ub294 1\uac1c node\uac00 \uc8fd\uc740 \uc0c1\ud0dc\uc5d0\uc11c copy \uc18d\ub3c4\uac00 \ub354 \ube60\ub984. \uc57d 1\/2. \ubcf5\uc81c\ud574\uc57c\ud560 \uac1c\uc218\uac00 3\uac1c\uc5d0\uc11c 2\uac1c\ub85c \uc904\uc5b4\uc11c \uc77c\uc2dc\uc801\uc73c\ub85c \uc18d\ub3c4\uac00 \ube68\ub77c\uc9c0\ub294 \uac83\uc73c\ub85c \ucd94\uc815. \ub178\ub4dc \ubcf5\uad6c\ud6c4 \ubc31\uadf8\ub77c\uc6b4\ub4dc \ubcf5\uc81c \uc791\uc5c5\uc774 \ubc1c\uc0dd\ud560 \uac83\uc73c\ub85c \uc608\uc0c1.
\ub2e4\uc218\uc758 \ud14c\uc2a4\ud2b8\uac00 \uc544\ub2c8\uae30 \ub54c\ubb38\uc5d0 \ucc38\uc870.<\/p>\n\n\n\n
\n
[root@cephtest01 log]# ceph -s
cluster:
id: 00468a4c-57da-11f0-8985-506b8db0fed8
health: HEALTH_WARN
insufficient standby MDS daemons available
1\/3 mons down, quorum cephtest01,cephtest02
2 osds down
1 host (2 osds) down
Degraded data redundancy: 1608\/30900 objects degraded (5.204%), 14 pgs degraded<\/p>\n\n\n\n
services:
mon: 3 daemons, quorum cephtest01,cephtest02 (age 6s), out of quorum: cephtest03
mgr: cephtest01.qwcvel(active, since 4d), standbys: cephtest02.jgcctu
mds: 2\/2 daemons up
osd: 6 osds: 4 up (since 5s), 6 in (since 4d)<\/p>\n\n\n\n
data:
volumes: 1\/1 healthy
pools: 3 pools, 65 pgs
objects: 10.30k objects, 40 GiB
usage: 122 GiB used, 478 GiB \/ 600 GiB avail
pgs: 1608\/30900 objects degraded (5.204%)
35 active+clean
15 stale+active+clean
14 active+undersized+degraded
1 active+undersized<\/p>\n<\/blockquote>\n\n\n\n
7. PG \uac1c\uc218 (ceph\uc758 \uacbd\uc6b0 fs\/block\/object \ubaa8\ub450 \uacb0\ub860\uc801\uc73c\ub85c object(\ub370\uc774\ud130\uac1d\uccb4)\ub85c \uc800\uc7a5\ub41c\ub2e4. \uc774 object\uc758 \ubc30\uce58\uadf8\ub8f9\uc774 PG(Placement Group).
-> client\uac00 \uc800\uc7a5\ud558\ub294 \ubaa8\ub4e0 object\ub294 PG\ub2e8\uc704\ub85c \uadf8\ub8f9\ud654\ub418\uc5b4 \uad00\ub9ac \ub428. CRUSH MAP\uc5d0 \uc758\ud574 \uac01 OSD\uc5d0 \ubd84\uc0b0 \uc800\uc7a5.
-> Pool\uc5d0\uc11c \uc774 PG\uc758 \uac1c\uc218\ub294 \uc131\ub2a5\uc5d0 \ud070 \uc601\ud5a5\uc744 \ubbf8\uce58\uba70 \uac1d\uccb4 \uc22b\uc790, OSD \uc22b\uc790\uc5d0 \ub530\ub77c \uc9c0\uc815\ud574\uc57c \uc131\ub2a5\ud5a5\uc0c1\uc774 \uac00\ub2a5\ud558\ub2e4. OSD\ub2f9 100-200\uc0ac\uc774\uac00 \uc801\ub2f9\ud558\ub2e4\uace0 \ud568.
\ucc38\uc870 : https:\/\/docs.redhat.com\/en\/documentation\/red_hat_ceph_storage\/3\/html\/storage_strategies_guide\/index<\/a> 
\uc758 PGs \ubd80\ubd84 \ucc38\uc870.
\uacc4\uc0b0\uae30 : https:\/\/docs.redhat.com\/en\/documentation\/red_hat_ceph_storage\/3\/html\/storage_strategies_guide\/placement_groups_pgs#pg_calculator<\/a><\/p>\n\n\n\n
!! reef \ubc84\uc804\uc5d0\uc11c\ub294 autoscale \uc9c0\uc6d0 pg\ub97c \uc790\ub3d9\uc73c\ub85c \ub298\ub9ac\uac70\ub098 \uc904\uc784.
\ud655\uc778 \uba85\ub839\uc5b4.
[ceph: root@cephtest01 \/]# ceph osd pool autoscale-status<\/p>\n\n\n\n
\"\"<\/figure>\n\n\n\n
pg \ubc0f pgp \ud655\uc778 (ceph osd pool POOLNAME pg_num)
pg : pool\uc5d0 \uc0dd\uc131\ub418\uc5b4 \uc788\ub294 pg\uc758 \ucd1d\uac1c\uc218.
pgp : CRUSH \uc54c\uace0\ub9ac\uc998\uc774 \uc2e4\uc81c\ub85c OSD\uc5d0 \ub9e4\ud551\ud560 \ub54c \uc0ac\uc6a9\ud558\ub294 PG \uc218.
;; \ub450 \uac1c\uac00 \ubcf4\ud1b5\uc740 \ub3d9\uc77c.

[ceph: root@cephtest01 \/]# ceph osd pool get cephfs_data pg_num
pg_num: 32
[ceph: root@cephtest01 \/]# ceph osd pool get cephfs_data pgp_num
pgp_num: 32<\/p>\n\n\n\n
CEPHFS\/Object\/Block Service \uc81c\uacf5.
RBD (RADOS Block Device) ; block device\ub97c ceph RADOS\uc5d0 \uc5f0\uacb0
RGW (RADOS Gateway) ; client object \uc811\uadfc\uc744 ceph RADOS\uc5d0 \uc5f0\uacb0
MDS(Metadata Server) ; cephfs client\ub97c ceph RADOS\uc5d0 \uc5f0\uacb0
MON (Monitor)
MGR (Manager)
librados ; RADOS\uc5d0 \uc811\uadfc\ud558\uae30 \uc704\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac, cephfs\ub294 MDS\ub97c \ud1b5\ud574 RADOS\uc5d0 \uc9c1\uc811 \uc811\uadfc \uac00\ub2a5
RADOS(Reliable Autonomic Distributed Object Store) ; CRUSH \uc54c\uace0\ub9ac\uc998\uc744 \ud1b5\ud574 \ub370\uc774\ud130 object\ub97c PG-OSD \uc5d0 \ubd84\uc0b0\uc800\uc7a5\ud558\ub294 Ceph \ud575\uc2ec \uc544\ud0a4\ud14d\uccd0, api\ub97c \uc81c\uc678\ud55c \ub300\ubd80\ubd84\uc758 ceph \uad6c\uc131\uc694\uc18c\uac00 \uc5ec\uae30\uc5d0 \uc18d\ud568. 
CRUSH Map(Controlled Replication Under Scalable Hashing) ; \ub370\uc774\ud130 \ubd84\uc0b0 \uc54c\uace0\ub9ac\uc998
PG(Placement Group) ; \ub370\uc774\ud130 object \uc758 \ubb36\ub294 \ub17c\ub9ac\ub2e8\uc704, PG \ub2e8\uc704\ub85c OSD\uc5d0 \ubd84\uc0b0\uc800\uc7a5
PGP(Placement Group for Placement) ; OSD\uc5d0 \ubb3c\ub9ac\uc801\uc73c\ub85c \ub9e4\ud551\ub418\ub294 PG \uac1c\uc218(pgp\ub294 pg\ubcf4\ub2e4 \ud074 \uc218 \uc5c6\uc74c.)
OSD(Object Storage Daemon)

<\/p>\n\n\n\n
<\/p>\n","protected":false},"excerpt":{"rendered":"
!! ceph reef \ubc84\uc804\ubd80\ud130\ub294 cephadm\uc744 \uc81c\uacf5\ud558\uc5ec \uad00\ub9ac\uba74\uc5d0\uc11c \ud3b8\uc758\uc131\uc744 \uc81c\uacf5. \ubc30\ud3ec\ub294 \ubaa8\ub450 Container\ub97c \ud1b5\ud574 \ubc30\ud3ec\ub418\uba70 \uc791\ub3d9 \uc5ed\uc2dc \ub9c8\ucc2c\uac00\uc9c0\ub85c \ucee8\ud14c\uc774\ub108\uae30\ubc18. 1. \uc0ac\uc804\ud655\uc778\uc0ac\ud56d 1.1. dnf update.(OS current \ubc84\uc804 […]<\/p>\n","protected":false},"author":1,"featured_media":601,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[117,10,11,200,162],"class_list":["post-580","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-tech","tag-config","tag-install","tag-linux","tag-osd","tag-rocky"],"_links":{"self":[{"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=\/wp\/v2\/posts\/580","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=580"}],"version-history":[{"count":17,"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=\/wp\/v2\/posts\/580\/revisions"}],"predecessor-version":[{"id":608,"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=\/wp\/v2\/posts\/580\/revisions\/608"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=\/wp\/v2\/media\/601"}],"wp:attachment":[{"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=580"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=580"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/iwgh.duckdns.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=580"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}