二、使用ceph-deploy部署ceph集群
1、实验环境
系统版本：ubuntu 18.04.5 LTS 内核参数：4.15.0-112-generic ceph版本：pacific/16.2.7 主机分配： #部署服务器ceph-deploy 192.168.1.200/192.168.88.200 ceph-deploy #两个ceph-mgr 管理服务器 192.168.1.201/192.168.88.201 ceph-mgr1 ceph-rgw2 192.168.1.202/192.168.88.202 ceph-mgr2 ceph-rgw2 #三台服务器作为ceph 集群Mon 监视服务器，每台服务器可以和ceph 集群的cluster 网络通信。 192.168.1.203/192.168.88.203 ceph-mon1 ceph-mds1 192.168.1.204/192.168.88.204 ceph-mon2 ceph-mds2 192.168.1.205/192.168.88.205 ceph-mon3 ceph-mds3 #四台服务器作为ceph 集群OSD 存储服务器，每台服务器支持两个网络，public 网络针对客户端访问，cluster 网络用于集群管理及数据同步，每台三块或以上的磁盘 192.168.1.206/192.168.88.206 ceph-node1 192.168.1.207/192.168.88.207 ceph-node2 192.168.1.208/192.168.88.208 ceph-node3 192.168.1.209/192.168.88.209 ceph-node4 #磁盘划分 #/dev/sdb /dev/sdc /dev/sdd #10G
2.1、所有节点配置清华源
cat >/etc/apt/source.list<2.2、为所有节点安装常用工具
apt install iproute2 ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev gcc openssh-server iotop unzip zip apt-transport-https ca-certificates curl software-properties-common -y
cat >/etc/sysctl.conf <2.4、
cat > /etc/security/limits.conf <记得重启服务器
2.5、所有节点同步时间配置
#安装cron并启动 apt install cron -y systemctl status cron.service #同步时间 /usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w #每5分钟同步一次时间 echo "*/5 * * * * /usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w" >> /var/spool/cron/crontabs/root2.6、所有节点配置hosts
cat >> /etc/hosts << EOF 192.168.1.200 ceph-deploy 192.168.1.201 ceph-mgr1 ceph-rgw1 192.168.1.202 ceph-mgr2 ceph-rgw2 192.168.1.203 ceph-mon1 ceph-mds1 192.168.1.204 ceph-mon2 ceph-mds2 192.168.1.205 ceph-mon3 ceph-mds3 192.168.1.206 ceph-node1 192.168.1.207 ceph-node2 192.168.1.208 ceph-node3 192.168.1.209 ceph-node4 EOF2.7、所有节点安装python2.7
ceph初始化时，需要用到python2.7
apt install python2.7 -y ln -sv /usr/bin/python2.7 /usr/bin/python2
#配置ceph仓库 sudo echo "deb https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic main" >> /etc/apt/sources.list #导入key wget -q -O- 'https://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc' | sudo apt-key add - #更新仓库源 apt update
# 允许用户以特权身份执行命令
echo "cephadmin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
# 1.创建ssh密钥对 cephadmin@ceph-deploy:~$ ssh-keygen Generating public/private rsa key pair. Enter file in which to save the key (/home/cephadmin/.ssh/id_rsa): Created directory '/home/cephadmin/.ssh'. Enter passphrase (empty for no passphrase): Enter same passphrase again: Your identification has been saved in /home/cephadmin/.ssh/id_rsa. Your public key has been saved in /home/cephadmin/.ssh/id_rsa.pub. The key fingerprint is: SHA256:NpcJMRHsniqjckUYAYRt2tUK6yEik/9xE17Q4K5VjTY cephadmin@ceph-deploy The key's randomart image is: +---[RSA 2048]----+ |++.. ..+=o | |. = ..o o= | | = * ..oE . | |B = o..ooo o | |o= o .o+S.+ | | o oo+.oo | | o.o o | |. . + . | | o.. o | +----[SHA256]-----+ cephadmin@ceph-deploy:~$ # 2.安装sshpass cephadmin@ceph-deploy:~$ sudo apt install sshpass Reading package lists... Done Building dependency tree Reading state information... Done The following NEW packages will be installed: sshpass 0 upgraded, 1 newly installed, 0 to remove and 164 not upgraded. Need to get 10.5 kB of archives. After this operation, 30.7 kB of additional disk space will be used. Get:1 https://mirrors.tuna.tsinghua.edu.cn/ubuntu bionic/universe amd64 sshpass amd64 1.06-1 [10.5 kB] Fetched 10.5 kB in 1s (16.6 kB/s) Selecting previously unselected package sshpass. (Reading database ... 72596 files and directories currently installed.) Preparing to unpack .../sshpass_1.06-1_amd64.deb ... Unpacking sshpass (1.06-1) ... Setting up sshpass (1.06-1) ... Processing triggers for man-db (2.8.3-2ubuntu0.1) ... # 3.ceph-deploy节点使用cephadmin用户分发密钥脚本 #!/bin/bash #目标主机列表 IP=" 192.168.1.200 192.168.1.201 192.168.1.202 192.168.1.203 192.168.1.204 192.168.1.205 192.168.1.206 192.168.1.207 192.168.1.208 192.168.1.209" for node in ${IP};do sshpass -p 123456 ssh-copy-id cephadmin@${node} -o StrictHostKeyChecking=no &> /dev/null if [ $? -eq 0 ];then echo "${node}----> 密钥分发success完成" else echo "${node}----> 密钥分发false失败" fi done # 4.使用脚本分发ssh密钥 cephadmin@ceph-deploy:~$ bash ssh_fenfa.sh 192.168.1.200----> 密钥分发success完成 192.168.1.201----> 密钥分发success完成 192.168.1.202----> 密钥分发success完成 192.168.1.203----> 密钥分发success完成 192.168.1.204----> 密钥分发success完成 192.168.1.205----> 密钥分发success完成 192.168.1.206----> 密钥分发success完成 192.168.1.207----> 密钥分发success完成 192.168.1.208----> 密钥分发success完成 192.168.1.209----> 密钥分发success完成3.4、在ceph-deploy节点部署ceph-deploy工具包
cephadmin@ceph-deploy:~$ sudo apt-cache madison ceph-deploy ceph-deploy | 2.0.1 | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic/main amd64 Packages ceph-deploy | 2.0.1 | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic/main i386 Packages ceph-deploy | 1.5.38-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu bionic/universe amd64 Packages ceph-deploy | 1.5.38-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu bionic/universe i386 Packages cephadmin@ceph-deploy:~$ sudo apt install ceph-deploy3.5、初始化mon节点
cephadmin@ceph-deploy:~$ mkdir ceph-cluster #保存当前集群的初始化配置信息 cephadmin@ceph-deploy:~$ cd ceph-cluster/ cephadmin@ceph-deploy:~/ceph-cluster$前期只先初始化ceph-mon1节点，ceph-mon2和ceph-mon3在集群部署完成后，再手动添加
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy new --cluster-network 192.168.88.0/24 --public-network 192.168.1.0/24 ceph-mon1 [ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy new --cluster-network 192.168.88.0/24 --public-network 192.168.1.0/24 ceph-mon1 [ceph_deploy.cli][INFO ] ceph-deploy options: [ceph_deploy.cli][INFO ] username : None [ceph_deploy.cli][INFO ] verbose : False [ceph_deploy.cli][INFO ] overwrite_conf : False [ceph_deploy.cli][INFO ] quiet : False [ceph_deploy.cli][INFO ] cd_conf :[ceph_deploy.cli][INFO ] cluster : ceph [ceph_deploy.cli][INFO ] ssh_copykey : True [ceph_deploy.cli][INFO ] mon : ['ceph-mon1'] [ceph_deploy.cli][INFO ] func : [ceph_deploy.cli][INFO ] public_network : 192.168.1.0/24 [ceph_deploy.cli][INFO ] ceph_conf : None [ceph_deploy.cli][INFO ] cluster_network : 192.168.88.0/24 [ceph_deploy.cli][INFO ] default_release : False [ceph_deploy.cli][INFO ] fsid : None [ceph_deploy.new][DEBUG ] Creating new cluster named ceph [ceph_deploy.new][INFO ] making sure passwordless SSH succeeds [ceph-mon1][DEBUG ] connected to host: ceph-deploy [ceph-mon1][INFO ] Running command: ssh -CT -o BatchMode=yes ceph-mon1 [ceph_deploy.new][WARNIN] could not connect via SSH [ceph_deploy.new][INFO ] will connect again with password prompt The authenticity of host 'ceph-mon1 (192.168.1.203)' can't be established. ECDSA key fingerprint is SHA256:8pWk1yGdewC9FEqZOzCC10VMmCDSqDVKS7l6W3iiK6U. Are you sure you want to continue connecting (yes/no)? yes Warning: Permanently added 'ceph-mon1' (ECDSA) to the list of known hosts. [ceph-mon1][DEBUG ] connected to host: ceph-mon1 [ceph-mon1][DEBUG ] detect platform information from remote host [ceph-mon1][DEBUG ] detect machine type [ceph_deploy.new][INFO ] adding public keys to authorized_keys [ceph-mon1][DEBUG ] append contents to file [ceph-mon1][DEBUG ] connection detected need for sudo [ceph-mon1][DEBUG ] connected to host: ceph-mon1 [ceph-mon1][DEBUG ] detect platform information from remote host [ceph-mon1][DEBUG ] detect machine type [ceph-mon1][DEBUG ] find the location of an executable [ceph-mon1][INFO ] Running command: sudo /bin/ip link show [ceph-mon1][INFO ] Running command: sudo /bin/ip addr show [ceph-mon1][DEBUG ] IP addresses found: [u'192.168.1.203', u'240e:388:9f20:5700:a00:27ff:fe04:6dcb', u'192.168.88.203'] [ceph_deploy.new][DEBUG ] Resolving host ceph-mon1 [ceph_deploy.new][DEBUG ] Monitor ceph-mon1 at 192.168.1.203 [ceph_deploy.new][DEBUG ] Monitor initial members are ['ceph-mon1'] [ceph_deploy.new][DEBUG ] Monitor addrs are [u'192.168.1.203'] [ceph_deploy.new][DEBUG ] Creating a random mon key... [ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring... [ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf... 初始化为验证
cephadmin@ceph-deploy:~/ceph-cluster$ ll total 20 drwxrwxr-x 2 cephadmin cephadmin 4096 Mar 30 18:28 ./ drwxr-xr-x 6 cephadmin cephadmin 4096 Mar 30 18:01 ../ -rw-rw-r-- 1 cephadmin cephadmin 266 Mar 30 18:28 ceph.conf #自动生成的配置文件 -rw-rw-r-- 1 cephadmin cephadmin 3938 Mar 30 18:28 ceph-deploy-ceph.log #日志文件 -rw------- 1 cephadmin cephadmin 73 Mar 30 18:28 ceph.mon.keyring #用于ceph mon 节点内部通讯认证的秘钥环文件 cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf [global] fsid = 164ae280-496b-4272-8d79-04f94b9a3b5a #ceph集群ID public_network = 192.168.1.0/24 cluster_network = 192.168.88.0/24 mon_initial_members = ceph-mon1 #可以用逗号做分割添加多个mon节点 mon_host = 192.168.1.203 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx3.6、初始化ceph-node节点
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --no-adjust-repos --nogpgcheck ceph-node1 ceph-node2 ceph-node3 ceph-node4 --no-adjust-repos：不推送repo仓库 --nogpgcheck：不进行包的校验此过程会在指定的ceph node 节点按照串行的方式逐个服务器安装epel 源和ceph 源并安装ceph所需软件
............ [ceph-node4][DEBUG ] Adding group ceph....done [ceph-node4][DEBUG ] Adding system user ceph....done [ceph-node4][DEBUG ] Setting system user ceph properties....done [ceph-node4][DEBUG ] chown: cannot access '/var/log/ceph/*.log*': No such file or directory [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. [ceph-node4][DEBUG ] Setting up radosgw (16.2.7-1bionic) ... [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. [ceph-node4][DEBUG ] Setting up ceph-base (16.2.7-1bionic) ... [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. [ceph-node4][DEBUG ] Setting up ceph-mgr (16.2.7-1bionic) ... [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. [ceph-node4][DEBUG ] Setting up ceph-osd (16.2.7-1bionic) ... [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. [ceph-node4][DEBUG ] Setting up ceph-mds (16.2.7-1bionic) ... [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. [ceph-node4][DEBUG ] Setting up ceph-mon (16.2.7-1bionic) ... [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. [ceph-node4][DEBUG ] Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. [ceph-node4][DEBUG ] Setting up ceph (16.2.7-1bionic) ... [ceph-node4][DEBUG ] Processing triggers for systemd (237-3ubuntu10.42) ... [ceph-node4][DEBUG ] Processing triggers for man-db (2.8.3-2ubuntu0.1) ... [ceph-node4][DEBUG ] Processing triggers for ureadahead (0.100.0-21) ... [ceph-node4][DEBUG ] Processing triggers for libc-bin (2.27-3ubuntu1.2) ... [ceph-node4][INFO ] Running command: sudo ceph --version [ceph-node4][DEBUG ] ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)3.7、配置mon节点并生成同步密钥
在各mon 节点按照组件ceph-mon,并通初始化mon 节点，mon 节点ha 还可以后期横向扩容
root@ceph-mon1:~# apt install ceph-mon root@ceph-mon2:~# apt install ceph-mon root@ceph-mon3:~# apt install ceph-mon在ceph-deploy节点初始化mon节点
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon create-initial [ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mon create-initial [ceph_deploy.cli][INFO ] ceph-deploy options: [ceph_deploy.cli][INFO ] username : None [ceph_deploy.cli][INFO ] verbose : False [ceph_deploy.cli][INFO ] overwrite_conf : False [ceph_deploy.cli][INFO ] subcommand : create-initial [ceph_deploy.cli][INFO ] quiet : False [ceph_deploy.cli][INFO ] cd_conf :[ceph_deploy.cli][INFO ] cluster : ceph [ceph_deploy.cli][INFO ] func : [ceph_deploy.cli][INFO ] ceph_conf : None [ceph_deploy.cli][INFO ] keyrings : None [ceph_deploy.cli][INFO ] default_release : False [ceph_deploy.mon][DEBUG ] Deploying mon, cluster ceph hosts ceph-mon1 [ceph_deploy.mon][DEBUG ] detecting platform for host ceph-mon1 ... [ceph-mon1][DEBUG ] connection detected need for sudo [ceph-mon1][DEBUG ] connected to host: ceph-mon1 [ceph-mon1][DEBUG ] detect platform information from remote host [ceph-mon1][DEBUG ] detect machine type [ceph-mon1][DEBUG ] find the location of an executable [ceph_deploy.mon][INFO ] distro info: Ubuntu 18.04 bionic [ceph-mon1][DEBUG ] determining if provided host has same hostname in remote [ceph-mon1][DEBUG ] get remote short hostname [ceph-mon1][DEBUG ] deploying mon to ceph-mon1 [ceph-mon1][DEBUG ] get remote short hostname [ceph-mon1][DEBUG ] remote hostname: ceph-mon1 [ceph-mon1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf [ceph-mon1][DEBUG ] create the mon path if it does not exist [ceph-mon1][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-ceph-mon1/done [ceph-mon1][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-ceph-mon1/done [ceph-mon1][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-ceph-mon1.mon.keyring [ceph-mon1][DEBUG ] create the monitor keyring file [ceph-mon1][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i ceph-mon1 --keyring /var/lib/ceph/tmp/ceph-ceph-mon1.mon.keyring --setuser 64045 --setgroup 64045 [ceph-mon1][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-ceph-mon1.mon.keyring [ceph-mon1][DEBUG ] create a done file to avoid re-doing the mon deployment [ceph-mon1][DEBUG ] create the init path if it does not exist [ceph-mon1][INFO ] Running command: sudo systemctl enable ceph.target [ceph-mon1][INFO ] Running command: sudo systemctl enable ceph-mon@ceph-mon1 [ceph-mon1][WARNIN] Created symlink /etc/systemd/system/ceph-mon.target.wants/ceph-mon@ceph-mon1.service → /lib/systemd/system/ceph-mon@.service. [ceph-mon1][INFO ] Running command: sudo systemctl start ceph-mon@ceph-mon1 [ceph-mon1][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon1.asok mon_status [ceph-mon1][DEBUG ] ******************************************************************************** [ceph-mon1][DEBUG ] status for monitor: mon.ceph-mon1 [ceph-mon1][DEBUG ] { [ceph-mon1][DEBUG ] "election_epoch": 3, [ceph-mon1][DEBUG ] "extra_probe_peers": [], [ceph-mon1][DEBUG ] "feature_map": { [ceph-mon1][DEBUG ] "mon": [ [ceph-mon1][DEBUG ] { [ceph-mon1][DEBUG ] "features": "0x3f01cfb9fffdffff", [ceph-mon1][DEBUG ] "num": 1, [ceph-mon1][DEBUG ] "release": "luminous" [ceph-mon1][DEBUG ] } [ceph-mon1][DEBUG ] ] [ceph-mon1][DEBUG ] }, [ceph-mon1][DEBUG ] "features": { [ceph-mon1][DEBUG ] "quorum_con": "4540138297136906239", [ceph-mon1][DEBUG ] "quorum_mon": [ [ceph-mon1][DEBUG ] "kraken", [ceph-mon1][DEBUG ] "luminous", [ceph-mon1][DEBUG ] "mimic", [ceph-mon1][DEBUG ] "osdmap-prune", [ceph-mon1][DEBUG ] "nautilus", [ceph-mon1][DEBUG ] "octopus", [ceph-mon1][DEBUG ] "pacific", [ceph-mon1][DEBUG ] "elector-pinging" [ceph-mon1][DEBUG ] ], [ceph-mon1][DEBUG ] "required_con": "2449958747317026820", [ceph-mon1][DEBUG ] "required_mon": [ [ceph-mon1][DEBUG ] "kraken", [ceph-mon1][DEBUG ] "luminous", [ceph-mon1][DEBUG ] "mimic", [ceph-mon1][DEBUG ] "osdmap-prune", [ceph-mon1][DEBUG ] "nautilus", [ceph-mon1][DEBUG ] "octopus", [ceph-mon1][DEBUG ] "pacific", [ceph-mon1][DEBUG ] "elector-pinging" [ceph-mon1][DEBUG ] ] [ceph-mon1][DEBUG ] }, [ceph-mon1][DEBUG ] "monmap": { [ceph-mon1][DEBUG ] "created": "2022-03-30T10:43:39.124553Z", [ceph-mon1][DEBUG ] "disallowed_leaders: ": "", [ceph-mon1][DEBUG ] "election_strategy": 1, [ceph-mon1][DEBUG ] "epoch": 1, [ceph-mon1][DEBUG ] "features": { [ceph-mon1][DEBUG ] "optional": [], [ceph-mon1][DEBUG ] "persistent": [ [ceph-mon1][DEBUG ] "kraken", [ceph-mon1][DEBUG ] "luminous", [ceph-mon1][DEBUG ] "mimic", [ceph-mon1][DEBUG ] "osdmap-prune", [ceph-mon1][DEBUG ] "nautilus", [ceph-mon1][DEBUG ] "octopus", [ceph-mon1][DEBUG ] "pacific", [ceph-mon1][DEBUG ] "elector-pinging" [ceph-mon1][DEBUG ] ] [ceph-mon1][DEBUG ] }, [ceph-mon1][DEBUG ] "fsid": "164ae280-496b-4272-8d79-04f94b9a3b5a", [ceph-mon1][DEBUG ] "min_mon_release": 16, [ceph-mon1][DEBUG ] "min_mon_release_name": "pacific", [ceph-mon1][DEBUG ] "modified": "2022-03-30T10:43:39.124553Z", [ceph-mon1][DEBUG ] "mons": [ [ceph-mon1][DEBUG ] { [ceph-mon1][DEBUG ] "addr": "192.168.1.203:6789/0", [ceph-mon1][DEBUG ] "crush_location": "{}", [ceph-mon1][DEBUG ] "name": "ceph-mon1", [ceph-mon1][DEBUG ] "priority": 0, [ceph-mon1][DEBUG ] "public_addr": "192.168.1.203:6789/0", [ceph-mon1][DEBUG ] "public_addrs": { [ceph-mon1][DEBUG ] "addrvec": [ [ceph-mon1][DEBUG ] { [ceph-mon1][DEBUG ] "addr": "192.168.1.203:3300", [ceph-mon1][DEBUG ] "nonce": 0, [ceph-mon1][DEBUG ] "type": "v2" [ceph-mon1][DEBUG ] }, [ceph-mon1][DEBUG ] { [ceph-mon1][DEBUG ] "addr": "192.168.1.203:6789", [ceph-mon1][DEBUG ] "nonce": 0, [ceph-mon1][DEBUG ] "type": "v1" [ceph-mon1][DEBUG ] } [ceph-mon1][DEBUG ] ] [ceph-mon1][DEBUG ] }, [ceph-mon1][DEBUG ] "rank": 0, [ceph-mon1][DEBUG ] "weight": 0 [ceph-mon1][DEBUG ] } [ceph-mon1][DEBUG ] ], [ceph-mon1][DEBUG ] "stretch_mode": false, [ceph-mon1][DEBUG ] "tiebreaker_mon": "" [ceph-mon1][DEBUG ] }, [ceph-mon1][DEBUG ] "name": "ceph-mon1", [ceph-mon1][DEBUG ] "outside_quorum": [], [ceph-mon1][DEBUG ] "quorum": [ [ceph-mon1][DEBUG ] 0 [ceph-mon1][DEBUG ] ], [ceph-mon1][DEBUG ] "quorum_age": 1, [ceph-mon1][DEBUG ] "rank": 0, [ceph-mon1][DEBUG ] "state": "leader", [ceph-mon1][DEBUG ] "stretch_mode": false, [ceph-mon1][DEBUG ] "sync_provider": [] [ceph-mon1][DEBUG ] } [ceph-mon1][DEBUG ] ******************************************************************************** [ceph-mon1][INFO ] monitor: mon.ceph-mon1 is running [ceph-mon1][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon1.asok mon_status [ceph_deploy.mon][INFO ] processing monitor mon.ceph-mon1 [ceph-mon1][DEBUG ] connection detected need for sudo [ceph-mon1][DEBUG ] connected to host: ceph-mon1 [ceph-mon1][DEBUG ] detect platform information from remote host [ceph-mon1][DEBUG ] detect machine type [ceph-mon1][DEBUG ] find the location of an executable [ceph-mon1][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon1.asok mon_status [ceph_deploy.mon][INFO ] mon.ceph-mon1 monitor has reached quorum! [ceph_deploy.mon][INFO ] all initial monitors are running and have formed quorum [ceph_deploy.mon][INFO ] Running gatherkeys... [ceph_deploy.gatherkeys][INFO ] Storing keys in temp directory /tmp/tmpBrv4wZ [ceph-mon1][DEBUG ] connection detected need for sudo [ceph-mon1][DEBUG ] connected to host: ceph-mon1 [ceph-mon1][DEBUG ] detect platform information from remote host [ceph-mon1][DEBUG ] detect machine type [ceph-mon1][DEBUG ] get remote short hostname [ceph-mon1][DEBUG ] fetch remote file [ceph-mon1][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --admin-daemon=/var/run/ceph/ceph-mon.ceph-mon1.asok mon_status [ceph-mon1][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon1/keyring auth get client.admin [ceph-mon1][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon1/keyring auth get client.bootstrap-mds [ceph-mon1][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon1/keyring auth get client.bootstrap-mgr [ceph-mon1][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon1/keyring auth get client.bootstrap-osd [ceph-mon1][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon1/keyring auth get client.bootstrap-rgw [ceph_deploy.gatherkeys][INFO ] Storing ceph.client.admin.keyring [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mds.keyring [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mgr.keyring [ceph_deploy.gatherkeys][INFO ] keyring 'ceph.mon.keyring' already exists [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-osd.keyring [ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-rgw.keyring [ceph_deploy.gatherkeys][INFO ] Destroy temp directory /tmp/tmpBrv4wZ 3.8、验证mon节点
验证在mon 定节点已经自动安装并启动了ceph-mon 服务，并且后期在ceph-deploy 节点初始化目录会生成一些bootstrap ceph mds/mgr/osd/rgw 等服务的keyring 认证文件，这些初始化文件拥有对ceph 集群的最高权限，所以一定要保存好
root@ceph-mon1:~# ps aux | grep ceph-mon ceph 5095 0.1 3.9 480460 39892 ? Ssl 18:43 0:00 /usr/bin/ceph-mon -f --cluster ceph --id ceph-mon1 --setuser ceph --setgroup ceph root 5695 0.0 0.1 11340 1084 pts/0 R+ 18:44 0:00 grep --color=auto ceph-mon3.9、分发admin密钥到node节点
在ceph-deploy 节点把配置文件和admin 密钥拷贝至Ceph 集群需要执行ceph 管理命令的节点，从而不需要后期通过ceph 命令对ceph 集群进行管理配置的时候每次都需要指定ceph-mon 节点地址和ceph.client.admin.keyring 文件,另外各ceph-mon 节点也需要同步ceph 的集群配置文件与认证文件
如果在ceph-deploy 节点管理集群
root@ceph-deploy:~# apt install ceph-common -y #先安装ceph 的公共组件 root@ceph-node1:~# apt install ceph-common -y root@ceph-node2:~# apt install ceph-common -y root@ceph-node3:~# apt install ceph-common -y root@ceph-node4:~# apt install ceph-common -y拷贝密钥到ceph-node1 ceph-node2 ceph-node3 ceph-node4
#在ceph-deploy上操作
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-node1 ceph-node2 ceph-node3 ceph-node4 [ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy admin ceph-node1 ceph-node2 ceph-node3 ceph-node4 [ceph_deploy.cli][INFO ] ceph-deploy options: [ceph_deploy.cli][INFO ] username : None [ceph_deploy.cli][INFO ] verbose : False [ceph_deploy.cli][INFO ] overwrite_conf : False [ceph_deploy.cli][INFO ] quiet : False [ceph_deploy.cli][INFO ] cd_conf :[ceph_deploy.cli][INFO ] cluster : ceph [ceph_deploy.cli][INFO ] client : ['ceph-node1', 'ceph-node2', 'ceph-node3', 'ceph-node4'] [ceph_deploy.cli][INFO ] func : [ceph_deploy.cli][INFO ] ceph_conf : None [ceph_deploy.cli][INFO ] default_release : False [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node1 [ceph-node1][DEBUG ] connection detected need for sudo [ceph-node1][DEBUG ] connected to host: ceph-node1 [ceph-node1][DEBUG ] detect platform information from remote host [ceph-node1][DEBUG ] detect machine type [ceph-node1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node2 [ceph-node2][DEBUG ] connection detected need for sudo [ceph-node2][DEBUG ] connected to host: ceph-node2 [ceph-node2][DEBUG ] detect platform information from remote host [ceph-node2][DEBUG ] detect machine type [ceph-node2][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node3 [ceph-node3][DEBUG ] connection detected need for sudo [ceph-node3][DEBUG ] connected to host: ceph-node3 [ceph-node3][DEBUG ] detect platform information from remote host [ceph-node3][DEBUG ] detect machine type [ceph-node3][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node4 [ceph-node4][DEBUG ] connection detected need for sudo [ceph-node4][DEBUG ] connected to host: ceph-node4 [ceph-node4][DEBUG ] detect platform information from remote host [ceph-node4][DEBUG ] detect machine type [ceph-node4][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf 3.10、验证密钥
root@ceph-node1:~# ll /etc/ceph/ total 20 drwxr-xr-x 2 root root 4096 Mar 30 18:49 ./ drwxr-xr-x 93 root root 4096 Mar 30 18:35 ../ -rw------- 1 root root 151 Mar 30 18:49 ceph.client.admin.keyring -rw-r--r-- 1 root root 266 Mar 30 18:49 ceph.conf -rw-r--r-- 1 root root 92 Dec 8 00:26 rbdmap -rw------- 1 root root 0 Mar 30 18:49 tmpUXXQIt root@ceph-node2:~# ll /etc/ceph/ total 20 drwxr-xr-x 2 root root 4096 Mar 30 18:49 ./ drwxr-xr-x 93 root root 4096 Mar 30 18:36 ../ -rw------- 1 root root 151 Mar 30 18:49 ceph.client.admin.keyring -rw-r--r-- 1 root root 266 Mar 30 18:49 ceph.conf -rw-r--r-- 1 root root 92 Dec 8 00:26 rbdmap -rw------- 1 root root 0 Mar 30 18:49 tmp3OQ18C root@ceph-node3:~# ll /etc/ceph/ total 20 drwxr-xr-x 2 root root 4096 Mar 30 18:49 ./ drwxr-xr-x 93 root root 4096 Mar 30 18:37 ../ -rw------- 1 root root 151 Mar 30 18:49 ceph.client.admin.keyring -rw-r--r-- 1 root root 266 Mar 30 18:49 ceph.conf -rw-r--r-- 1 root root 92 Dec 8 00:26 rbdmap -rw------- 1 root root 0 Mar 30 18:49 tmpZmT34F root@ceph-node4:~# ll /etc/ceph/ total 20 drwxr-xr-x 2 root root 4096 Mar 30 18:49 ./ drwxr-xr-x 93 root root 4096 Mar 30 18:39 ../ -rw------- 1 root root 151 Mar 30 18:49 ceph.client.admin.keyring -rw-r--r-- 1 root root 266 Mar 30 18:49 ceph.conf -rw-r--r-- 1 root root 92 Dec 8 00:26 rbdmap -rw------- 1 root root 0 Mar 30 18:49 tmpiR4BpD3.11、在ceph-node节点对密钥授权
认证文件的属主和属组为了安全考虑，默认设置为了root 用户和root 组，如果需要ceph用户也能执行ceph 命令，那么就需要对ceph 用户进行授权。
root@ceph-node1:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring root@ceph-node2:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring root@ceph-node3:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring root@ceph-node4:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring3.12、部署ceph-mgr节点
(1)在ceph-mgr1和ceph-mgr2上安装ceph-mgr包
root@ceph-mgr1:~# apt install -y ceph-mgr root@ceph-mgr2:~# apt install -y ceph-mgr(2)在ceph-deploy上初始化ceph-mgr节点
#只初始化ceph-mgr01节点
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mgr create ceph-mgr1 [ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mgr create ceph-mgr1 [ceph_deploy.cli][INFO ] ceph-deploy options: [ceph_deploy.cli][INFO ] username : None [ceph_deploy.cli][INFO ] verbose : False [ceph_deploy.cli][INFO ] mgr : [('ceph-mgr1', 'ceph-mgr1')] [ceph_deploy.cli][INFO ] overwrite_conf : False [ceph_deploy.cli][INFO ] subcommand : create [ceph_deploy.cli][INFO ] quiet : False [ceph_deploy.cli][INFO ] cd_conf :[ceph_deploy.cli][INFO ] cluster : ceph [ceph_deploy.cli][INFO ] func : [ceph_deploy.cli][INFO ] ceph_conf : None [ceph_deploy.cli][INFO ] default_release : False [ceph_deploy.mgr][DEBUG ] Deploying mgr, cluster ceph hosts ceph-mgr1:ceph-mgr1 The authenticity of host 'ceph-mgr1 (192.168.1.201)' can't be established. ECDSA key fingerprint is SHA256:8pWk1yGdewC9FEqZOzCC10VMmCDSqDVKS7l6W3iiK6U. Are you sure you want to continue connecting (yes/no)? yes Warning: Permanently added 'ceph-mgr1' (ECDSA) to the list of known hosts. [ceph-mgr1][DEBUG ] connection detected need for sudo [ceph-mgr1][DEBUG ] connected to host: ceph-mgr1 [ceph-mgr1][DEBUG ] detect platform information from remote host [ceph-mgr1][DEBUG ] detect machine type [ceph_deploy.mgr][INFO ] Distro info: Ubuntu 18.04 bionic [ceph_deploy.mgr][DEBUG ] remote host will use systemd [ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to ceph-mgr1 [ceph-mgr1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf [ceph-mgr1][WARNIN] mgr keyring does not exist yet, creating one [ceph-mgr1][DEBUG ] create a keyring file [ceph-mgr1][DEBUG ] create path recursively if it doesn't exist [ceph-mgr1][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.ceph-mgr1 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-ceph-mgr1/keyring [ceph-mgr1][INFO ] Running command: sudo systemctl enable ceph-mgr@ceph-mgr1 [ceph-mgr1][WARNIN] Created symlink /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@ceph-mgr1.service → /lib/systemd/system/ceph-mgr@.service. [ceph-mgr1][INFO ] Running command: sudo systemctl start ceph-mgr@ceph-mgr1 [ceph-mgr1][INFO ] Running command: sudo systemctl enable ceph.target 3.13、验证ceph-mgr节点
在ceph-mgr1上
root@ceph-mgr1:~# ps aux | grep ceph-mgr ceph 8033 8.8 31.6 1228332 319424 ? Ssl 18:56 0:03 /usr/bin/ceph-mgr -f --cluster ceph --id ceph-mgr1 --setuser ceph --setgroup ceph root 8228 0.0 0.1 14436 1064 pts/0 S+ 18:57 0:00 grep --color=auto ceph-mgr3.14、配置ceph-deploy管理ceph集群
#ceph-deploy管理ceph集群环境设置
cephadmin@ceph-deploy:~/ceph-cluster$ sudo apt install -y ceph-common cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-deploy cephadmin@ceph-deploy:~/ceph-cluster$ sudo setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring#ceph-deploy管理ceph集群信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s cluster: id: 164ae280-496b-4272-8d79-04f94b9a3b5a health: HEALTH_WARN mon is allowing insecure global_id reclaim OSD count 0 < osd_pool_default_size 3 services: mon: 1 daemons, quorum ceph-mon1 (age 14m) mgr: ceph-mgr1(active, since 2m) osd: 0 osds: 0 up, 0 in data: pools: 0 pools, 0 pgs objects: 0 objects, 0 B usage: 0 B used, 0 B / 0 B avail pgs:禁用非安全模式通信
cephadmin@ceph-deploy:~/ceph-cluster$ ceph config set mon auth_allow_insecure_global_id_reclaim false cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s cluster: id: 164ae280-496b-4272-8d79-04f94b9a3b5a health: HEALTH_WARN OSD count 0 < osd_pool_default_size 3 services: mon: 1 daemons, quorum ceph-mon1 (age 16m) mgr: ceph-mgr1(active, since 3m) osd: 0 osds: 0 up, 0 in data: pools: 0 pools, 0 pgs objects: 0 objects, 0 B usage: 0 B used, 0 B / 0 B avail pgs:ceph集群个组件版本
cephadmin@ceph-deploy:~/ceph-cluster$ ceph version ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable) cephadmin@ceph-deploy:~/ceph-cluster$ ceph versions { "mon": { "ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)": 1 }, "mgr": { "ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)": 1 }, "osd": {}, "mds": {}, "overall": { "ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)": 2 } }3.15、准备osd节点
#前面执行了node节点的初始化，如下步骤，后面的操作可以不执行
#擦除磁盘之前通过deploy 节点对node 节点执行安装ceph 基本运行环境。
#在ceph-deploy上操作#cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node1 #cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node2 #cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node3 #cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node43.16、列出ceph node 节点磁盘
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy disk list ceph-node1 [ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf [ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk list ceph-node1 [ceph_deploy.cli][INFO ] ceph-deploy options: [ceph_deploy.cli][INFO ] username : None [ceph_deploy.cli][INFO ] verbose : False [ceph_deploy.cli][INFO ] debug : False [ceph_deploy.cli][INFO ] overwrite_conf : False [ceph_deploy.cli][INFO ] subcommand : list [ceph_deploy.cli][INFO ] quiet : False [ceph_deploy.cli][INFO ] cd_conf :[ceph_deploy.cli][INFO ] cluster : ceph [ceph_deploy.cli][INFO ] host : ['ceph-node1'] [ceph_deploy.cli][INFO ] func : [ceph_deploy.cli][INFO ] ceph_conf : None [ceph_deploy.cli][INFO ] default_release : False [ceph-node1][DEBUG ] connection detected need for sudo [ceph-node1][DEBUG ] connected to host: ceph-node1 [ceph-node1][DEBUG ] detect platform information from remote host [ceph-node1][DEBUG ] detect machine type [ceph-node1][DEBUG ] find the location of an executable [ceph-node1][INFO ] Running command: sudo fdisk -l [ceph-node1][INFO ] Disk /dev/sda: 20 GiB, 21474836480 bytes, 41943040 sectors [ceph-node1][INFO ] Disk /dev/sdb: 10 GiB, 10737418240 bytes, 20971520 sectors [ceph-node1][INFO ] Disk /dev/sdc: 10 GiB, 10737418240 bytes, 20971520 sectors [ceph-node1][INFO ] Disk /dev/sdd: 10 GiB, 10737418240 bytes, 20971520 sectors 3.17、使用ceph-deploy disk zap擦除各ceph node的ceph数据磁盘
#在ceph-deploy上操作 ceph-deploy disk zap ceph-node1 /dev/sdb ceph-deploy disk zap ceph-node1 /dev/sdc ceph-deploy disk zap ceph-node1 /dev/sdd ceph-deploy disk zap ceph-node2 /dev/sdb ceph-deploy disk zap ceph-node2 /dev/sdc ceph-deploy disk zap ceph-node2 /dev/sdd ceph-deploy disk zap ceph-node3 /dev/sdb ceph-deploy disk zap ceph-node3 /dev/sdc ceph-deploy disk zap ceph-node3 /dev/sdd ceph-deploy disk zap ceph-node4 /dev/sdb ceph-deploy disk zap ceph-node4 /dev/sdc ceph-deploy disk zap ceph-node4 /dev/sdd3.18、添加osd
#在ceph-deploy上操作 ceph-deploy osd create ceph-node1 --data /dev/sdb ceph-deploy osd create ceph-node1 --data /dev/sdc ceph-deploy osd create ceph-node1 --data /dev/sdd ceph-deploy osd create ceph-node2 --data /dev/sdb ceph-deploy osd create ceph-node2 --data /dev/sdc ceph-deploy osd create ceph-node2 --data /dev/sdd ceph-deploy osd create ceph-node3 --data /dev/sdb ceph-deploy osd create ceph-node3 --data /dev/sdc ceph-deploy osd create ceph-node3 --data /dev/sdd ceph-deploy osd create ceph-node4 --data /dev/sdb ceph-deploy osd create ceph-node4 --data /dev/sdc ceph-deploy osd create ceph-node4 --data /dev/sdd3.19、验证osd
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s cluster: id: 164ae280-496b-4272-8d79-04f94b9a3b5a health: HEALTH_OK services: mon: 1 daemons, quorum ceph-mon1 (age 28m) mgr: ceph-mgr1(active, since 15m) osd: 12 osds: 12 up (since 12s), 12 in (since 21s) data: pools: 1 pools, 1 pgs objects: 0 objects, 0 B usage: 74 MiB used, 120 GiB / 120 GiB avail pgs: 1 active+clean3.20、设置osd服务自启动
默认就已经为自启动, node 节点添加完成后,开源测试node 服务器重启后，OSD 是否会自动启动
# 验证 root@ceph-node1:~# ps -ef|grep osd ceph 7045 1 0 19:08 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 0 --setuser ceph --setgroup ceph ceph 8720 1 0 19:09 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 1 --setuser ceph --setgroup ceph ceph 10386 1 0 19:09 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 2 --setuser ceph --setgroup ceph root 10885 917 0 19:13 pts/0 00:00:00 grep --color=auto osd root@ceph-node2:~# ps -ef|grep osd ceph 6883 1 0 19:09 ? 00:00:02 /usr/bin/ceph-osd -f --cluster ceph --id 3 --setuser ceph --setgroup ceph ceph 8561 1 0 19:09 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 4 --setuser ceph --setgroup ceph ceph 10227 1 0 19:09 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 5 --setuser ceph --setgroup ceph root 10726 905 0 19:13 pts/0 00:00:00 grep --color=auto osd root@ceph-node3:~# ps -ef|grep osd ceph 7096 1 0 19:09 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 6 --setuser ceph --setgroup ceph ceph 8775 1 0 19:10 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 7 --setuser ceph --setgroup ceph ceph 10436 1 0 19:10 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 8 --setuser ceph --setgroup ceph root 10935 998 0 19:13 pts/0 00:00:00 grep --color=auto osd root@ceph-node4:~# ps -ef|grep osd ceph 6836 1 1 19:10 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 9 --setuser ceph --setgroup ceph ceph 8508 1 0 19:10 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 10 --setuser ceph --setgroup ceph ceph 10371 1 0 19:12 ? 00:00:00 /usr/bin/ceph-osd -f --cluster ceph --id 11 --setuser ceph --setgroup ceph root 10872 904 0 19:13 pts/0 00:00:00 grep --color=auto osd # 设置开机自启动 root@ceph-node1:~# systemctl enable ceph-osd@0 ceph-osd@1 ceph-osd@2 Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@0.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@1.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@2.service → /lib/systemd/system/ceph-osd@.service. root@ceph-node2:~# systemctl enable ceph-osd@3 ceph-osd@4 ceph-osd@5 Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@3.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@4.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@5.service → /lib/systemd/system/ceph-osd@.service. root@ceph-node3:~# systemctl enable ceph-osd@6 ceph-osd@7 ceph-osd@8 Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@6.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@7.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@8.service → /lib/systemd/system/ceph-osd@.service. root@ceph-node4:~# systemctl enable ceph-osd@9 ceph-osd@10 ceph-osd@11 Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@9.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@10.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@11.service → /lib/systemd/system/ceph-osd@.service.3.21、ceph-deploy的命令
$ ceph-deploy --help new：开始部署一个新的ceph 存储集群，并生成CLUSTER.conf 集群配置文件和keyring 认证文件。 install: 在远程主机上安装ceph 相关的软件包, 可以通过--release 指定安装的版本。 rgw：管理RGW 守护程序(RADOSGW,对象存储网关)。 mgr：管理MGR 守护程序(ceph-mgr,Ceph Manager DaemonCeph 管理器守护程序)。 mds：管理MDS 守护程序(Ceph Metadata Server，ceph 源数据服务器)。 mon：管理MON 守护程序(ceph-mon,ceph 监视器)。 gatherkeys：从指定获取提供新节点的验证keys，这些keys 会在添加新的MON/OSD/MD加入的时候使用。 disk：管理远程主机磁盘。 osd：在远程主机准备数据磁盘，即将指定远程主机的指定磁盘添加到ceph 集群作为osd 使用。 repo： 远程主机仓库管理。 admin：推送ceph 集群配置文件和client.admin 认证文件到远程主机。 config：将ceph.conf 配置文件推送到远程主机或从远程主机拷贝。 uninstall：从远端主机删除安装包。 purgedata：从/var/lib/ceph 删除ceph 数据,会删除/etc/ceph 下的内容。 purge: 删除远端主机的安装包和所有数据。 forgetkeys：从本地主机删除所有的验证keyring, 包括client.admin, monitor, bootstrap 等 认证文件。 pkg： 管理远端主机的安装包。 calamari：安装并配置一个calamari web 节点，calamari 是一个web 监控平台。3.22、测试上传与下载数据
存取数据时，客户端必须首先连接至 RADOS 集群上某存储池，然后根据对象名称由相关的CRUSH 规则完成数据对象寻址。于是，为了测试集群的数据存取功能，这里首先创建一个用于测试的存储池 mypool，并设定其 PG 数量为 32 个
# 创建存储池 cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create mypool 32 32 pool 'mypool' created cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool ls device_health_metrics mypool cephadmin@ceph-deploy:~/ceph-cluster$ ceph pg ls-by-pool mypool | awk '{print $1,$2,$15}' PG OBJECTS ACTING 3.0 0 [8,4,11]p8 3.1 0 [0,4,10]p0 3.2 0 [10,6,5]p10 3.3 0 [10,4,0]p10 3.4 0 [1,7,11]p1 3.5 0 [5,2,6]p5 3.6 0 [0,11,4]p0 3.7 0 [2,6,9]p2 3.8 0 [11,1,7]p11 3.9 0 [8,11,5]p8 3.a 0 [3,9,8]p3 3.b 0 [1,10,8]p1 3.c 0 [5,0,7]p5 3.d 0 [3,7,2]p3 3.e 0 [7,11,4]p7 3.f 0 [5,8,11]p5 3.10 0 [10,7,5]p10 3.11 0 [7,4,10]p7 3.12 0 [9,3,0]p9 3.13 0 [3,9,8]p3 3.14 0 [3,10,2]p3 3.15 0 [9,6,3]p9 3.16 0 [5,1,9]p5 3.17 0 [10,5,8]p10 3.18 0 [10,6,1]p10 3.19 0 [1,4,10]p1 3.1a 0 [8,1,9]p8 3.1b 0 [6,4,10]p6 3.1c 0 [5,1,9]p5 3.1d 0 [9,4,7]p9 3.1e 0 [2,6,11]p2 3.1f 0 [11,0,5]p11 * NOTE: afterwards # 上传文件 cephadmin@ceph-deploy:~/ceph-cluster$ sudo rados put msg1 /var/log/syslog --pool=mypool # 把syslog文件上传到mypool并指定对象id为msg1 # 列出文件 cephadmin@ceph-deploy:~/ceph-cluster$ rados ls --pool=mypool msg1 # 文件信息 cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd map mypool msg1 osdmap e96 pool 'mypool' (3) object 'msg1' -> pg 3.c833d430 (3.10) -> up ([10,7,5], p10) acting ([10,7,5], p10) 表示文件放在了存储池 id为2的c833d430的PG上,10为当前PG的id, 2.10表示数据是在id为2的存储池当中id为10的PG 中存储，在线的OSD编号15,13,10，主OSD为5，活动的OSD 15,13,10，三个OSD表示数据放一共3个副本，PG中的OSD是ceph的 crush算法计算出三份数据保存在哪些 OSD # 下载文件 cephadmin@ceph-deploy:~/ceph-cluster$ sudo rados get msg1 --pool=mypool /opt/my.txt cephadmin@ceph-deploy:~/ceph-cluster$ ls /opt/ my.txt # 修改文件 cephadmin@ceph-deploy:~/ceph-cluster$ sudo rados put msg1 /etc/passwd --pool=mypool cephadmin@ceph-deploy:~/ceph-cluster$ sudo rados get msg1 --pool=mypool /opt/2.txt cephadmin@ceph-deploy:~/ceph-cluster$ tail /opt/2.txt lxd:x:105:65534::/var/lib/lxd/:/bin/false uuidd:x:106:110::/run/uuidd:/usr/sbin/nologin dnsmasq:x:107:65534:dnsmasq,,,:/var/lib/misc:/usr/sbin/nologin landscape:x:108:112::/var/lib/landscape:/usr/sbin/nologin sshd:x:109:65534::/run/sshd:/usr/sbin/nologin pollinate:x:110:1::/var/cache/pollinate:/bin/false jack:x:1000:1000:jack,,,:/home/jack:/bin/bash statd:x:111:65534::/var/lib/nfs:/usr/sbin/nologin cephadmin:x:2022:2022::/home/cephadmin:/bin/bash ceph:x:64045:64045:Ceph storage service:/var/lib/ceph:/usr/sbin/nologin # 删除文件 cephadmin@ceph-deploy:~/ceph-cluster$ sudo rados rm msg1 --pool=mypool cephadmin@ceph-deploy:~/ceph-cluster$ rados ls --pool=mypool cephadmin@ceph-deploy:~/ceph-cluster$4、扩展Ceph集群高可用
4.1、扩展ceph-mon节点
Ceph-mon 是原生具备自选举以实现高可用机制的ceph 服务，节点数量通常是奇数。
#在ceph-mon节点安装ceph-mon root@ceph-mon2:~# apt install ceph-mon root@ceph-mon3:~# apt install ceph-mon #在ceph-deploy节点添加mon节点 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon add ceph-mon2 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon add ceph-mon3ceph-mon结果验证
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s cluster: id: 164ae280-496b-4272-8d79-04f94b9a3b5a health: HEALTH_OK services: mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 7s) mgr: ceph-mgr1(active, since 26m) osd: 12 osds: 12 up (since 10m), 12 in (since 10m) data: pools: 1 pools, 1 pgs objects: 0 objects, 0 B usage: 74 MiB used, 120 GiB / 120 GiB avail pgs: 1 active+clean #ceph-mon的状态 cephadmin@ceph-deploy:~/ceph-cluster$ ceph quorum_status --format json-pretty { "election_epoch": 12, "quorum": [ 0, 1, 2 ], "quorum_names": [ "ceph-mon1", "ceph-mon2", "ceph-mon3" ], "quorum_leader_name": "ceph-mon1", "quorum_age": 38, "features": { "quorum_con": "4540138297136906239", "quorum_mon": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ] }, "monmap": { "epoch": 3, "fsid": "164ae280-496b-4272-8d79-04f94b9a3b5a", "modified": "2022-03-30T11:22:33.628858Z", "created": "2022-03-30T10:43:39.124553Z", "min_mon_release": 16, "min_mon_release_name": "pacific", "election_strategy": 1, "disallowed_leaders: ": "", "stretch_mode": false, "tiebreaker_mon": "", "features": { "persistent": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ], "optional": [] }, "mons": [ { "rank": 0, "name": "ceph-mon1", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "192.168.1.203:3300", "nonce": 0 }, { "type": "v1", "addr": "192.168.1.203:6789", "nonce": 0 } ] }, "addr": "192.168.1.203:6789/0", "public_addr": "192.168.1.203:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 1, "name": "ceph-mon2", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "192.168.1.204:3300", "nonce": 0 }, { "type": "v1", "addr": "192.168.1.204:6789", "nonce": 0 } ] }, "addr": "192.168.1.204:6789/0", "public_addr": "192.168.1.204:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 2, "name": "ceph-mon3", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "192.168.1.205:3300", "nonce": 0 }, { "type": "v1", "addr": "192.168.1.205:6789", "nonce": 0 } ] }, "addr": "192.168.1.205:6789/0", "public_addr": "192.168.1.205:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" } ] } }4.2、扩展ceph-mgr节点
#在ceph-mgr2节点上安装ceph-mgr root@ceph-mgr2:~#apt install -y ceph-mgr #在ceph-deploy上添加ceph-mgr2 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mgr create ceph-mgr2 #在ceph-deploy上同步配置文件到ceph-mgr2 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-mgr2ceph-mgr验证
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s cluster: id: 164ae280-496b-4272-8d79-04f94b9a3b5a health: HEALTH_OK services: mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 2m) mgr: ceph-mgr1(active, since 71m), standbys: ceph-mgr2 osd: 12 osds: 12 up (since 56m), 12 in (since 56m) data: pools: 1 pools, 1 pgs objects: 0 objects, 0 B usage: 74 MiB used, 120 GiB / 120 GiB avail pgs: 1 active+clean5、部署cephfs
5.1、安装ceph-mds软件包
如果要使用cephFS，需要部署cephfs 服务。MDS服务与mon服务部署在一起（最好分开部署）
在所有ceph-mon上安装ceph-mds
root@ceph-mon1:~# apt install ceph-mds root@ceph-mon2:~# apt install ceph-mds root@ceph-mon3:~# apt install ceph-mds5.2、在ceph-deploy上把mds服务添加到ceph集群
在ceph-deploy上操作
ceph-deploy mds create ceph-mds1 ceph-deploy mds create ceph-mds2 ceph-deploy mds create ceph-mds35.3、创建CephFS metadata和data存储池
# 创建cephfs-metadata cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-metadata 32 32 pool 'cephfs-metadata' created # 创建cephfs-data cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-data 64 64 pool 'cephfs-data' created # 创建一个叫mycephfs的cephFS cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs new mycephfs cephfs-metadata cephfs-data new fs with metadata pool 4 and data pool 55.4、验证CephFS
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs ls name: mycephfs, metadata pool: cephfs-metadata, data pools: [cephfs-data ] cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs mycephfs - 0 clients ======== RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active ceph-mds1 Reqs: 0 /s 10 13 12 0 POOL TYPE USED AVAIL cephfs-metadata metadata 96.0k 37.9G cephfs-data data 0 37.9G STANDBY MDS ceph-mds2 ceph-mds3 MDS version: ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable) # 验证cephfs服务的状态 cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat mycephfs:1 {0=ceph-mds1=up:active} 2 up:standby5.5、MDS高可用
为了测试，我们把ceph-mgr2复用为mds
#添加第4台mds ceph-mgr2 #在ceph-mgr2上安装ceph-mds root@ceph-mgr2:~# apt install ceph-mds #在ceph-deploy上把mgr2添加进mds cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mds create ceph-mgr2 #现在mds的状态为1主3备 cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs - 0 clients ======== RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active ceph-mds1 Reqs: 0 /s 10 13 12 0 POOL TYPE USED AVAIL cephfs-metadata metadata 96.0k 37.9G cephfs-data data 0 37.9G STANDBY MDS ceph-mds2 ceph-mds3 ceph-mgr2 MDS version: ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)设置每个Rank 的备份MDS，也就是如果此Rank 当前的MDS 出现问题马上切换到另个MDS。设置备份的方法有很多，常用选项如下
mds_standby_replay：值为true 或false，true 表示开启replay 模式，这种模式下主MDS 内的数量将实时与从MDS 同步，如果主宕机，从可以快速的切换。如果为false 只有宕机的时候才去同步数据，这样会有一段时间的中断。 mds_standby_for_name：设置当前MDS 进程只用于备份于指定名称的MDS。 mds_standby_for_rank：设置当前MDS 进程只用于备份于哪个Rank，通常为Rank 编号。另外在存在之个CephFS 文件系统中，还可以使用mds_standby_for_fscid 参数来为指定不同的文件系统。 mds_standby_for_fscid：指定CephFS 文件系统ID，需要联合mds_standby_for_rank 生效，如果设置mds_standby_for_rank，那么就是用于指定文件系统的指定Rank，如果没有设置，就是指定文件系统的所有Rank。5.5.1、当前cephfs集群状态
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs - 0 clients ======== RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active ceph-mds1 Reqs: 0 /s 10 13 12 0 POOL TYPE USED AVAIL cephfs-metadata metadata 96.0k 37.9G cephfs-data data 0 37.9G STANDBY MDS ceph-mds2 ceph-mds3 ceph-mgr2 MDS version: ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)5.5.2、当前文件系统状态
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs get mycephfs Filesystem 'mycephfs' (1) fs_name mycephfs epoch 6 flags 12 created 2022-03-31T18:50:41.454911+0800 modified 2022-03-31T18:50:42.467695+0800 tableserver 0 root 0 session_timeout 60 session_autoclose 300 max_file_size 1099511627776 required_client_features {} last_failure 0 last_failure_osd_epoch 0 compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} max_mds 1 in 0 up {0=54258} failed damaged stopped data_pools [5] metadata_pool 4 inline_data disabled balancer standby_count_wanted 1 [mds.ceph-mds1{0:54258} state up:active seq 59 addr [v2:192.168.1.203:6800/1944489540,v1:192.168.1.203:6801/1944489540] compat {c=[1],r=[1],i=[7ff]}]5.5.3、
目前有四个mds 服务器，但是有一个主三个备，可以优化一下部署架构，设置为为两主两备。
#设置同时活跃的主mds最大值为2 cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs set mycephfs max_mds 2 cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat mycephfs:2 {0=ceph-mds1=up:active,1=ceph-mgr2=up:active} 2 up:standby cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs - 0 clients ======== RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active ceph-mds1 Reqs: 0 /s 10 13 12 0 1 active ceph-mgr2 Reqs: 0 /s 10 13 11 0 POOL TYPE USED AVAIL cephfs-metadata metadata 168k 37.9G cephfs-data data 0 37.9G STANDBY MDS ceph-mds2 ceph-mds3 MDS version: ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)5.5.4、MDS高可用优化
目前的状态是ceph-mds1 和ceph-mgr2 分别是active 状态，ceph-mds2和ceph-mds3 分别处于standby 状态，现在可以将ceph-mds2设置为ceph-mds1 的standby，将ceph-mds3 设置为ceph-mgr2的standby，以实现每个主都有一个固定备份角色的结构，则修改配置文件如
[global] fsid = 164ae280-496b-4272-8d79-04f94b9a3b5a public_network = 192.168.1.0/24 cluster_network = 192.168.88.0/24 mon_initial_members = ceph-mon1 mon_host = 192.168.1.203 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx [mds.ceph-mds2] mds_standby_for_name = ceph-mds1 mds_standby_replay = true [mds.ceph-mds3] mds_standby_for_name = ceph-mgr2 mds_standby_replay = true分发配置文件并重启mds 服务:
#分发配置文件保证各mds 服务重启有效 cephadmin@ceph-deploy:/etc/ceph$ ceph-deploy --overwrite-conf config push ceph-mon3 cephadmin@ceph-deploy:/etc/ceph$ ceph-deploy --overwrite-conf config push ceph-mon2 cephadmin@ceph-deploy:/etc/ceph$ ceph-deploy --overwrite-conf config push ceph-mon1 cephadmin@ceph-deploy:/etc/ceph$ ceph-deploy --overwrite-conf config push ceph-mgr2 #重启mds节点，先重启active节点ceph-mds1和ceph-mgr2,再重启standby节点ceph-mds2和ceph-mds3 cephadmin@ceph-mon1:~$ systemctl restart ceph-mds@ceph-mds1.service cephadmin@ceph-mgr2:~$ systemctl restart ceph-mds@ceph-mgr2.service cephadmin@ceph-mon2:~$ systemctl restart ceph-mds@ceph-mds2.service cephadmin@ceph-mon3:~$ systemctl restart ceph-mds@ceph-mds3.service验证mds
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs - 0 clients ======== RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active ceph-mds2 Reqs: 0 /s 10 13 12 0 1 active ceph-mds1 Reqs: 0 /s 10 13 11 0 POOL TYPE USED AVAIL cephfs-metadata metadata 168k 37.9G cephfs-data data 0 37.9G STANDBY MDS ceph-mgr2 ceph-mds3 MDS version: ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)查看active 和standby 对应关系：
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs get mycephfs Filesystem 'mycephfs' (1) fs_name mycephfs epoch 31 flags 12 created 2022-03-31T18:50:41.454911+0800 modified 2022-03-31T19:16:16.871263+0800 tableserver 0 root 0 session_timeout 60 session_autoclose 300 max_file_size 1099511627776 required_client_features {} last_failure 0 last_failure_osd_epoch 139 compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} max_mds 2 in 0,1 up {0=54477,1=54459} failed damaged stopped data_pools [5] metadata_pool 4 inline_data disabled balancer standby_count_wanted 1 [mds.ceph-mds2{0:54477} state up:active seq 6 addr [v2:192.168.1.204:6800/1714341319,v1:192.168.1.204:6801/1714341319] compat {c=[1],r=[1],i=[7ff]}] [mds.ceph-mds1{1:54459} state up:active seq 7 addr [v2:192.168.1.203:6800/579540118,v1:192.168.1.203:6801/579540118] compat {c=[1],r=[1],i=[7ff]}]6、部署RadosGW
将ceph-mgr1、ceph-mgr2 服务器部署为高可用的radosGW 服务
6.1、在ceph-mgr1和ceph-mgr2上安装ceph-radosgw
root@ceph-mgr1:~# apt install radosgw -y root@ceph-mgr2:~# apt install radosgw -y
#注意：如果前面有做ceph.conf的配置变动，需要将ceph.conf重新拷贝到ceph-rgw1和cpeh-rgw2上 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy --overwrite-conf rgw create ceph-mgr1 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy --overwrite-conf rgw create ceph-mgr26.3、验证radosgw的状态
在mgr节点上查看radosgw节点信息
root@ceph-mgr1:~# ps aux | grep radosgw ceph 2782 0.5 2.8 6277308 57248 ? Ssl 19:22 0:00 /usr/bin/radosgw -f --cluster ceph --name client.rgw.ceph-rgw1 --setuser ceph --setgroup ceph root 3478 0.0 0.0 14436 1068 pts/0 R+ 19:25 0:00 grep --color=auto radosgw root@ceph-mgr2:~# ps aux | grep radosgw ceph 2907 0.4 2.7 6277280 56888 ? Ssl 19:22 0:00 /usr/bin/radosgw -f --cluster ceph --name client.rgw.ceph-rgw2 --setuser ceph --setgroup ceph root 3567 0.0 0.0 14436 1048 pts/0 S+ 19:26 0:00 grep --color=auto radosgw在ceph-deploy上查看集群的radosgw信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s cluster: id: 164ae280-496b-4272-8d79-04f94b9a3b5a health: HEALTH_OK services: mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 2h) mgr: ceph-mgr2(active, since 2h), standbys: ceph-mgr1 mds: 2/2 daemons up, 2 standby osd: 12 osds: 12 up (since 119m), 12 in (since 24h) rgw: 2 daemons active (2 hosts, 1 zones) data: volumes: 1/1 healthy pools: 8 pools, 233 pgs objects: 230 objects, 8.5 KiB usage: 203 MiB used, 120 GiB / 120 GiB avail pgs: 233 active+clean6.4、测试访问radosgw服务
cephadmin@ceph-deploy:~/ceph-cluster$ curl 192.168.1.201:7480 <?xml version="1.0" encoding="UTF-8"?>cephadmin@ceph-deploy:~/ceph-cluster$ cephadmin@ceph-deploy:~/ceph-cluster$ cephadmin@ceph-deploy:~/ceph-cluster$ curl 192.168.1.202:7480 <?xml version="1.0" encoding="UTF-8"?> anonymous cephadmin@ceph-deploy:~/ceph-cluster$ anonymous