- Make sure that 'ceph-Host' has enabled Nested KVM Virtualization:
[root@base ~]# virsh dumpxml ceph-host | grep "cpu mode" <cpu mode='host-passthrough'/>
- Make sure that you are using the correct snapshot:
[root@base ~]# virsh snapshot-current --name ceph-host 00_ready_to_deploy
- Check VirtualBox version:
[ab@ceph-host ~]$ VBoxManage --version 5.1.26r117224
- Install 'vagrant' using the following HowTo:
https://www.tecmint.com/how-to-install-vagrant-on-centos-7/[root@ceph-host ~]# yum -y install https://releases.hashicorp.com/vagrant/1.9.8/vagrant_1.9.8_x86_64.rpm
[root@ceph-host ~]# vagrant --version Vagrant 1.9.8
- Install 'git':
[root@ceph-host ~]# yum -y install git
[root@ceph-host ~]# git --version git version 1.8.3.1
- As 'ab' user who is part of 'vboxusers' group, clone ceph-cookbook repositories to your VirtualBox host machine:
[ab@ceph-host ~]$ hostname ceph-host.lab
[ab@ceph-host ~]$ id uid=1000(ab) gid=1000(ab) groups=1000(ab),983(vboxusers) context=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023
ab@ceph-host ~]$ cd ~
[ab@ceph-host ~]$ git clone https://github.com/ksingh7/ceph-cookbook.git Cloning into 'ceph-cookbook'... remote: Counting objects: 122, done. remote: Total 122 (delta 0), reused 0 (delta 0), pack-reused 122 Receiving objects: 100% (122/122), 752.72 KiB | 0 bytes/s, done. Resolving deltas: 100% (62/62), done.
[ab@ceph-host ~]$ pwd /home/ab [ab@ceph-host ~]$ ls -lhd ceph-cookbook/ drwxrwxr-x. 5 ab ab 4.0K Sep 2 19:49 ceph-cookbook/ [ab@ceph-host ~]$ ls -lh ceph-cookbook/ total 72K drwxrwxr-x. 2 ab ab 4.0K Sep 2 19:49 ceph-dokan -rw-rw-r--. 1 ab ab 215 Sep 2 19:49 keystonerc_admin -rw-rw-r--. 1 ab ab 180 Sep 2 19:49 keystonerc_demo -rw-rw-r--. 1 ab ab 30K Sep 2 19:49 openstack-answerfile.cfg -rw-rw-r--. 1 ab ab 896 Sep 2 19:49 post-deploy.sh -rw-rw-r--. 1 ab ab 1.7K Sep 2 19:49 rbdmap -rw-rw-r--. 1 ab ab 16K Sep 2 19:49 Vagrantfile drwxrwxr-x. 2 ab ab 4.0K Sep 2 19:49 vsm
- Change directory to the cloned git directory and lunch the three VirtualBox Nodes [[You have to execute the following command from GUI session using 'ab' username]]:
[ab@ceph-host ~]$ pwd /home/ab
[ab@ceph-host ~]$ cd ceph-cookbook/
[ab@ceph-host ceph-cookbook]$ vagrant up ceph-node1 ceph-node2 ceph-node3
Bringing machine 'ceph-node1' up with 'virtualbox' provider...
Bringing machine 'ceph-node2' up with 'virtualbox' provider...
Bringing machine 'ceph-node3' up with 'virtualbox' provider...
==> ceph-node1: Box 'centos7-standard' could not be found. Attempting to find and install...
ceph-node1: Box Provider: virtualbox
ceph-node1: Box Version: >= 0
==> ceph-node1: Box file was not detected as metadata. Adding it directly...
==> ceph-node1: Adding box 'centos7-standard' (v0) for provider: virtualbox
ceph-node1: Downloading: https://www.dropbox.com/s/hiarmp3cdzjy94o/centos7-standard.box?dl=1
==> ceph-node1: Successfully added box 'centos7-standard' (v0) for 'virtualbox'!
==> ceph-node1: Importing base box 'centos7-standard'...
==> ceph-node1: Matching MAC address for NAT networking...
==> ceph-node1: Setting the name of the VM: ceph-node1
==> ceph-node1: Clearing any previously set network interfaces...
==> ceph-node1: Preparing network interfaces based on configuration...
ceph-node1: Adapter 1: nat
ceph-node1: Adapter 2: hostonly
==> ceph-node1: Forwarding ports...
ceph-node1: 22 (guest) => 2222 (host) (adapter 1)
==> ceph-node1: Running 'pre-boot' VM customizations...
==> ceph-node1: Booting VM...
==> ceph-node1: Waiting for machine to boot. This may take a few minutes...
ceph-node1: SSH address: 127.0.0.1:2222
ceph-node1: SSH username: vagrant
ceph-node1: SSH auth method: private key
ceph-node1:
ceph-node1: Vagrant insecure key detected. Vagrant will automatically replace
ceph-node1: this with a newly generated keypair for better security.
ceph-node1:
ceph-node1: Inserting generated public key within guest...
ceph-node1: Removing insecure key from the guest if it's present...
ceph-node1: Key inserted! Disconnecting and reconnecting using new SSH key...
==> ceph-node1: Machine booted and ready!
==> ceph-node1: Checking for guest additions in VM...
ceph-node1: The guest additions on this VM do not match the installed version of
ceph-node1: VirtualBox! In most cases this is fine, but in rare cases it can
ceph-node1: prevent things such as shared folders from working properly. If you see
ceph-node1: shared folder errors, please make sure the guest additions within the
ceph-node1: virtual machine match the version of VirtualBox you have installed on
ceph-node1: your host and reload your VM.
ceph-node1:
ceph-node1: Guest Additions Version: 4.3.22
ceph-node1: VirtualBox Version: 5.1
==> ceph-node1: Setting hostname...
==> ceph-node1: Configuring and enabling network interfaces...
ceph-node1: SSH address: 127.0.0.1:2222
ceph-node1: SSH username: vagrant
ceph-node1: SSH auth method: private key
==> ceph-node1: Running provisioner: shell...
ceph-node1: Running: /tmp/vagrant-shell20170902-3782-pviums.sh
==> ceph-node2: Box 'centos7-standard' could not be found. Attempting to find and install...
ceph-node2: Box Provider: virtualbox
ceph-node2: Box Version: >= 0
==> ceph-node2: Box file was not detected as metadata. Adding it directly...
==> ceph-node2: Adding box 'centos7-standard' (v0) for provider: virtualbox
==> ceph-node2: Importing base box 'centos7-standard'...
==> ceph-node2: Matching MAC address for NAT networking...
==> ceph-node2: Setting the name of the VM: ceph-node2
==> ceph-node2: Fixed port collision for 22 => 2222. Now on port 2200.
==> ceph-node2: Clearing any previously set network interfaces...
==> ceph-node2: Preparing network interfaces based on configuration...
ceph-node2: Adapter 1: nat
ceph-node2: Adapter 2: hostonly
==> ceph-node2: Forwarding ports...
ceph-node2: 22 (guest) => 2200 (host) (adapter 1)
==> ceph-node2: Running 'pre-boot' VM customizations...
==> ceph-node2: Booting VM...
==> ceph-node2: Waiting for machine to boot. This may take a few minutes...
ceph-node2: SSH address: 127.0.0.1:2200
ceph-node2: SSH username: vagrant
ceph-node2: SSH auth method: private key
ceph-node2:
ceph-node2: Vagrant insecure key detected. Vagrant will automatically replace
ceph-node2: this with a newly generated keypair for better security.
ceph-node2:
ceph-node2: Inserting generated public key within guest...
ceph-node2: Removing insecure key from the guest if it's present...
ceph-node2: Key inserted! Disconnecting and reconnecting using new SSH key...
==> ceph-node2: Machine booted and ready!
==> ceph-node2: Checking for guest additions in VM...
ceph-node2: The guest additions on this VM do not match the installed version of
ceph-node2: VirtualBox! In most cases this is fine, but in rare cases it can
ceph-node2: prevent things such as shared folders from working properly. If you see
ceph-node2: shared folder errors, please make sure the guest additions within the
ceph-node2: virtual machine match the version of VirtualBox you have installed on
ceph-node2: your host and reload your VM.
ceph-node2:
ceph-node2: Guest Additions Version: 4.3.22
ceph-node2: VirtualBox Version: 5.1
==> ceph-node2: Setting hostname...
==> ceph-node2: Configuring and enabling network interfaces...
ceph-node2: SSH address: 127.0.0.1:2200
ceph-node2: SSH username: vagrant
ceph-node2: SSH auth method: private key
==> ceph-node2: Running provisioner: shell...
ceph-node2: Running: /tmp/vagrant-shell20170902-3782-1z300y.sh
==> ceph-node3: Box 'centos7-standard' could not be found. Attempting to find and install...
ceph-node3: Box Provider: virtualbox
ceph-node3: Box Version: >= 0
==> ceph-node3: Box file was not detected as metadata. Adding it directly...
==> ceph-node3: Adding box 'centos7-standard' (v0) for provider: virtualbox
==> ceph-node3: Importing base box 'centos7-standard'...
==> ceph-node3: Matching MAC address for NAT networking...
==> ceph-node3: Setting the name of the VM: ceph-node3
==> ceph-node3: Fixed port collision for 22 => 2222. Now on port 2201.
==> ceph-node3: Clearing any previously set network interfaces...
==> ceph-node3: Preparing network interfaces based on configuration...
ceph-node3: Adapter 1: nat
ceph-node3: Adapter 2: hostonly
==> ceph-node3: Forwarding ports...
ceph-node3: 22 (guest) => 2201 (host) (adapter 1)
==> ceph-node3: Running 'pre-boot' VM customizations...
==> ceph-node3: Booting VM...
==> ceph-node3: Waiting for machine to boot. This may take a few minutes...
ceph-node3: SSH address: 127.0.0.1:2201
ceph-node3: SSH username: vagrant
ceph-node3: SSH auth method: private key
ceph-node3:
ceph-node3: Vagrant insecure key detected. Vagrant will automatically replace
ceph-node3: this with a newly generated keypair for better security.
ceph-node3:
ceph-node3: Inserting generated public key within guest...
ceph-node3: Removing insecure key from the guest if it's present...
ceph-node3: Key inserted! Disconnecting and reconnecting using new SSH key...
==> ceph-node3: Machine booted and ready!
==> ceph-node3: Checking for guest additions in VM...
ceph-node3: The guest additions on this VM do not match the installed version of
ceph-node3: VirtualBox! In most cases this is fine, but in rare cases it can
ceph-node3: prevent things such as shared folders from working properly. If you see
ceph-node3: shared folder errors, please make sure the guest additions within the
ceph-node3: virtual machine match the version of VirtualBox you have installed on
ceph-node3: your host and reload your VM.
ceph-node3:
ceph-node3: Guest Additions Version: 4.3.22
ceph-node3: VirtualBox Version: 5.1
==> ceph-node3: Setting hostname...
==> ceph-node3: Configuring and enabling network interfaces...
ceph-node3: SSH address: 127.0.0.1:2201
ceph-node3: SSH username: vagrant
ceph-node3: SSH auth method: private key
==> ceph-node3: Running provisioner: shell...
ceph-node3: Running: /tmp/vagrant-shell20170902-3782-1qbpjpv.sh
- Check the status of your virtual machines:
[ab@ceph-host ceph-cookbook]$ pwd /home/ab/ceph-cookbook [ab@ceph-host ceph-cookbook]$ vagrant status ceph-node1 ceph-node2 ceph-node3 Current machine states: ceph-node1 running (virtualbox) ceph-node2 running (virtualbox) ceph-node3 running (virtualbox) This environment represents multiple VMs. The VMs are all listed above with their current state. For more information about a specific VM, run `vagrant status NAME`.
- Above three nodes have the following:
root password: 'vagrant'username 'vagrant' with 'vagrant' password.
'vagrant' username has full SUDO rule without asking for password
- According to Ceph Cookbook, 'vboxnet0' Network Should be 192.168.1.0/24. so we need to fix DHCP range from (192.168.56.101-192.168.56as the following:
Before
After
- Restart the three VMs:
[ab@ceph-host ceph-cookbook]$ pwd /home/ab/ceph-cookbook
[ab@ceph-host ceph-cookbook]$ vagrant ssh ceph-node1 Last login: Sun Apr 5 19:05:19 2015 [vagrant@ceph-node1 ~]$ sudo reboot Connection to 127.0.0.1 closed by remote host. Connection to 127.0.0.1 closed.
[ab@ceph-host ceph-cookbook]$ vagrant ssh ceph-node2 Last login: Sun Apr 5 19:05:19 2015 [vagrant@ceph-node2 ~]$ sudo reboot Connection to 127.0.0.1 closed by remote host. Connection to 127.0.0.1 closed.
[ab@ceph-host ceph-cookbook]$ vagrant ssh ceph-node3 Last login: Sun Apr 5 19:05:19 2015 [vagrant@ceph-node3 ~]$ sudo reboot Connection to 127.0.0.1 closed by remote host. Connection to 127.0.0.1 closed.
- Collect Info from each node. (vagrant will create /dev/sd{b,cd} for Ceph usage):
ceph-node1
[ab@ceph-host ceph-cookbook]$ pwd /home/ab/ceph-cookbook [ab@ceph-host ceph-cookbook]$ vagrant ssh ceph-node1 Last login: Sat Sep 2 21:34:34 2017 from 10.0.2.2
[vagrant@ceph-node1 ~]$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 08:00:27:c5:46:4e brd ff:ff:ff:ff:ff:ff
inet 10.0.2.15/24 brd 10.0.2.255 scope global dynamic enp0s3
valid_lft 86310sec preferred_lft 86310sec
inet6 fe80::a00:27ff:fec5:464e/64 scope link
valid_lft forever preferred_lft forever
3: enp0s8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 08:00:27:af:36:58 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.101/24 brd 192.168.1.255 scope global enp0s8
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:feaf:3658/64 scope link
valid_lft forever preferred_lft forever
[vagrant@ceph-node1 ~]$ hostname
ceph-node1
[vagrant@ceph-node1 ~]$ cat /proc/partitions
major minor #blocks name
11 0 1048575 sr0
8 16 20971520 sdb
8 48 20971520 sdd
8 32 20971520 sdc
8 0 8388608 sda
8 1 512000 sda1
8 2 7875584 sda2
253 0 839680 dm-0
253 1 7032832 dm-1
[vagrant@ceph-node1 ~]$ grep ceph-node /etc/hosts
127.0.0.1 ceph-node1 ceph-node1
192.168.1.101 ceph-node1
192.168.1.102 ceph-node2
192.168.1.103 ceph-node3
192.168.1.104 ceph-node4
192.168.1.105 ceph-node5
[vagrant@ceph-node1 ~]$ logout Connection to 127.0.0.1 closed.
ceph-node2
[ab@ceph-host ceph-cookbook]$ pwd /home/ab/ceph-cookbook
[ab@ceph-host ceph-cookbook]$ vagrant ssh ceph-node2
Last login: Sat Sep 2 21:45:50 2017 from 10.0.2.2
[vagrant@ceph-node2 ~]$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 08:00:27:c5:46:4e brd ff:ff:ff:ff:ff:ff
inet 10.0.2.15/24 brd 10.0.2.255 scope global dynamic enp0s3
valid_lft 85763sec preferred_lft 85763sec
inet6 fe80::a00:27ff:fec5:464e/64 scope link
valid_lft forever preferred_lft forever
3: enp0s8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 08:00:27:44:73:09 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.102/24 brd 192.168.1.255 scope global enp0s8
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fe44:7309/64 scope link
valid_lft forever preferred_lft forever
[vagrant@ceph-node2 ~]$ hostname
ceph-node2
[vagrant@ceph-node2 ~]$ cat /proc/partitions
major minor #blocks name
11 0 1048575 sr0
8 0 8388608 sda
8 1 512000 sda1
8 2 7875584 sda2
8 32 20971520 sdc
8 48 20971520 sdd
8 16 20971520 sdb
253 0 839680 dm-0
253 1 7032832 dm-1
[vagrant@ceph-node2 ~]$ grep ceph-node /etc/hosts
127.0.0.1 ceph-node2 ceph-node2
192.168.1.101 ceph-node1
192.168.1.102 ceph-node2
192.168.1.103 ceph-node3
192.168.1.104 ceph-node4
192.168.1.105 ceph-node5
[vagrant@ceph-node2 ~]$ logout Connection to 127.0.0.1 closed.
ceph-node3
[ab@ceph-host ceph-cookbook]$ pwd /home/ab/ceph-cookbook
[ab@ceph-host ceph-cookbook]$ vagrant ssh ceph-node3
Last login: Sat Sep 2 21:49:24 2017 from 10.0.2.2
[vagrant@ceph-node3 ~]$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 08:00:27:c5:46:4e brd ff:ff:ff:ff:ff:ff
inet 10.0.2.15/24 brd 10.0.2.255 scope global dynamic enp0s3
valid_lft 85568sec preferred_lft 85568sec
inet6 fe80::a00:27ff:fec5:464e/64 scope link
valid_lft forever preferred_lft forever
3: enp0s8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 08:00:27:9c:12:4b brd ff:ff:ff:ff:ff:ff
inet 192.168.1.103/24 brd 192.168.1.255 scope global enp0s8
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fe9c:124b/64 scope link
valid_lft forever preferred_lft forever
[vagrant@ceph-node3 ~]$ hostname
ceph-node3
[vagrant@ceph-node3 ~]$ cat /proc/partitions
major minor #blocks name
8 48 20971520 sdd
8 16 20971520 sdb
8 32 20971520 sdc
8 0 8388608 sda
8 1 512000 sda1
8 2 7875584 sda2
11 0 1048575 sr0
253 0 839680 dm-0
253 1 7032832 dm-1
[vagrant@ceph-node3 ~]$ grep ceph-node /etc/hosts
127.0.0.1 ceph-node3 ceph-node3
192.168.1.101 ceph-node1
192.168.1.102 ceph-node2
192.168.1.103 ceph-node3
192.168.1.104 ceph-node4
192.168.1.105 ceph-node5
[vagrant@ceph-node3 ~]$ logout Connection to 127.0.0.1 closed.
- Check Network connectivity between the three nodes:
[ab@ceph-host ceph-cookbook]$ pwd /home/ab/ceph-cookbook
[ab@ceph-host ceph-cookbook]$ vagrant ssh ceph-node1
Last login: Sat Sep 2 21:37:53 2017 from 10.0.2.2
[vagrant@ceph-node1 ~]$ for i in {1..3}; do echo ceph-node${i} ; echo "=========="; ping -c3 ceph-node${i}; echo ""; done
ceph-node1
==========
PING ceph-node1 (127.0.0.1) 56(84) bytes of data.
64 bytes from ceph-node1 (127.0.0.1): icmp_seq=1 ttl=64 time=1.76 ms
64 bytes from ceph-node1 (127.0.0.1): icmp_seq=2 ttl=64 time=5.85 ms
64 bytes from ceph-node1 (127.0.0.1): icmp_seq=3 ttl=64 time=0.241 ms
--- ceph-node1 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2006ms
rtt min/avg/max/mdev = 0.241/2.620/5.852/2.369 ms
ceph-node2
==========
PING ceph-node2 (192.168.1.102) 56(84) bytes of data.
64 bytes from ceph-node2 (192.168.1.102): icmp_seq=1 ttl=64 time=1.21 ms
64 bytes from ceph-node2 (192.168.1.102): icmp_seq=2 ttl=64 time=4.72 ms
64 bytes from ceph-node2 (192.168.1.102): icmp_seq=3 ttl=64 time=6.50 ms
--- ceph-node2 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2009ms
rtt min/avg/max/mdev = 1.218/4.151/6.509/2.199 ms
ceph-node3
==========
PING ceph-node3 (192.168.1.103) 56(84) bytes of data.
64 bytes from ceph-node3 (192.168.1.103): icmp_seq=1 ttl=64 time=6.74 ms
64 bytes from ceph-node3 (192.168.1.103): icmp_seq=2 ttl=64 time=4.98 ms
64 bytes from ceph-node3 (192.168.1.103): icmp_seq=3 ttl=64 time=3.61 ms
--- ceph-node3 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2023ms
rtt min/avg/max/mdev = 3.618/5.116/6.747/1.282 ms
- Generate root SSH keys for ceph-node1 and copy the keys to ceph-node2 and ceph-node3. The password for root user on these VMs is vagrant:
[ab@ceph-host ceph-cookbook]$ pwd /home/ab/ceph-cookbook
[ab@ceph-host ceph-cookbook]$ vagrant ssh ceph-node1 Last login: Sat Sep 2 21:52:29 2017 from 10.0.2.2 [vagrant@ceph-node1 ~]$ sudo su - Last login: Sun Apr 5 19:05:17 EEST 2015 on tty1 [root@ceph-node1 ~]# ssh-keygen Generating public/private rsa key pair. Enter file in which to save the key (/root/.ssh/id_rsa): Created directory '/root/.ssh'. Enter passphrase (empty for no passphrase): Enter same passphrase again: Your identification has been saved in /root/.ssh/id_rsa. Your public key has been saved in /root/.ssh/id_rsa.pub. The key fingerprint is: b9:71:9f:fe:83:5d:2b:57:aa:6e:4d:75:7a:72:41:83 root@ceph-node1 The key's randomart image is: +--[ RSA 2048]----+ | . | | E o | | . .| | . .o| | S . oo| | + . .+ =| | . o= *o| | .o.=o | | o+o+. | +-----------------+
[root@ceph-node1 ~]# ssh-copy-id root@ceph-node2 The authenticity of host 'ceph-node2 (192.168.1.102)' can't be established. ECDSA key fingerprint is af:2a:a5:74:a7:0b:f5:5b:ef:c5:4b:2a:fe:1d:30:8e. Are you sure you want to continue connecting (yes/no)? yes /bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed /bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys root@ceph-node2's password: Number of key(s) added: 1 Now try logging into the machine, with: "ssh 'root@ceph-node2'" and check to make sure that only the key(s) you wanted were added.
[root@ceph-node1 ~]# ssh-copy-id root@ceph-node3 The authenticity of host 'ceph-node3 (192.168.1.103)' can't be established. ECDSA key fingerprint is af:2a:a5:74:a7:0b:f5:5b:ef:c5:4b:2a:fe:1d:30:8e. Are you sure you want to continue connecting (yes/no)? yes /bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed /bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys root@ceph-node3's password: Number of key(s) added: 1 Now try logging into the machine, with: "ssh 'root@ceph-node3'" and check to make sure that only the key(s) you wanted were added.
[root@ceph-node1 ~]# ssh ceph-node2 hostname ceph-node2 [root@ceph-node1 ~]# ssh ceph-node3 hostname ceph-node3
- Enable ports that are required by the Ceph MON, OSD, and MDS on the operating system's firewall. Execute the following commands on all VMs:
ceph-node1
[root@ceph-node1 ~]# firewall-cmd --zone=public --add-port=6789/tcp --permanent success [root@ceph-node1 ~]# firewall-cmd --zone=public --add-port=6800-7100/tcp --permanent success [root@ceph-node1 ~]# firewall-cmd --reload success [root@ceph-node1 ~]# firewall-cmd --zone=public --list-all public (default, active) interfaces: enp0s3 enp0s8 sources: services: dhcpv6-client ssh ports: 6789/tcp 6800-7100/tcp masquerade: no forward-ports: icmp-blocks: rich rules:
ceph-node2
[root@ceph-node2 ~]# firewall-cmd --zone=public --add-port=6789/tcp --permanent success [root@ceph-node2 ~]# firewall-cmd --zone=public --add-port=6800-7100/tcp --permanent success [root@ceph-node2 ~]# firewall-cmd --reload success [root@ceph-node2 ~]# firewall-cmd --zone=public --list-all public (default, active) interfaces: enp0s3 enp0s8 sources: services: dhcpv6-client ssh ports: 6789/tcp 6800-7100/tcp masquerade: no forward-ports: icmp-blocks: rich rules:
ceph-node3
[root@ceph-node3 ~]# firewall-cmd --zone=public --add-port=6789/tcp --permanent success [root@ceph-node3 ~]# firewall-cmd --zone=public --add-port=6800-7100/tcp --permanent success [root@ceph-node3 ~]# firewall-cmd --reload success [root@ceph-node3 ~]# firewall-cmd --zone=public --list-all public (default, active) interfaces: enp0s3 enp0s8 sources: services: dhcpv6-client ssh ports: 6789/tcp 6800-7100/tcp masquerade: no forward-ports: icmp-blocks: rich rules:
- Make sure that SELinux is disabled:
ceph-node1
[root@ceph-node1 ~]# grep -i =disabled /etc/sysconfig/selinux SELINUX=disabled
ceph-node2
[root@ceph-node2 ~]# grep -i =disabled /etc/sysconfig/selinux SELINUX=disabled
ceph-node3
[root@ceph-node3 ~]# grep -i =disabled /etc/sysconfig/selinux SELINUX=disabled
- Install and configure ntp on all VMs:
ceph-node1
[root@ceph-node1 ~]# yum install ntp ntpdate -y
[root@ceph-node1 ~]# ntpq -p
remote refid st t when poll reach delay offset jitter
==============================================================================
*ntp1.torix.ca .PPS. 1 u 43 64 3 16.191 1.989 0.441
+cloud1.pifts.ne 138.39.23.13 2 u 43 64 3 3.813 -0.144 0.978
-briareus.schult 129.6.15.30 2 u 42 64 3 2.752 -1.775 1.038
+host1.hosttechn 213.251.128.249 2 u 42 64 3 2.922 0.758 0.974
[root@ceph-node1 ~]# systemctl enable ntpd.service ntpdate.service
[root@ceph-node1 ~]# systemctl is-enabled ntpd.service ntpdate.service
enabled
enabled
[root@ceph-node1 ~]# reboot
ceph-node2
[root@ceph-node2 ~]# yum install ntp ntpdate -y
[root@ceph-node2 ~]# ntpq -p
remote refid st t when poll reach delay offset jitter
==============================================================================
time.srv.ualber 129.128.153.62 2 u 1 64 1 80.838 -17.006 0.730
dense.utcc.utor 128.100.100.100 2 u 2 64 1 24.918 -2.570 0.817
istntpprd-02.co .GPS. 1 u 1 64 1 80.757 -14.073 1.381
mirror3.rafal.c 53.27.192.223 2 u 2 64 1 15.996 0.421 0.031
[root@ceph-node2 ~]# systemctl enable ntpd.service ntpdate.service
[root@ceph-node2 ~]# systemctl is-enabled ntpd.service ntpdate.service
enabled
enabled
[root@ceph-node2 ~]# reboot
ceph-node3
[root@ceph-node3 ~]# yum install ntp ntpdate -y
[root@ceph-node3 ~]# ntpq -p
remote refid st t when poll reach delay offset jitter
==============================================================================
ground.corbina. 193.11.166.20 2 u 1 64 1 145.908 -4.968 0.031
CPEdeadbeefcafe 3.83.151.234 2 u - 64 1 49.744 -5.998 0.031
sanction.trebor 192.5.41.209 2 u 1 64 1 9.880 -2.212 0.031
mirror3.rafal.c .INIT. 16 u - 64 0 0.000 0.000 0.031
[root@ceph-node3 ~]# systemctl enable ntpd.service ntpdate.service
[root@ceph-node3 ~]# systemctl is-enabled ntpd.service ntpdate.service
enabled
enabled
[root@ceph-node3 ~]# reboot
- Take cold Snapshot:
[vagrant@ceph-node1 ~]$ sudo poweroff [vagrant@ceph-node2 ~]$ sudo poweroff [vagrant@ceph-node3 ~]$ sudo poweroff [root@ceph-host ~]# poweroff
virsh # snapshot-create-as ceph-host 01_before_OS_update Domain snapshot 01_before_OS_update created virsh # snapshot-current ceph-host --name 01_before_OS_update
- Add repositories on all nodes for the Ceph "jewel" version and update yum:
Do the following from GUI one VM at time and the others are OFF:
ceph-node1
[root@ceph-node1 ~]# rpm -Uhv http://download.ceph.com/rpm-jewel/el7/noarch/ceph-release-1-1.el7.noarch.rpm Retrieving http://download.ceph.com/rpm-jewel/el7/noarch/ceph-release-1-1.el7.noarch.rpm warning: /var/tmp/rpm-tmp.JcDZZh: Header V4 RSA/SHA256 Signature, key ID 460f3994: NOKEY Preparing... ################################# [100%] Updating / installing... 1:ceph-release-1-1.el7 ################################# [100%] [root@ceph-node1 ~]# yum update -y [root@ceph-node1 ~]# reboot
ceph-node2
[root@ceph-node2 ~]# rpm -Uhv http://download.ceph.com/rpm-jewel/el7/noarch/ceph-release-1-1.el7.noarch.rpm Retrieving http://download.ceph.com/rpm-jewel/el7/noarch/ceph-release-1-1.el7.noarch.rpm warning: /var/tmp/rpm-tmp.JcDZZh: Header V4 RSA/SHA256 Signature, key ID 460f3994: NOKEY Preparing... ################################# [100%] Updating / installing... 1:ceph-release-1-1.el7 ################################# [100%] [root@ceph-node2 ~]# yum update -y [root@ceph-node2 ~]# reboot
ceph-node3
[root@ceph-node3 ~]# rpm -Uhv http://download.ceph.com/rpm-jewel/el7/noarch/ceph-release-1-1.el7.noarch.rpm Retrieving http://download.ceph.com/rpm-jewel/el7/noarch/ceph-release-1-1.el7.noarch.rpm warning: /var/tmp/rpm-tmp.JcDZZh: Header V4 RSA/SHA256 Signature, key ID 460f3994: NOKEY Preparing... ################################# [100%] Updating / installing... 1:ceph-release-1-1.el7 ################################# [100%] [root@ceph-node3 ~]# yum update -y [root@ceph-node3 ~]# reboot
- Take "cold" snapshot:
[vagrant@ceph-node1 ~]$ sudo poweroff [vagrant@ceph-node2 ~]$ sudo poweroff [vagrant@ceph-node3 ~]$ sudo poweroff [root@ceph-host ~]# poweroff
virsh # snapshot-create-as ceph-host 02_ready_to_deploy_ceph Domain snapshot 02_ready_to_deploy_ceph created virsh # snapshot-current ceph-host --name 02_ready_to_deploy_ceph virsh # snapshot-list ceph-host Name Creation Time State ------------------------------------------------------------ 00_ready_to_deploy 2017-09-01 16:51:59 +0200 shutoff 01_before_OS_update 2017-09-03 13:40:17 +0200 shutoff 02_ready_to_deploy_ceph 2017-09-03 14:46:38 +0200 shutoff
