All Ceph operator should agree with me when i say that Ceph installer doesn't really works as expected so far.
Yes, i'm talking about ceph-deploy and the main reason why i'm posting this guide about deploying Ceph with Ansible.
At this post, i will show how to install a Ceph cluster with Ansible on baremetal servers.
Copy sample vars files, we will configure our environment in these variable files.
Next step is configure the inventory with our servers, i don\'t really like use /etc/ansible/host file, i prefer create a new file per environment inside playbook\'s folder.
Create a file with the following content, use you own IPs to match your servers on the desired role inside the cluster
Edit site.yml file, i will remove/comment mds nodes since i\'m not going to use them.
Here we configure from where ceph packages are going to be installed, for now we use upstream code with the stable release Infernalis.
Here we configure some OSD options, like journal size and what networks will be used by public and cluster data replication
I will use auto discovery option to allow ceph ansible select empy or not used devices in my servers to create OSDs.
| Of course you can use other options, i\'ll highly suggest you to read variable comments, as they provide valuable information about usage. | We\'re ready to deploy ceph with ansible with our custom inventory_hosts file.
After a while, you will have a fully functional ceph cluster.
| Maybe you find some issues or bugs when running the playbooks. | There is a lot of efforts to fix issues on upstream repository. If a new bug is encountered, please, post a issue right here. | https://github.com/ceph/ceph-ansible/issues
You can check your cluster status with ceph -s. we can see all OSDs are up and pgs active/clean.
| We are going to do some tests. | Create a pool
Query the placement group where you file was uploaded, a similar output will prompts
Copy [root@ceph-mon1 ~]# ceph pg 1.39 query
{
"state": "active+clean",
"snap_trimq": "[]",
"epoch": 13,
"up": [
1,
0
],
"acting": [
1,
0
],
"actingbackfill": [
"0",
"1"
],
"info": {
"pgid": "1.39",
"last_update": "13'500",
"last_complete": "13'500",
"log_tail": "0'0",
"last_user_version": 500,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 11,
"last_epoch_started": 12,
"last_epoch_clean": 13,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 11,
"same_interval_since": 11,
"same_primary_since": 11,
"last_scrub": "0'0",
"last_scrub_stamp": "2016-03-16 21:13:08.883121",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2016-03-16 21:13:08.883121",
"last_clean_scrub_stamp": "0.000000"
},
"stats": {
"version": "13'500",
"reported_seq": "505",
"reported_epoch": "13",
"state": "active+clean",
"last_fresh": "2016-03-16 21:24:40.930724",
"last_change": "2016-03-16 21:14:09.874086",
"last_active": "2016-03-16 21:24:40.930724",
"last_peered": "2016-03-16 21:24:40.930724",
"last_clean": "2016-03-16 21:24:40.930724",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-03-16 21:24:40.930724",
"last_undegraded": "2016-03-16 21:24:40.930724",
"last_fullsized": "2016-03-16 21:24:40.930724",
"mapping_epoch": 11,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 11,
"last_epoch_clean": 13,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2016-03-16 21:13:08.883121",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2016-03-16 21:13:08.883121",
"last_clean_scrub_stamp": "0.000000",
"log_size": 500,
"ondisk_log_size": 500,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 2097152000,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 2,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 500,
"num_write_kb": 2048000,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0
},
"up": [
1,
0
],
"acting": [
1,
0
],
"blocked_by": [],
"up_primary": 1,
"acting_primary": 1
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 12,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "0",
"pgid": "1.39",
"last_update": "13'500",
"last_complete": "13'500",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 11,
"last_epoch_started": 12,
"last_epoch_clean": 13,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2016-03-16 21:13:08.883121",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2016-03-16 21:13:08.883121",
"last_clean_scrub_stamp": "0.000000"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0
},
"up": [],
"acting": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 12,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Active",
"enter_time": "2016-03-16 21:13:36.769083",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "0",
"scrubber.active": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2016-03-16 21:13:09.216260"
}
],
"agent_state": {}
} That\'s all for now.
