Merge pull request #97 from stackhpc/xena-cephadm

Xena: Add playbooks & configuration for Cephadm

Author: markgoddard

etc/kayobe/ansible/cephadm-crush-rules.yml

@@ -0,0 +1,11 @@
+---
+- name: Define Ceph CRUSH rules
+  gather_facts: false
+  hosts: mons
+  become: true
+  tags:
+    - cephadm
+    - cephadm-crush-rules
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.crush_rules

etc/kayobe/ansible/cephadm-deploy.yml

@@ -0,0 +1,11 @@
+---
+- name: Deploy Ceph using Cephadm
+  any_errors_fatal: true
+  gather_facts: true
+  hosts: ceph
+  tags:
+    - cephadm
+    - cephadm-deploy
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.cephadm

etc/kayobe/ansible/cephadm-ec-profiles.yml

@@ -0,0 +1,11 @@
+---
+- name: Define Ceph EC profiles
+  gather_facts: false
+  hosts: mons
+  become: true
+  tags:
+    - cephadm
+    - cephadm-ec-profiles
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.ec_profiles

etc/kayobe/ansible/cephadm-gather-keys.yml

@@ -0,0 +1,82 @@
+---
+- name: Gather Ceph configuration and keys and populate kayobe-config
+  gather_facts: false
+  hosts: mons
+  run_once: true
+  tags:
+    - cephadm
+    - cephadm-gather-keys
+  vars:
+    # Map from an OpenStack service to the directory in which to store Ceph keys for it.
+    kolla_service_to_key_dir:
+      cinder-backup: cinder/cinder-backup
+      cinder-volume: cinder/cinder-volume
+      glance: glance
+      manila: manila
+      nova: nova
+    # Map from an OpenStack service to the directory in which to store Ceph configuration for it.
+    kolla_service_to_conf_dir:
+      cinder-backup: cinder
+      cinder-volume: cinder
+      glance: glance
+      manila: manila
+      nova: nova
+  tasks:
+    - name: Get Ceph keys
+      stackhpc.cephadm.cephadm_key:
+        name: "{{ item }}"
+        state: info
+      register: cephadm_key_info
+      become: true
+      loop: "{{ kolla_ceph_services | selectattr('required') | map(attribute='keys') | flatten | unique }}"
+
+    - name: Generate ceph.conf
+      command: "cephadm shell -- ceph config generate-minimal-conf"
+      become: true
+      register: cephadm_ceph_conf
+      changed_when: false
+
+    - name: Ensure Kolla config directories are present
+      file:
+        state: directory
+        path: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.name] }}"
+      loop: "{{ kolla_ceph_services | selectattr('required') }}"
+      loop_control:
+        label:
+          service: "{{ item.name }}"
+      delegate_to: localhost
+
+    - name: Save Ceph keys to Kayobe configuration
+      vars:
+        key_info: "{{ cephadm_key_info.results | selectattr('item', 'equalto', item.1) | first }}"
+        cephadm_key: "{{ (key_info.stdout | from_json | first)['key'] }}"
+        cephadm_user: "{{ (key_info.stdout | from_json | first)['entity'] }}"
+      copy:
+        content: "{{ cephadm_key }}"
+        dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.0.name] }}/ceph.{{ cephadm_user }}.keyring"
+      loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}"
+      loop_control:
+        label:
+          service: "{{ item.0.name }}"
+          key: "{{ item.1 }}"
+      delegate_to: localhost
+      notify: Please add and commit the Kayobe configuration
+
+    - name: Save ceph.conf to Kayobe configuration
+      copy:
+        content: "{{ cephadm_ceph_conf.stdout }}"
+        dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf"
+      loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}"
+      loop_control:
+        label:
+          service: "{{ item.0.name }}"
+          key: "{{ item.1 }}"
+      delegate_to: localhost
+      notify: Please add and commit the Kayobe configuration
+
+  handlers:
+    - name: Please add and commit the Kayobe configuration
+      debug:
+        msg: >-
+          Please add and commit the Ceph configuration files and keys in Kayobe
+          configuration. Remember to encrypt the keys using Ansible Vault.

etc/kayobe/ansible/cephadm-keys.yml

@@ -0,0 +1,11 @@
+---
+- name: Generate Ceph keys
+  gather_facts: false
+  hosts: mons
+  become: true
+  tags:
+    - cephadm
+    - cephadm-keys
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.keys

etc/kayobe/ansible/cephadm-pools.yml

@@ -0,0 +1,11 @@
+---
+- name: Define Ceph pools
+  gather_facts: false
+  hosts: mons
+  become: true
+  tags:
+    - cephadm
+    - cephadm-keys
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.pools

etc/kayobe/ansible/cephadm.yml

@@ -0,0 +1,7 @@
+---
+# Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys.
+- import_playbook: cephadm-deploy.yml
+- import_playbook: cephadm-ec-profiles.yml
+- import_playbook: cephadm-crush-rules.yml
+- import_playbook: cephadm-pools.yml
+- import_playbook: cephadm-keys.yml

etc/kayobe/ansible/requirements.yml

@@ -1,5 +1,7 @@
 ---
 collections:
+  - name: stackhpc.cephadm
+    version: 1.7.0
   - name: stackhpc.pulp
     version: 0.3.0
   - name: pulp.squeezer

etc/kayobe/cephadm.yml

@@ -0,0 +1,100 @@
+---
+###############################################################################
+# Cephadm deployment configuration.
+
+# Ceph release name.
+cephadm_ceph_release: "pacific"
+
+# Ceph FSID.
+#cephadm_fsid:
+
+# Ceph container image.
+cephadm_image: "{{ stackhpc_docker_registry }}/ceph/ceph:{{ cephadm_image_tag }}"
+
+# Ceph container image tag.
+cephadm_image_tag: "v16.2.5"
+
+# Ceph container image registry URL.
+cephadm_registry_url: "{{ stackhpc_docker_registry }}"
+
+# Ceph container image registry username.
+cephadm_registry_username: "{{ stackhpc_docker_registry_username }}"
+
+# Ceph container image registry password.
+cephadm_registry_password: "{{ stackhpc_docker_registry_password }}"
+
+# Ceph public network interface.
+cephadm_public_interface: "{{ storage_net_name | net_interface }}"
+
+# Ceph public network subnet CIDR.
+cephadm_public_network: "{{ storage_net_name | net_cidr }}"
+
+# Ceph cluster network interface.
+cephadm_cluster_interface: "{{ storage_mgmt_net_name | net_interface }}"
+
+# Ceph cluster network subnet CIDR.
+cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}"
+
+# Whether to enable firewalld for Ceph storage hosts.
+#cephadm_enable_firewalld:
+
+# Ceph OSD specification.
+#cephadm_osd_spec:
+
+###############################################################################
+# Ceph post-deployment configuration.
+
+# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role
+# for format.
+#cephadm_ec_profiles:
+
+# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format.
+#cephadm_crush_rules:
+
+# List of Ceph pools. See stackhpc.cephadm.pools role for format.
+#cephadm_pools:
+
+# List of Cephx keys. See stackhpc.cephadm.keys role for format.
+#cephadm_keys:
+
+###############################################################################
+# Kolla Ceph auto-configuration.
+
+# List of Kolla Ansible services that require Ceph configuration files.
+kolla_ceph_services:
+  - name: cinder-backup
+    keys:
+      - client.cinder
+      - client.cinder-backup
+    required: "{{ kolla_ceph_cinder_backup_required | bool }}"
+  - name: cinder-volume
+    keys:
+      - client.cinder
+    required: "{{ kolla_ceph_cinder_volume_required | bool }}"
+  - name: glance
+    keys:
+      - client.glance
+    required: "{{ kolla_ceph_glance_required | bool }}"
+  - name: manila
+    keys:
+      - client.manila
+    required: "{{ kolla_ceph_manila_required | bool }}"
+  - name: nova
+    keys:
+      - client.cinder
+    required: "{{ kolla_ceph_nova_required | bool }}"
+
+# Whether to generate Ceph configuration for Cinder volume.
+kolla_ceph_cinder_volume_required: "{{ kolla_enable_cinder | bool }}"
+
+# Whether to generate Ceph configuration for Cinder backup.
+kolla_ceph_cinder_backup_required: "{{ kolla_enable_cinder_backup | default(false) | bool }}"
+
+# Whether to generate Ceph configuration for Glance.
+kolla_ceph_glance_required: "{{ kolla_enable_glance | bool }}"
+
+# Whether to generate Ceph configuration for Manila.
+kolla_ceph_manila_required: "{{ kolla_enable_manila | bool }}"
+
+# Whether to generate Ceph configuration for Nova.
+kolla_ceph_nova_required: "{{ kolla_enable_nova | bool }}"

etc/kayobe/inventory/groups

@@ -97,3 +97,18 @@ overcloud
 mgmt-switches
 ctl-switches
 hs-switches
+
+###############################################################################
+# Ceph groups.
+
+[ceph:children]
+mons
+mgrs
+osds
+rgws
+
+# Empty groups to provide declaration of groups required for Cephadm.
+[mons]
+[mgrs]
+[osds]
+[rgws]

etc/kayobe/pulp.yml

@@ -308,7 +308,7 @@ stackhpc_pulp_distribution_rpm_production:
 stackhpc_release_pulp_registry_url: "{{ stackhpc_release_pulp_url }}"
 
 # List of all image names.
-stackhpc_pulp_images:
+stackhpc_pulp_images_kolla:
   - barbican-api
   - barbican-base
   - barbican-keystone-listener
@@ -429,33 +429,64 @@ stackhpc_pulp_images:
   - redis-sentinel
 
 # Common parameters for container image repositories.
-stackhpc_pulp_repository_container_repos_common:
+stackhpc_pulp_repository_container_repos_kolla_common:
   url: "{{ stackhpc_release_pulp_registry_url }}"
   policy: on_demand
   remote_username: "{{ stackhpc_release_pulp_username }}"
   remote_password: "{{ stackhpc_release_pulp_password }}"
   state: present
+  required: true
 
-# List of container image repositories.
-stackhpc_pulp_repository_container_repos: >-
+# List of Kolla container image repositories.
+stackhpc_pulp_repository_container_repos_kolla: >-
   {%- set repos = [] -%}
-  {%- for image in stackhpc_pulp_images -%}
+  {%- for image in stackhpc_pulp_images_kolla -%}
   {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%}
   {%- set repo = {"name": image_repo} -%}
-  {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_common | combine(repo)) -%}
+  {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_kolla_common | combine(repo)) -%}
   {%- endfor -%}
   {{ repos }}
 
-# Common parameters for container image distributions.
-stackhpc_pulp_distribution_container_common:
+# Common parameters for Kolla container image distributions.
+stackhpc_pulp_distribution_container_kolla_common:
   state: present
+  required: true
 
-# List of container image distributions.
-stackhpc_pulp_distribution_container: >-
+# List of Kolla container image distributions.
+stackhpc_pulp_distribution_container_kolla: >-
   {%- set distributions = [] -%}
-  {%- for image in stackhpc_pulp_images -%}
+  {%- for image in stackhpc_pulp_images_kolla -%}
   {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%}
   {%- set distribution = {"name": image_repo, "repository": image_repo, "base_path": image_repo} -%}
-  {%- set _ = distributions.append(stackhpc_pulp_distribution_container_common | combine(distribution)) -%}
+  {%- set _ = distributions.append(stackhpc_pulp_distribution_container_kolla_common | combine(distribution)) -%}
   {%- endfor -%}
   {{ distributions }}
+
+# Whether to sync Ceph container images.
+stackhpc_sync_ceph_images: false
+
+# List of Ceph container image repositories.
+stackhpc_pulp_repository_container_repos_ceph:
+  - name: "ceph/ceph"
+    url: "https://quay.io"
+    policy: on_demand
+    state: present
+    required: "{{ stackhpc_sync_ceph_images | bool }}"
+
+# List of Ceph container image distributions.
+stackhpc_pulp_distribution_container_ceph:
+  - name: ceph
+    repository: ceph/ceph
+    base_path: ceph/ceph
+    state: present
+    required: "{{ stackhpc_sync_ceph_images | bool }}"
+
+# List of container image repositories.
+stackhpc_pulp_repository_container_repos: >-
+  {{ (stackhpc_pulp_repository_container_repos_kolla +
+      stackhpc_pulp_repository_container_repos_ceph) | selectattr('required') }}
+
+# List of container image distributions.
+stackhpc_pulp_distribution_container: >-
+  {{ (stackhpc_pulp_distribution_container_kolla +
+      stackhpc_pulp_distribution_container_ceph) | selectattr('required') }}