Merge pull request #89 from stackhpc/ubuntu

Add support for Ubuntu repositories

Author: markgoddard

README.rst

@@ -230,7 +230,7 @@ ci-aio
 Prerequisites
 ^^^^^^^^^^^^^
 
-* a CentOS Stream 8 host
+* a CentOS Stream 8 or Ubuntu Focal 20.04 host
 * access to the local Pulp server
 
 Setup
@@ -329,7 +329,7 @@ service, and pushed there once built.
 Prerequisites
 ^^^^^^^^^^^^^
 
-* a CentOS Stream 8 host
+* a CentOS Stream 8 or Ubuntu Focal 20.04 host
 * access to the local Pulp server
 
 Setup

etc/kayobe/ansible/pulp-repo-promote-production.yml

@@ -6,4 +6,5 @@
     - import_role:
         name: stackhpc.pulp.pulp_distribution
       vars:
-        pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_production }}"
+        pulp_distribution_deb: "{{ stackhpc_pulp_distribution_deb_production | selectattr('required') }}"
+        pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_production | selectattr('required') }}"

etc/kayobe/ansible/pulp-repo-publish.yml

@@ -7,9 +7,11 @@
         name: stackhpc.pulp.pulp_publication
       # NOTE: use intermediate variable to avoid publishing containers.
       vars:
-        pulp_publication_rpm: "{{ stackhpc_pulp_publication_rpm_development }}"
+        pulp_publication_deb: "{{ stackhpc_pulp_publication_deb_development | selectattr('required') }}"
+        pulp_publication_rpm: "{{ stackhpc_pulp_publication_rpm_development | selectattr('required') }}"
 
     - import_role:
         name: stackhpc.pulp.pulp_distribution
       vars:
-        pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_development }}"
+        pulp_distribution_deb: "{{ stackhpc_pulp_distribution_deb_development | selectattr('required') }}"
+        pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_development | selectattr('required') }}"

etc/kayobe/ansible/pulp-repo-sync.yml

@@ -7,4 +7,5 @@
         name: stackhpc.pulp.pulp_repository
       # NOTE: use intermediate variable to avoid syncing containers.
       vars:
-        pulp_repository_rpm_repos: "{{ stackhpc_pulp_repository_rpm_repos }}"
+        pulp_repository_deb_repos: "{{ stackhpc_pulp_repository_deb_repos | selectattr('required') }}"
+        pulp_repository_rpm_repos: "{{ stackhpc_pulp_repository_rpm_repos | selectattr('required') }}"

etc/kayobe/ansible/purge-command-not-found.yml

@@ -0,0 +1,33 @@
+---
+# Currently the pulp_deb plugin in Pulp does not support certain types of
+# content, including i18n files and command-not-found indices. This breaks APT
+# when the command-not-found is installed. This playbook can be used to
+# uninstall the package, prior to running any other APT commands. It may be
+# installed as a hook to the 'host configure' commands.
+# See https://github.com/pulp/pulp_deb/issues/419
+# FIXME: If used as a hook, this playbook matches all hosts, so will run
+# against the seed, even when running 'overcloud host configure'. Depending on
+# the stage of deployment, some hosts may be unreachable. This could be fixed
+# by implementing this playbook separately for each group.
+- name: Purge command-not-found package
+  hosts: seed-hypervisor:seed:overcloud:infra-vms
+  gather_facts: false
+  vars:
+    ansible_user: "{{ bootstrap_user }}"
+    # We can't assume that a virtualenv exists at this point, so use the system
+    # python interpreter.
+    ansible_python_interpreter: /usr/bin/python3
+    # Work around no known_hosts entry on first boot.
+    ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+  tasks:
+    - name: Purge command-not-found package
+      package:
+        name:
+          - command-not-found
+          - python3-command-not-found
+        purge: true
+        state: absent
+      become: true
+      failed_when: false
+      ignore_unreachable: true
+      when: ansible_facts.os_family == "Debian"

etc/kayobe/apt.yml

@@ -11,6 +11,58 @@
 # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}.
 #apt_proxy_https:
 
+# List of Apt configuration options. Each item is a dict with the following
+# keys:
+# * content: free-form configuration file content
+# * filename: name of a file in /etc/apt/apt.conf.d/ in which to write the
+#   configuration
+# Default is an empty list.
+apt_config:
+  # NOTE: Currently the Pulp verbatim publisher does not sync translation
+  # files, which results in apt update failing. Disable translations until this
+  # is resolved.
+  - content: |
+      Acquire::Languages "none";
+    filename: 99no-languages
+
+# List of apt keys. Each item is a dict containing the following keys:
+# * url: URL of key
+# * filename: Name of a file in which to store the downloaded key
+# Default is an empty list.
+apt_keys:
+  - url: "https://download.docker.com/linux/ubuntu/gpg"
+    filename: docker.asc
+
+# A list of Apt repositories.
+# * types: whitespace-separated list of repository types, e.g. deb or deb-src
+#   (optional, default is 'deb')
+# * url: URL of the repository
+# * suites: whitespace-separated list of suites, e.g. focal (optional, default
+#   is ansible_facts.distribution_release)
+# * components: whitespace-separated list of components, e.g. main (optional,
+#   default is 'main')
+# * signed_by: whitespace-separated list of names of GPG keyring files in
+#   apt_keys_path (optional, default is unset)
+# * architecture: whitespace-separated list of architectures that will be used
+#   (optional, default is unset)
+# Default is an empty list.
+apt_repositories:
+  - url: "{{ stackhpc_repo_ubuntu_focal_url }}"
+    suites: "{{ ansible_facts.distribution_release }} {{ ansible_facts.distribution_release }}-updates {{ ansible_facts.distribution_release }}-backports"
+    components: main restricted universe multiverse
+  - url: "{{ stackhpc_repo_ubuntu_focal_security_url }}"
+    suites: "{{ ansible_facts.distribution_release }}-security"
+    components: main restricted universe multiverse
+  - url: "{{ stackhpc_repo_docker_ce_ubuntu_url }}"
+    suites: "{{ ansible_facts.distribution_release }}"
+    components: stable
+    signed_by: docker.asc
+
+# Whether to disable repositories in /etc/apt/sources.list. This may be used
+# when replacing the distribution repositories via apt_repositories.
+# Default is false.
+apt_disable_sources_list: true
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

etc/kayobe/environments/ci-aio/stackhpc-ci.yml

@@ -37,6 +37,11 @@ stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
 stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}"
 stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}"
 stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}"
+stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
+stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
+stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
+stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
+
 
 # Host and port of container registry.
 # Push built images to the development Pulp service registry.

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -56,6 +56,10 @@ stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
 stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}"
 stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}"
 stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}"
+stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
+stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
+stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
+stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
 
 # Host and port of container registry.
 # Push built images to the Ark registry.

etc/kayobe/kolla.yml

@@ -183,17 +183,26 @@ stackhpc_third_party_repos:
   - url: "{{ stackhpc_repo_treasuredata_4_url }}"
     file: "td.repo"
 
+# List of repositories for Ubuntu focal.
+stackhpc_ubuntu_focal_repos:
+  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal main universe"
+  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe"
+  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe"
+  - "deb {{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe"
+  - "deb {{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/wallaby main"
+
 # Dict mapping Jinja2 block names in kolla's Docker images to their contents.
 kolla_build_blocks:
   base_header: |
+    {% if kolla_base_distro == 'centos' %}
     RUN \
     {% for repo in stackhpc_centos_stream_repos %}
         sed -i -e 's/^\(mirrorlist *=.*\)/#\1/g' \
                -e 's/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/#baseurl.*/a baseurl={{ repo.url }}' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \
     {% endif %}
     {% endfor %}
-
+    {% endif %}
   base_centos_repo_overrides_post_yum: |
     {# fixme #}
         && \
@@ -202,7 +211,13 @@ kolla_build_blocks:
                -e 's/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/#baseurl.*/a baseurl={{ repo.url }}' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
-
+  base_ubuntu_package_sources_list: |
+    RUN \
+        rm -f /etc/apt/sources.list && \
+    {% for repo in stackhpc_ubuntu_focal_repos %}
+        echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
+    {% endif %}
+    {% endfor %}
   grafana_plugins_install: |
     RUN grafana-cli plugins install vonage-status-panel
   ironic_inspector_header: |
@@ -225,15 +240,11 @@ kolla_build_blocks:
   prometheus_blackbox_exporter_repository_version: |
     ARG blackbox_exporter_version='0.19.0'
 
-# Dict mapping image customization variable names to their values.
-# Each variable takes the form:
-# <image name>_<customization>_<operation>
-# Hyphens in the image name must be replaced with underscores. The
-# customization is most commonly packages. The operation should be one of
-# override, append or remove. The value should be a list.
-kolla_build_customizations:
+kolla_build_customizations_common:
   ironic_inspector_pip_packages_append:
     - /additions/*
+
+kolla_build_customizations_centos:
   ovn_base_packages_override:
     - ovn-2021-21.12.0
   ovn_controller_packages_override:
@@ -252,6 +263,16 @@ kolla_build_customizations:
     - python3-openvswitch2.16
     - tcpdump
 
+kolla_build_customizations_ubuntu: {}
+
+# Dict mapping image customization variable names to their values.
+# Each variable takes the form:
+# <image name>_<customization>_<operation>
+# Hyphens in the image name must be replaced with underscores. The
+# customization is most commonly packages. The operation should be one of
+# override, append or remove. The value should be a list.
+kolla_build_customizations: "{{ kolla_build_customizations_common | combine(kolla_build_customizations_centos if kolla_base_distro == 'centos' else kolla_build_customizations_ubuntu) }}"
+
 ###############################################################################
 # Kolla-ansible inventory configuration.
 

etc/kayobe/kolla/globals.yml

@@ -1,7 +1,12 @@
+# yamllint disable-file
 ---
+enable_docker_repo: "{% raw %}{{ ansible_facts.os_family == 'RedHat' }}{% endraw %}"
+
 docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}"
 docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg"
 
+{# We are not currently providing container images for Ubuntu in Wallaby. #}
+{% if kolla_base_distro == 'centos' %}
 barbican_tag: wallaby-20220311T133847
 cinder_tag: wallaby-20220309T083940
 cloudkitty_tag: wallaby-20220119T122428
@@ -22,3 +27,4 @@ prometheus_jiralert_tag: wallaby-20220119T122428
 prometheus_libvirt_exporter_tag: wallaby-20220325T122042
 prometheus_msteams_tag: wallaby-20220119T122428
 prometheus_openstack_exporter_tag: wallaby-20220705T132206
+{% endif %}