Merge stackhpc/wallaby into stackhpc/xena

markgoddard · markgoddard · commit 4388f8db3b6a · 2022-07-22T11:29:31.000+01:00
diff --git a/README.rst b/README.rst
@@ -230,7 +230,7 @@ ci-aio
 Prerequisites
 ^^^^^^^^^^^^^
 
-* a CentOS Stream 8 host
+* a CentOS Stream 8 or Ubuntu Focal 20.04 host
 * access to the local Pulp server
 
 Setup
@@ -329,7 +329,7 @@ service, and pushed there once built.
 Prerequisites
 ^^^^^^^^^^^^^
 
-* a CentOS Stream 8 host
+* a CentOS Stream 8 or Ubuntu Focal 20.04 host
 * access to the local Pulp server
 
 Setup
diff --git a/etc/kayobe/ansible/pulp-repo-promote-production.yml b/etc/kayobe/ansible/pulp-repo-promote-production.yml
@@ -6,4 +6,5 @@
     - import_role:
         name: stackhpc.pulp.pulp_distribution
       vars:
-        pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_production }}"
+        pulp_distribution_deb: "{{ stackhpc_pulp_distribution_deb_production | selectattr('required') }}"
+        pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_production | selectattr('required') }}"
diff --git a/etc/kayobe/ansible/pulp-repo-publish.yml b/etc/kayobe/ansible/pulp-repo-publish.yml
@@ -7,9 +7,11 @@
         name: stackhpc.pulp.pulp_publication
       # NOTE: use intermediate variable to avoid publishing containers.
       vars:
-        pulp_publication_rpm: "{{ stackhpc_pulp_publication_rpm_development }}"
+        pulp_publication_deb: "{{ stackhpc_pulp_publication_deb_development | selectattr('required') }}"
+        pulp_publication_rpm: "{{ stackhpc_pulp_publication_rpm_development | selectattr('required') }}"
 
     - import_role:
         name: stackhpc.pulp.pulp_distribution
       vars:
-        pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_development }}"
+        pulp_distribution_deb: "{{ stackhpc_pulp_distribution_deb_development | selectattr('required') }}"
+        pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_development | selectattr('required') }}"
diff --git a/etc/kayobe/ansible/pulp-repo-sync.yml b/etc/kayobe/ansible/pulp-repo-sync.yml
@@ -7,4 +7,5 @@
         name: stackhpc.pulp.pulp_repository
       # NOTE: use intermediate variable to avoid syncing containers.
       vars:
-        pulp_repository_rpm_repos: "{{ stackhpc_pulp_repository_rpm_repos }}"
+        pulp_repository_deb_repos: "{{ stackhpc_pulp_repository_deb_repos | selectattr('required') }}"
+        pulp_repository_rpm_repos: "{{ stackhpc_pulp_repository_rpm_repos | selectattr('required') }}"
diff --git a/etc/kayobe/ansible/purge-command-not-found.yml b/etc/kayobe/ansible/purge-command-not-found.yml
@@ -0,0 +1,33 @@
+---
+# Currently the pulp_deb plugin in Pulp does not support certain types of
+# content, including i18n files and command-not-found indices. This breaks APT
+# when the command-not-found is installed. This playbook can be used to
+# uninstall the package, prior to running any other APT commands. It may be
+# installed as a hook to the 'host configure' commands.
+# See https://github.com/pulp/pulp_deb/issues/419
+# FIXME: If used as a hook, this playbook matches all hosts, so will run
+# against the seed, even when running 'overcloud host configure'. Depending on
+# the stage of deployment, some hosts may be unreachable. This could be fixed
+# by implementing this playbook separately for each group.
+- name: Purge command-not-found package
+  hosts: seed-hypervisor:seed:overcloud:infra-vms
+  gather_facts: false
+  vars:
+    ansible_user: "{{ bootstrap_user }}"
+    # We can't assume that a virtualenv exists at this point, so use the system
+    # python interpreter.
+    ansible_python_interpreter: /usr/bin/python3
+    # Work around no known_hosts entry on first boot.
+    ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+  tasks:
+    - name: Purge command-not-found package
+      package:
+        name:
+          - command-not-found
+          - python3-command-not-found
+        purge: true
+        state: absent
+      become: true
+      failed_when: false
+      ignore_unreachable: true
+      when: ansible_facts.os_family == "Debian"
diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml
@@ -17,16 +17,23 @@
 # * filename: name of a file in /etc/apt/apt.conf.d/ in which to write the
 #   configuration
 # Default is an empty list.
-#apt_config:
+apt_config:
+  # NOTE: Currently the Pulp verbatim publisher does not sync translation
+  # files, which results in apt update failing. Disable translations until this
+  # is resolved.
+  - content: |
+      Acquire::Languages "none";
+    filename: 99no-languages
 
 # List of apt keys. Each item is a dict containing the following keys:
 # * url: URL of key
-# * filename: Name of a file in which to store the downloaded key. The
-#   extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise.
+# * filename: Name of a file in which to store the downloaded key
 # Default is an empty list.
-#apt_keys:
+apt_keys:
+  - url: "https://download.docker.com/linux/ubuntu/gpg"
+    filename: docker.asc
 
-# A list of Apt repositories. Each item is a dict with the following keys:
+# A list of Apt repositories.
 # * types: whitespace-separated list of repository types, e.g. deb or deb-src
 #   (optional, default is 'deb')
 # * url: URL of the repository
@@ -39,12 +46,22 @@
 # * architecture: whitespace-separated list of architectures that will be used
 #   (optional, default is unset)
 # Default is an empty list.
-#apt_repositories:
+apt_repositories:
+  - url: "{{ stackhpc_repo_ubuntu_focal_url }}"
+    suites: "{{ ansible_facts.distribution_release }} {{ ansible_facts.distribution_release }}-updates {{ ansible_facts.distribution_release }}-backports"
+    components: main restricted universe multiverse
+  - url: "{{ stackhpc_repo_ubuntu_focal_security_url }}"
+    suites: "{{ ansible_facts.distribution_release }}-security"
+    components: main restricted universe multiverse
+  - url: "{{ stackhpc_repo_docker_ce_ubuntu_url }}"
+    suites: "{{ ansible_facts.distribution_release }}"
+    components: stable
+    signed_by: docker.asc
 
 # Whether to disable repositories in /etc/apt/sources.list. This may be used
 # when replacing the distribution repositories via apt_repositories.
 # Default is false.
-#apt_disable_sources_list:
+apt_disable_sources_list: true
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
diff --git a/etc/kayobe/containers/squid_proxy/pre.yml b/etc/kayobe/containers/squid_proxy/pre.yml
@@ -0,0 +1,14 @@
+---
+- name: Ensure /srv/docker/squid directory exists
+  file:
+    path: /srv/docker/squid
+    state: directory
+    mode: 0755
+  become: true
+
+- name: Copy modified squid.conf
+  copy:
+    src: "{{ kayobe_config_path }}/containers/squid_proxy/squid.conf"
+    dest: /srv/docker/squid/
+    mode: 0644
+  become: true
diff --git a/etc/kayobe/containers/squid_proxy/squid.conf b/etc/kayobe/containers/squid_proxy/squid.conf
@@ -0,0 +1,75 @@
+#
+# Recommended minimum configuration:
+#
+
+# Example rule allowing access from your local networks.
+# Adapt to list your (internal) IP networks from where browsing
+# should be allowed
+acl localnet src 10.0.0.0/8	# RFC1918 possible internal network
+acl localnet src 172.16.0.0/12	# RFC1918 possible internal network
+acl localnet src 192.168.0.0/16	# RFC1918 possible internal network
+acl localnet src fc00::/7       # RFC 4193 local private network range
+acl localnet src fe80::/10      # RFC 4291 link-local (directly plugged) machines
+
+acl SSL_ports port 443
+acl Safe_ports port 80		# http
+acl Safe_ports port 21		# ftp
+acl Safe_ports port 443		# https
+acl Safe_ports port 70		# gopher
+acl Safe_ports port 210		# wais
+acl Safe_ports port 1025-65535	# unregistered ports
+acl Safe_ports port 280		# http-mgmt
+acl Safe_ports port 488		# gss-http
+acl Safe_ports port 591		# filemaker
+acl Safe_ports port 777		# multiling http
+acl CONNECT method CONNECT
+
+#
+# Recommended minimum Access Permission configuration:
+#
+# Deny requests to certain unsafe ports
+http_access deny !Safe_ports
+
+# Deny CONNECT to other than secure SSL ports
+http_access deny CONNECT !SSL_ports
+
+# Only allow cachemgr access from localhost
+http_access allow localhost manager
+http_access deny manager
+
+# We strongly recommend the following be uncommented to protect innocent
+# web applications running on the proxy server who think the only
+# one who can access services on "localhost" is a local user
+#http_access deny to_localhost
+
+#
+# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS
+#
+
+# Example rule allowing access from your local networks.
+# Adapt localnet in the ACL section to list your (internal) IP networks
+# from where browsing should be allowed
+http_access allow localnet
+http_access allow localhost
+
+# And finally deny all other access to this proxy
+http_access deny all
+
+# Squid normally listens to port 3128
+http_port 3128
+
+# Uncomment and adjust the following to add a disk cache directory.
+cache_dir ufs /var/spool/squid 4096 16 256
+cache_mem 768 MB
+maximum_object_size_in_memory 64 MB
+
+# Leave coredumps in the first cache dir
+coredump_dir /var/spool/squid
+
+#
+# Add any of your own refresh_pattern entries above these.
+#
+refresh_pattern ^ftp:		1440	20%	10080
+refresh_pattern ^gopher:	1440	0%	1440
+refresh_pattern -i (/cgi-bin/|\?) 0	0%	0
+refresh_pattern .		0	20%	4320
diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
@@ -44,6 +44,11 @@ stackhpc_repo_mariadb_10_5_centos8_version: "{{ stackhpc_pulp_repo_mariadb_10_5_
 stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}"
 stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}"
 stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}"
+stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
+stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
+stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
+stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
+
 
 # Host and port of container registry.
 # Push built images to the development Pulp service registry.
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -63,6 +63,10 @@ stackhpc_repo_mariadb_10_5_centos8_version: "{{ stackhpc_pulp_repo_mariadb_10_5_
 stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}"
 stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}"
 stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}"
+stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
+stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
+stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
+stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
 
 # Host and port of container registry.
 # Push built images to the Ark registry.
diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
@@ -192,17 +192,26 @@ stackhpc_third_party_repos:
   - url: "{{ stackhpc_repo_treasuredata_4_url }}"
     file: "td.repo"
 
+# List of repositories for Ubuntu focal.
+stackhpc_ubuntu_focal_repos:
+  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal main universe"
+  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe"
+  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe"
+  - "deb {{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe"
+  - "deb {{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/wallaby main"
+
 # Dict mapping Jinja2 block names in kolla's Docker images to their contents.
 kolla_build_blocks:
   base_header: |
+    {% if kolla_base_distro == 'centos' %}
     RUN \
     {% for repo in stackhpc_centos_stream_repos %}
         sed -i -e 's/^\(mirrorlist *=.*\)/#\1/g' \
                -e 's/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/#baseurl.*/a baseurl={{ repo.url }}' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \
     {% endif %}
     {% endfor %}
-
+    {% endif %}
   base_centos_repo_overrides_post_yum: |
     {# fixme #}
         && \
@@ -211,7 +220,13 @@ kolla_build_blocks:
                -e 's/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/#baseurl.*/a baseurl={{ repo.url }}' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
-
+  base_ubuntu_package_sources_list: |
+    RUN \
+        rm -f /etc/apt/sources.list && \
+    {% for repo in stackhpc_ubuntu_focal_repos %}
+        echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
+    {% endif %}
+    {% endfor %}
   grafana_plugins_install: |
     RUN grafana-cli plugins install vonage-status-panel
   ironic_inspector_header: |
@@ -240,11 +255,13 @@ kolla_build_blocks:
 # Hyphens in the image name must be replaced with underscores. The
 # customization is most commonly packages. The operation should be one of
 # override, append or remove. The value should be a list.
-kolla_build_customizations:
-  base_yum_repo_files_remove:
-    - proxysql.repo
+kolla_build_customizations_common:
   ironic_inspector_pip_packages_append:
     - /additions/*
+
+kolla_build_customizations_centos:
+  base_yum_repo_files_remove:
+    - proxysql.repo
   ovn_base_packages_override:
     - ovn-2021-21.12.0
   ovn_controller_packages_override:
@@ -263,6 +280,16 @@ kolla_build_customizations:
     - python3-openvswitch2.16
     - tcpdump
 
+kolla_build_customizations_ubuntu: {}
+
+# Dict mapping image customization variable names to their values.
+# Each variable takes the form:
+# <image name>_<customization>_<operation>
+# Hyphens in the image name must be replaced with underscores. The
+# customization is most commonly packages. The operation should be one of
+# override, append or remove. The value should be a list.
+kolla_build_customizations: "{{ kolla_build_customizations_common | combine(kolla_build_customizations_centos if kolla_base_distro == 'centos' else kolla_build_customizations_ubuntu) }}"
+
 ###############################################################################
 # Kolla-ansible inventory configuration.
 
diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
@@ -1,3 +1,6 @@
+# yamllint disable-file
 ---
+enable_docker_repo: "{% raw %}{{ ansible_facts.os_family == 'RedHat' }}{% endraw %}"
+
 docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}"
 docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg"
diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml
@@ -10,6 +10,14 @@
 # is False. This will change in a future release.
 #overcloud_dib_build_host_images:
 
+# List of overcloud host disk images to build. Each element is a dict defining
+# an image in a format accepted by the stackhpc.os-images role. Default is to
+# build an image named "deployment_image" configured with the overcloud_dib_*
+# variables defined below: {"name": "deployment_image", "elements": "{{
+# overcloud_dib_elements }}", "env": "{{ overcloud_dib_env_vars }}",
+# "packages": "{{ overcloud_dib_packages }}"}.
+#overcloud_dib_host_images:
+
 # DIB base OS element. Default is {{ os_distribution }}.
 #overcloud_dib_os_element:
 
@@ -45,6 +53,19 @@
 # List of DIB packages to install. Default is to install no extra packages.
 #overcloud_dib_packages:
 
+# List of default git repositories containing Diskimage Builder (DIB) elements.
+# See stackhpc.os-images role for usage. Default is empty.
+#overcloud_dib_git_elements_default:
+
+# List of additional git repositories containing Diskimage Builder (DIB)
+# elements. See stackhpc.os-images role for usage. Default is empty.
+#overcloud_dib_git_elements_extra:
+
+# List of git repositories containing Diskimage Builder (DIB) elements. See
+# stackhpc.os-images role for usage. Default is a combination of
+# overcloud_dib_git_elements_default and overcloud_dib_git_elements_extra.
+#overcloud_dib_git_elements:
+
 # Upper constraints file for installing packages in the virtual environment
 # used for building overcloud host disk images. Default is {{
 # pip_upper_constraints_file }}.
diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml
@@ -10,6 +10,7 @@ stackhpc_pulp_repo_centos_stream_8_openstack_xena_version: 20220612T035123
 stackhpc_pulp_repo_centos_stream_8_opstools_version: 20220617T100837
 stackhpc_pulp_repo_centos_stream_8_powertools_version: 20220607T061247
 stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version: 20220525T030654
+stackhpc_pulp_repo_docker_ce_ubuntu_version: 20220708T132615
 stackhpc_pulp_repo_docker_version: 20220607T061247
 stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version: 20220525T030654
 stackhpc_pulp_repo_epel_modular_version: 20220604T032730
@@ -19,3 +20,6 @@ stackhpc_pulp_repo_mariadb_10_5_centos8_version: 20220609T110556
 stackhpc_pulp_repo_rabbitmq_erlang_version: 20220616T113902
 stackhpc_pulp_repo_rabbitmq_server_version: 20220602T033149
 stackhpc_pulp_repo_treasuredata_4_version: 20220429T160649
+stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20220712T155732
+stackhpc_pulp_repo_ubuntu_focal_security_version: 20220708T132615
+stackhpc_pulp_repo_ubuntu_focal_version: 20220708T132615
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
diff --git a/etc/kayobe/seed-vm.yml b/etc/kayobe/seed-vm.yml
diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
