add cross-arch aarch64 container image builds

Author: bbezak

.github/workflows/stackhpc-container-image-build.yml

@@ -92,8 +92,8 @@ jobs:
       - name: Generate build matrix
         id: set-matrix
         run: |
-          comma=""
           echo -n "matrix={\"distro\": [" >> $GITHUB_OUTPUT
+          comma=""
           if [[ ${{ inputs.rocky-linux-9 }} == 'true' ]]; then
             echo -n "$comma\"rocky\"" >> $GITHUB_OUTPUT
             comma=", "
@@ -102,7 +102,7 @@ jobs:
             echo -n "$comma\"ubuntu\"" >> $GITHUB_OUTPUT
             comma=", "
           fi
-          echo "]}" >> $GITHUB_OUTPUT
+          echo "], \"arch\": [\"amd64\", \"aarch64\"]}" >> $GITHUB_OUTPUT
 
       - name: Display container datetime tag
         run: |
@@ -122,6 +122,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }}
+      # Exclude ubuntu aarch64 builds for now
+      exclude:
+        - distro: ubuntu
+          arch: aarch64
     needs:
       - generate-tag
       - runner-selection
@@ -173,7 +177,12 @@ jobs:
 
       - name: Get Kolla tag
         id: write-kolla-tag
-        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}"  >> $GITHUB_OUTPUT
+        run: |
+          kolla_tag="${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ needs.generate-tag.outputs.datetime_tag }}"
+          if [[ "${{ matrix.distro }}" == 'rocky' ]]; then
+            kolla_tag="$kolla_tag-${{ matrix.arch }}"
+          fi
+          echo "kolla-tag=$kolla_tag" >> $GITHUB_OUTPUT
 
       - name: Configure localhost as a seed
         run: |
@@ -203,8 +212,11 @@ jobs:
         continue-on-error: true
         run: |
           args="${{ inputs.regexes }}"
+          if [[ "${{ matrix.arch }}" = 'aarch64' ]]; then
+            args="$args -e kolla_base_arch=${{ matrix.arch }}"
+          fi
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
+          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
@@ -217,26 +229,37 @@ jobs:
         run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
         if: inputs.overcloud
 
+      - name: Copy build configs to output directory
+        run: sudo cp -rnL /opt/kayobe/etc/kolla/* image-build-logs/
+        if: inputs.overcloud
+
+      - name: Get Kolla tag for seed images
+        id: write-kolla-tag-seed
+        run: |
+          kolla_tag_seed="${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ needs.generate-tag.outputs.datetime_tag }}"
+          echo "kolla-tag-seed=$kolla_tag_seed" >> $GITHUB_OUTPUT
+        if: inputs.seed
+
       - name: Build kolla seed images
         id: build_seed_images
         continue-on-error: true
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
+          args="$args -e kolla_tag=${{ steps.write-kolla-tag-seed.outputs.kolla-tag-seed }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe seed container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.seed
+        if: inputs.seed && matrix.arch == 'amd64'
 
       - name: Copy seed container image build logs to output directory
         run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
         if: inputs.seed
 
       - name: Get built container images
-        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}" > ${{ matrix.distro }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
@@ -297,7 +320,7 @@ jobs:
       - name: Upload output artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.distro }}-logs
+          name: ${{ matrix.distro }}-${{ matrix.arch }}-logs
           path: image-build-logs
           retention-days: 7
         if: ${{ !cancelled() }}

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -49,33 +49,34 @@ stackhpc_repo_mirror_password: !vault |
 
 # Build against released Pulp repository versions.
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
-stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version }}"
+stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version }}"
 stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}"
 stackhpc_repo_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_version }}"
 stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_security_version }}"
 stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: ""
 stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
 stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}"
 stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}"
-stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}"
-stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}"
-stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_9_opstools_version }}"
-stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version }}"
-stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_centos_stream_9_docker_version }}"
-stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_rhel_9_treasuredata_5_version }}"
-stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_rhel_9_mariadb_10_11_version }}"
-stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_rhel_9_influxdb_version }}"
-stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_epel_9_version }}"
+stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version }}"
+stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_caracal_version }}"
+stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version }}"
+stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_reef_version }}"
+stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version }}"
+stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}"
+stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}"
+stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}"
+stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_multiarch_epel_9_version }}"
 stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}"
 stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}"
+
 ## Use derived vars from etc/kayobe/pulp.yml to switch between
 ## minor Rocky versions using stackhpc_pulp_repo_rocky_x_minor_version
 stackhpc_repo_rocky_9_baseos_version: "{{ stackhpc_pulp_repo_rocky_9_baseos_version }}"
 stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_pulp_repo_rocky_9_appstream_version }}"
 stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}"
 stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
 stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
-stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
+stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_multiarch_rocky_9_sig_security_common_version }}"
 stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}"
 
 # Rocky-and-CI-specific Pulp urls

etc/kayobe/kolla.yml

@@ -214,8 +214,8 @@ stackhpc_rocky_9_third_party_repos:
     file: "CentOS-Ceph-Reef.repo"
     tag: "centos-ceph-reef"
   - url: "{{ stackhpc_repo_rhel9_rabbitmq_erlang_url }}"
-    file: "rabbitmq_rabbitmq-erlang.repo"
-    tag: "rabbitmq_rabbitmq-erlang"
+    file: "{{ 'copr-rabbitmq-erlang.repo' if kolla_base_arch == 'aarch64' else 'rabbitmq_rabbitmq-erlang.repo' }}"
+    tag: "{{ 'openstack-kolla-copr-rabbitmq-kolla' if kolla_base_arch == 'aarch64' else 'rabbitmq_rabbitmq-erlang' }}"
   - url: "{{ stackhpc_repo_rhel9_rabbitmq_server_url }}"
     file: "rabbitmq_rabbitmq-server.repo"
     tag: "rabbitmq_rabbitmq-server"
@@ -266,13 +266,18 @@ kolla_build_blocks:
         mkdir -p /etc/yum.repos.d.backup && \
         tar -czf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d . && \
     {% endif %}
+        echo "===== Before sed modifications I =====" && \
+        cat /etc/yum.repos.d/*.repo && \
     {% for repo in stackhpc_yum_repos %}
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \
     {% endif %}
     {% endfor %}
+        && \
+        echo "===== After sed modifications I =====" && \
+        cat /etc/yum.repos.d/*.repo
     {% else %}
     RUN \
         rm /etc/apt/sources.list && \
@@ -293,13 +298,18 @@ kolla_build_blocks:
         tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
         tar -czf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d . && \
     {% endif %}
+        echo "===== Before sed modifications II =====" && \
+        cat /etc/yum.repos.d/*.repo && \
     {% for repo in base_centos_repo_overrides_post_yum_list %}
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[ \t]*\(https.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
+        && \
+        echo "===== After sed modifications II =====" && \
+        cat /etc/yum.repos.d/*.repo \
     {% endif %}
   # With the UCA keyring installed we can now add all repos.
   base_ubuntu_package_sources_list: |

etc/kayobe/pulp-repo-versions.yml

@@ -2,24 +2,35 @@
 # This file is autogenerated by Ansible using the following workflow:
 # https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-update-kayobe.yml
 stackhpc_pulp_repo_centos_stream_9_docker_version: 20241210T000909
+stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20241210T012225
 stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version: 20250217T103110
+stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version: 20250218T032120
 stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version: 20241212T022636
+stackhpc_pulp_repo_centos_stream_9_openstack_caracal_aarch64_version: 20241212T020643
 stackhpc_pulp_repo_centos_stream_9_opstools_version: 20231213T031318
+stackhpc_pulp_repo_centos_stream_9_opstools_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version: 20240923T233036
+stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_ceph_reef_debian_version: 20240925T152022
 stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version: 20241218T154614
 stackhpc_pulp_repo_elrepo_9_version: 20241129T235743
+stackhpc_pulp_repo_elrepo_9_aarch64_version: 20241001T011548
 stackhpc_pulp_repo_epel_9_version: 20241216T235733
+stackhpc_pulp_repo_epel_9_aarch64_version: 20241217T012754
 stackhpc_pulp_repo_grafana_version: 20241216T002739
 stackhpc_pulp_repo_opensearch_2_x_version: 20241106T010702
 stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20241106T010702
 stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20241217T002152
+stackhpc_pulp_repo_rhel9_rabbitmq_erlang_aarch64_version: 20241213T015928
 stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20241217T002152
 stackhpc_pulp_repo_rhel_9_influxdb_version: 20241217T002152
+stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version: 20241217T012754
 stackhpc_pulp_repo_rhel_9_mariadb_10_11_version: 20241102T004913
+stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version: 20241102T015940
 stackhpc_pulp_repo_rhel_9_rabbitmq_erlang_version: 20240711T091318
 stackhpc_pulp_repo_rhel_9_rabbitmq_server_version: 20240711T091318
 stackhpc_pulp_repo_rhel_9_treasuredata_5_version: 20241115T002028
+stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version: 20241115T010217
 stackhpc_pulp_repo_rocky_9_1_appstream_version: 20231207T013715
 stackhpc_pulp_repo_rocky_9_1_baseos_version: 20231206T014015
 stackhpc_pulp_repo_rocky_9_1_crb_version: 20231211T120328
@@ -38,14 +49,20 @@ stackhpc_pulp_repo_rocky_9_3_highavailability_version: 20240510T001129
 stackhpc_pulp_repo_rocky_9_4_appstream_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_4_baseos_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_4_crb_version: 20240816T002610
+stackhpc_pulp_repo_rocky_9_5_crb_aarch64_version: 20241217T012754
 stackhpc_pulp_repo_rocky_9_4_extras_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_4_highavailability_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_5_appstream_version: 20241217T005008
+stackhpc_pulp_repo_rocky_9_5_appstream_aarch64_version: 20241217T012754
 stackhpc_pulp_repo_rocky_9_5_baseos_version: 20241216T013503
+stackhpc_pulp_repo_rocky_9_5_baseos_aarch64_version: 20241216T011954
 stackhpc_pulp_repo_rocky_9_5_crb_version: 20241217T005008
 stackhpc_pulp_repo_rocky_9_5_extras_version: 20241216T004230
+stackhpc_pulp_repo_rocky_9_5_extras_aarch64_version: 20241216T011954
 stackhpc_pulp_repo_rocky_9_5_highavailability_version: 20241202T003154
+stackhpc_pulp_repo_rocky_9_5_highavailability_aarch64_version: 20241203T010516
 stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20241127T003858
+stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version: 20241127T011228
 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20241217T045049
 stackhpc_pulp_repo_ubuntu_jammy_security_version: 20241217T071258
 stackhpc_pulp_repo_ubuntu_jammy_version: 20241217T071258

etc/kayobe/pulp.yml

@@ -216,11 +216,11 @@ stackhpc_pulp_sync_rocky_9: "{{ os_distribution == 'rocky' }}"
 stackhpc_pulp_repo_rocky_9_minor_version: 5
 # Rocky 9 Snapshot versions. The defaults use the appropriate version from
 # pulp-repo-versions.yml for the selected minor release.
-stackhpc_pulp_repo_rocky_9_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_appstream_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_baseos_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_extras_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_crb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_crb_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_highavailability_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_highavailability_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
+stackhpc_pulp_repo_rocky_9_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_baseos' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_appstream' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_extras' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_crb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_crb' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_highavailability_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_highavailability' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
 
 # Whether to sync packages common to all RHEL 9 derivatives.
 stackhpc_pulp_sync_el_9: "{{ stackhpc_pulp_sync_rocky_9 | bool }}"