Run the compiler validation suite for Android

.github/workflows/pull_request.yml

@@ -23,27 +23,38 @@ jobs:
             *.log
 
    android-build:
-     name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK
+     name: Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} ${{ matrix.runner }} (compiler=${{ matrix.build-compiler }})
      strategy:
        fail-fast: false
        matrix:
-         #build-type: ['docker']
-         build-type: ['docker', 'local']
-         # blank arch builds all (aarch64,x86_64,armv7)
-         #arch: ['']
-         # builds only x86_64 to speed up the validation
-         #arch: ['x86_64']
-         # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs
-         arch: ['x86_64', '']
-         swift-version: ['release', 'swift-6.2-branch', 'development']
-     runs-on: ubuntu-24.04
+         include:
+           - swift-version: 'swift-6.2-branch'
+             build-type: 'docker'
+             build-compiler: '1'
+             runner: 'self-hosted'
+           - swift-version: 'development'
+             build-type: 'docker'
+             build-compiler: '1'
+             runner: 'self-hosted'
+           - swift-version: 'swift-6.2-branch'
+             build-type: 'docker'
+             build-compiler: '0'
+             runner: 'ubuntu-24.04'
+           - swift-version: 'development'
+             build-type: 'docker'
+             build-compiler: '0'
+             runner: 'ubuntu-24.04'
+     runs-on: ${{ matrix.runner }}
+     # 15 hour timeout
+     timeout-minutes: 900
      steps:
       - name: Free Disk Space
+        if: ${{ matrix.runner != 'self-hosted' }}
         run: |
           df -h
           # brings available space from 25G to 32G
           # otherwise we sometimes run out of space during the build
-          sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache /usr/local/graalvm/ /usr/local/.ghcup/ /usr/local/lib/node_modules /usr/local/share/boost
           sudo docker image prune --all --force
           sudo docker builder prune -a
           df -h
@@ -53,6 +64,8 @@ jobs:
           # these variabes are used by build-docker and build-local
           # to determine which Swift version to build for
           echo "BUILD_SCHEME=${{ matrix.swift-version }}" >> $GITHUB_ENV
+          # pass the build-compiler matrix through to the build script
+          echo "BUILD_COMPILER=${{ matrix.build-compiler }}" >> $GITHUB_ENV
           echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV
           echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV
       - name: Checkout repository
@@ -61,7 +74,7 @@ jobs:
         if: ${{ matrix.build-type == 'local' }}
         working-directory: swift-ci/sdks/android
         run: |
-          sudo apt install -q ninja-build patchelf
+          sudo apt install -q patchelf build-essential cmake ninja-build python3 golang git gnupg2 libcurl4-openssl-dev libedit-dev libicu-dev libncurses5-dev libpython3-dev libsqlite3-dev libxml2-dev rsync uuid-dev uuid-runtime tzdata curl unzip
           ./build-local ${BUILD_SCHEME} ${WORKDIR}
       - name: Build Android SDK (Docker)
         if: ${{ matrix.build-type == 'docker' }}
@@ -99,6 +112,9 @@ jobs:
           if [[ "${{ matrix.build-type }}" == 'local' ]]; then
             ARTIFACT_NAME="${ARTIFACT_NAME}-local"
           fi
+          if [[ "${{ matrix.build-compiler }}" == '1' ]]; then
+            ARTIFACT_NAME="${ARTIFACT_NAME}-hostbuild"
+          fi
           # artifacts need a unique name so we suffix with the matrix arch(s)
           if [[ ! -z "${{ matrix.arch }}" ]]; then
             ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')"
@@ -117,6 +133,7 @@ jobs:
           name: ${{ steps.info.outputs.artifact-name }}
           path: ${{ steps.info.outputs.artifact-path }}
       - name: Cleanup
+        if: ${{ matrix.runner != 'self-hosted' }}
         run: |
           # need to free up some space or else when installing we get: No space left on device
           df -h
@@ -125,6 +142,7 @@ jobs:
           sudo docker builder prune -a
           df -h
       - name: Install artifactbundle
+        if: ${{ matrix.runner != 'self-hosted' }}
         shell: bash
         run: |
           set -ex
@@ -135,6 +153,7 @@ jobs:
           echo "ANDROID_NDK_ROOT=" >> $GITHUB_ENV
 
       - name: Create Demo Project
+        if: ${{ matrix.runner != 'self-hosted' }}
         run: |
           cd ${{ runner.temp }}
           mkdir DemoProject
@@ -149,6 +168,7 @@ jobs:
           echo 'import Android' >> Sources/DemoProject/DemoProject.swift
       - name: Test Demo Project on Android
         uses: skiptools/swift-android-action@main
+        if: false # disabled because we aren't building Testing
         with:
           # only test for the complete arch SDK build to speed up CI
           #run-tests: ${{ matrix.arch == '' }}
@@ -164,7 +184,6 @@ jobs:
       - name: Test swift-algorithms
         uses: skiptools/swift-android-action@main
         with:
-          run-tests: ${{ matrix.arch == '' }}
           package-path: swift-algorithms
           installed-sdk: ${{ steps.info.outputs.sdk-id }}
           installed-swift: ${{ steps.info.outputs.swift-root }}

README.md

@@ -2,9 +2,9 @@
 
 <img src="https://swift.org/assets/images/swift.svg" alt="Swift logo" height="70" >
 
-### Docker images for [Swift](https://swift.org).
+###  images for [Swift](https://swift.org).
 
-#### You can find the Docker Hub repo here: [https://hub.docker.com/_/swift/](https://hub.docker.com/_/swift/)
+#### You can find the  Hub repo here: [https://hub.docker.com/_/swift/](https://hub.docker.com/_/swift/)
 
 #### Nightly image tags are published here: [https://hub.docker.com/r/swiftlang/swift](https://hub.docker.com/r/swiftlang/swift)
 

swift-ci/sdks/android/Dockerfile

@@ -18,6 +18,7 @@ ARG SWIFT_TOOLCHAIN_URL=
 RUN apt-get -q update \
     && DEBIAN_FRONTEND=noninteractive apt-get -q install -y \
     build-essential \
+    clang-19 \
     cmake \
     ninja-build \
     python3 \
@@ -54,7 +55,7 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \
 COPY scripts/install-swift.sh /scripts/install-swift.sh
 RUN chmod ugo+x /scripts/install-swift.sh
 RUN /scripts/install-swift.sh /usr/local/swift
-ENV PATH="/usr/local/swift/bin:${PATH}"
+ENV PATH="/usr/lib/llvm-19/bin:/usr/local/swift/bin:${PATH}"
 
 ARG ANDROID_NDK_VERSION=
 

swift-ci/sdks/android/build-docker

@@ -27,7 +27,7 @@ source ./scripts/toolchain-vars.sh
 
 # Check-out and patch the sources
 ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG}
-${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project
+${WORKDIR}/patches/apply-patches.sh ${WORKDIR}/source/swift-project
 
 mkdir -p ${WORKDIR}/products
 chmod ugo+rwx ${WORKDIR}/products
@@ -59,9 +59,10 @@ $DOCKER run -i --rm \
           -v ${WORKDIR}/products:/products:rw \
           ${CONTAINER_NAME} \
           /scripts/build.sh \
-            --source-dir /source \
-            --products-dir /products \
-            --host-toolchain /usr/local/swift \
-            --android-api ${ANDROID_API} \
-            --ndk-home /usr/local/ndk/${ANDROID_NDK_VERSION} \
-            --archs ${TARGET_ARCHS}
+            --source-dir "/source" \
+            --products-dir "/products" \
+            --host-toolchain "/usr/local/swift" \
+            --build-compiler "${BUILD_COMPILER}" \
+            --android-api "${ANDROID_API}" \
+            --ndk-home "/usr/local/ndk/${ANDROID_NDK_VERSION}" \
+            --archs "${TARGET_ARCHS}"

swift-ci/sdks/android/build-local

@@ -25,16 +25,18 @@ WORKDIR=$(realpath ${WORKDIR})
 HOST_OS=ubuntu$(lsb_release -sr)
 source ./scripts/toolchain-vars.sh
 
-SWIFT_ROOT=${WORKDIR}/host-toolchain
-HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr
-if [[ ! -d "$HOST_TOOLCHAIN" ]]; then
-    ./scripts/install-swift.sh ${HOST_TOOLCHAIN}
-fi
+if [[ "${BUILD_COMPILER}" != "1" ]]; then
+    SWIFT_ROOT=${WORKDIR}/host-toolchain
+    HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr
+    if [[ ! -d "$HOST_TOOLCHAIN" ]]; then
+        ./scripts/install-swift.sh ${HOST_TOOLCHAIN}
+    fi
 
-$HOST_TOOLCHAIN/bin/swift --version
+    $HOST_TOOLCHAIN/bin/swift --version
 
-# ensure the correct Swift is first in the PATH
-export PATH=$HOST_TOOLCHAIN/bin:$PATH
+    # ensure the correct Swift is first in the PATH
+    export PATH=$HOST_TOOLCHAIN/bin:$PATH
+fi
 
 export ANDROID_NDK_HOME=${WORKDIR}/ndk/${ANDROID_NDK_VERSION}
 
@@ -50,15 +52,16 @@ fi
 
 # Check-out and patch the sources
 ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG}
-${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project
+${WORKDIR}/patches/apply-patches.sh ${WORKDIR}/source/swift-project
 
 mkdir -p ${WORKDIR}/products
 
 ./scripts/build.sh \
-    --source-dir ${WORKDIR}/source \
-    --products-dir ${WORKDIR}/products \
-    --build-dir ${WORKDIR}/build \
-    --host-toolchain ${HOST_TOOLCHAIN} \
-    --android-api ${ANDROID_API} \
-    --ndk-home ${ANDROID_NDK_HOME} \
-    --archs ${TARGET_ARCHS}
+    --source-dir "${WORKDIR}/source" \
+    --products-dir "${WORKDIR}/products" \
+    --build-dir "${WORKDIR}/build" \
+    --build-compiler "${BUILD_COMPILER}" \
+    --host-toolchain "${HOST_TOOLCHAIN}" \
+    --android-api "${ANDROID_API}" \
+    --ndk-home "${ANDROID_NDK_HOME}" \
+    --archs "${TARGET_ARCHS}"

swift-ci/sdks/android/patches/apply-patches.sh

@@ -0,0 +1,41 @@
+#!/bin/bash -ex
+
+patches_dir=$(dirname $(realpath -- "${BASH_SOURCE[0]}"))
+cd ${1}
+
+# patch the patch, which seems to only be needed for an API less than 28
+# https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110
+perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' ${patches_dir}/swift-android.patch
+
+# remove the need to link in android-execinfo
+perl -pi -e 's;dispatch android-execinfo;dispatch;g' ${patches_dir}/swift-android.patch
+
+case "${BUILD_SCHEME}" in
+    release)
+        git apply -v -C1 ${patches_dir}/swift-android.patch
+        git apply -v -C1 ${patches_dir}/swift-android-testing-release.patch
+        ;;
+    swift-*-branch)
+        git apply -v -C1 ${patches_dir}/swift-android.patch
+        git apply -v -C1 ${patches_dir}/swift-android-testing-except-release.patch
+        ;;
+    development)
+        git apply -v -C1 ${patches_dir}/swift-android.patch
+        git apply -v -C1 ${patches_dir}/swift-android-trunk-libdispatch.patch
+        git apply -v -C1 ${patches_dir}/swift-android-testing-except-release.patch
+        ;;
+    *)
+        echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}"
+        exit 1
+        ;;
+esac
+
+perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift
+# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport)
+perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift
+
+# need to un-apply libandroid-spawn since we don't need it for API28+
+perl -pi -e 's;MATCHES "Android";MATCHES "AndroidDISABLED";g' llbuild/lib/llvm/Support/CMakeLists.txt
+perl -pi -e 's; STREQUAL Android\); STREQUAL AndroidDISABLED\);g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt
+
+