semaphore.yml

# $schema: https://raw.githubusercontent.com/semaphoreio/semaphore/98bb4752c79049d36324e8c76451cd704c7924f3/plumber/spec/priv/v1.0.yml
# yaml-language-server: $schema=https://raw.githubusercontent.com/semaphoreio/semaphore/98bb4752c79049d36324e8c76451cd704c7924f3/plumber/spec/priv/v1.0.yml
# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common
# template and configurations in service.yml.
# Any modifications made to ths file will be overwritten by the generated content in nightly runs.
# For more information, please refer to the page:
# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI
version: v1.0
name: build-test-release
agent:
  machine:
    type: s1-prod-ubuntu24-04-amd64-1

fail_fast:
  cancel:
    when: "true"

execution_time_limit:
  hours: 1

queue:
  - when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
    processing: parallel

global_job_config:
  prologue:
    commands:
      - checkout
      - if [[ $SEMAPHORE_GIT_BRANCH =~ ^7\..* ]]; then sem-version java 8; else sem-version java 17; fi
      - . cache-maven restore
      - export GIT_COMMIT=$(git rev-parse --verify HEAD --short)
      - export BUILD_NUMBER=$(echo $SEMAPHORE_WORKFLOW_ID | cut -f1 -d"-")
      - export BRANCH_TAG=$(echo $SEMAPHORE_GIT_BRANCH | tr / -)
      # For PR Builds using Packaging
      - if [ $BRANCH_TAG == "master" ]; then export BUILD_KEY=$(pinto get-master-version); else export BUILD_KEY=$BRANCH_TAG; fi
      - export PACKAGING_BUCKET="s3://dev-confluent-packages-654654529379-us-west-2/$BRANCH_TAG/"
      - export LATEST_PACKAGING_BUILD_NUMBER=$(aws s3 ls $PACKAGING_BUCKET --no-paginate | grep 'PRE' | awk '{print $NF}' | awk '{print substr($1, 1, length($1)-1)}' | sort -n | tail -n 1)
      # Check if version is complete, otherwise use the previous version
      - (aws s3 ls $PACKAGING_BUCKET$LATEST_PACKAGING_BUILD_NUMBER/deb/ && aws s3 ls $PACKAGING_BUCKET$LATEST_PACKAGING_BUILD_NUMBER/rpm/ && aws s3 ls 
        $PACKAGING_BUCKET$LATEST_PACKAGING_BUILD_NUMBER/archive/) || export LATEST_PACKAGING_BUILD_NUMBER=$(aws s3 ls $PACKAGING_BUCKET --no-paginate | grep 'PRE' | awk '{print $NF}' | awk '{print 
        substr($1, 1, length($1)-1)}' | sort -n | tail -n 2 | head -n 1)
      - export CONFLUENT_VERSION=$(pinto get-version --build $BUILD_KEY --key confluent.version)
      - export DEFAULT_OS_TYPE="ubi"
      - export URL_CONFLUENT_VERSION=$(echo $CONFLUENT_VERSION | awk -F . '{print $1"."$2}')
      - export PACKAGES_URL="https://s3-us-west-2.amazonaws.com/dev-confluent-packages-654654529379-us-west-2/$BRANCH_TAG/$LATEST_PACKAGING_BUILD_NUMBER/PACKAGE_TYPE/$URL_CONFLUENT_VERSION"
      - export PACKAGING_BUILD_NUMBER=$LATEST_PACKAGING_BUILD_NUMBER
      - >-
        if [[ $IS_BETA || $IS_HOTFIX || $IS_POST ]]; then
            export PLATFORM_LABEL=$(echo $BRANCH_TAG | awk -F - '{print "~"$2}')
        elif [[ ! $IS_RELEASE ]]; then
            export PLATFORM_LABEL="~SNAPSHOT"
        else
            export PLATFORM_LABEL=""
        fi
      # Guard -D flags: fabric8 fails when build args resolve to null from empty -D values (e.g. -DCONFLUENT_VERSION=)
      - |
        export PACKAGING_BUILD_ARGS=""
        if [[ -n "$CONFLUENT_VERSION" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_VERSION=$CONFLUENT_VERSION"; fi
        if [[ -n "$PLATFORM_LABEL" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL"; fi
        if [[ -n "$CONFLUENT_DEB_VERSION" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION"; fi
        if [[ -n "$ALLOW_UNSIGNED" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DALLOW_UNSIGNED=$ALLOW_UNSIGNED"; fi
      - >-
        if [[ $IS_RELEASE && $PACKAGING_BUILD_NUMBER ]]; then
          if [[ $IS_RC ]]; then
              export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/staging-confluent-packages-maven-654654529379-us-west-2/v$BRANCH_NAME/maven"
              if [[ $PACKAGES_MAVEN_URL ]]; then
                  export MAVEN_PACKAGES_URL=$PACKAGES_MAVEN_URL
              fi
          elif [[ $IS_HOTFIX ]]; then
              export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/dev-confluent-packages-maven-654654529379-us-west-2/$BRANCH_TAG/maven"
              if [[ $PACKAGES_MAVEN_URL ]]; then
                  export MAVEN_PACKAGES_URL=$PACKAGES_MAVEN_URL
              fi
          fi
          # Overwrite maven global configuration
          gh api repos/confluentinc/depot/contents/python/cp_devtools/templates/template_resources/maven/maven_semaphore_cp_dockerfile_settings.xml | jq -r '.content' | base64 -d > template-settings.xml
          [[ -f ~/.m2/settings.xml ]] && mv ~/.m2/settings.xml ~/.m2/settings.xml.bak; envsubst < template-settings.xml > ~/.m2/settings.xml
        else
          echo "This job is not a isHotfixJob or isRcJob (What we know how to handle) - and we don't know how to handle it"
        fi
      - export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/"
      - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/"
      - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_PROD_REGISTRY
      - export LATEST_TAG=$BRANCH_TAG-latest
      - export DOCKER_UPSTREAM_TAG="$LATEST_TAG"
      - export DOCKER_REPOS="confluentinc/kafka-streams-examples"
      - export COMMUNITY_DOCKER_REPOS=""
      - |
        export COMMUNITY_MVN_PL_ARGS=""
        if [[ $SKIP_COMMUNITY == "True" ]]; then
          # Filter out community repos from DOCKER_REPOS
          DOCKER_REPOS=$(comm -23 <(echo "$DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs)
          export DOCKER_REPOS
          echo "DOCKER_REPOS after skipping community images - $DOCKER_REPOS"

          # Check if current DOCKER_IMAGE is in community repos, skip job execution
          for skip_repo in $COMMUNITY_DOCKER_REPOS; do
            if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
              echo "Skipping build for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
              export SEMAPHORE_JOB_RESULT=passed
              return 130
            fi
          done
        fi
      - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER"
      - export AMD_ARCH=.amd64
      - export ARM_ARCH=.arm64
blocks:
  - name: Validation
    dependencies: []
    run:
      when: "branch = 'master'"
    task:
      jobs:
        - name: Validation
          commands:
            - ci-sem-pint -c
  - name: Build, Test, & Scan AMD
    dependencies: ["Validation"]
    run:
      when: "pull_request =~ '.*'"
    task:
      jobs:
        - name: Build, Test, & Scan ubi9
          commands:
            - export OS_TAG="-ubi9"
            - export DOCKER_UPSTREAM_TAG="${DOCKER_UPSTREAM_TAG}${OS_TAG}"
            - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG
            - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH
            - ci-tools ci-update-version --direct-pom-edit
            - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$PACKAGES_URL"
            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker-fabric8 clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY 
              -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH 
              -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $COMMUNITY_MVN_PL_ARGS
            - . cache-maven store
            - >-
              for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
              do
                cve-scan $dev_image
              done
            - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
      epilogue:
        always:
          commands:
            - . publish-test-results
            - artifact push workflow target/test-results
            - artifact push workflow target --destination target-AMD
  - name: Build & Test ARM
    dependencies: ["Validation"]
    run:
      when: "pull_request =~ '.*'"
    task:
      agent:
        machine:
          type: s1-prod-ubuntu24-04-arm64-1
      jobs:
        - name: Build & Test ubi9
          commands:
            - export OS_TAG="-ubi9"
            - export DOCKER_UPSTREAM_TAG="${DOCKER_UPSTREAM_TAG}${OS_TAG}"
            - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG
            - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH
            - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$PACKAGES_URL"
            - ci-tools ci-update-version --direct-pom-edit
            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker-fabric8 clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY 
              -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH 
              -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $COMMUNITY_MVN_PL_ARGS
            - . cache-maven store
            - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
      epilogue:
        always:
          commands:
            - . publish-test-results
            - artifact push workflow target/test-results
            - artifact push workflow target --destination target-ARM
after_pipeline:
  task:
    agent:
      machine:
        type: s1-prod-ubuntu24-04-arm64-00
    jobs:
      - name: Metrics
        commands:
          - emit-ci-metrics -p -a test-results
      - name: Publish Test Results
        commands:
          - test-results gen-pipeline-report
      - name: SonarQube
        commands:
          - checkout
          - artifact pull workflow target-AMD
          - artifact pull workflow target-ARM
          - emit-sonarqube-data --run_only_sonar_scan