From bf1466bc89a511caa80ef8da1625dc0e00033aac Mon Sep 17 00:00:00 2001 From: Shruti Chaturvedi Date: Tue, 7 Feb 2023 22:13:09 +0530 Subject: [PATCH 1/4] Integrate Uffizzi --- .github/workflows/uffizzi-build.yml | 102 +++++++++++++ .github/workflows/uffizzi-preview.yml | 88 +++++++++++ docker-compose.uffizzi.yml | 33 ++++ python-requirements.txt | 2 +- util/container/Dockerfile.uffizzi | 212 ++++++++++++++++++++++++++ util/container/start.sh | 2 +- 6 files changed, 437 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/uffizzi-build.yml create mode 100644 .github/workflows/uffizzi-preview.yml create mode 100644 docker-compose.uffizzi.yml create mode 100644 util/container/Dockerfile.uffizzi diff --git a/.github/workflows/uffizzi-build.yml b/.github/workflows/uffizzi-build.yml new file mode 100644 index 0000000000000..48ef97f52ebab --- /dev/null +++ b/.github/workflows/uffizzi-build.yml @@ -0,0 +1,102 @@ +name: Build PR Image +on: + pull_request: + types: [opened, synchronize, reopened, closed, review_requested] + +jobs: + build-opentitan: + name: Build and push `OpenTitan` + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.meta.outputs.tags }} + if: ${{ github.event.action != 'closed' }} + steps: + - name: Checkout git repo + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Generate UUID image name + id: uuid + run: echo "UUID_WORKER=$(uuidgen)" >> $GITHUB_ENV + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: registry.uffizzi.com/${{ env.UUID_WORKER }} + tags: | + type=raw,value=60d + + - name: Build and Push Image to registry.uffizzi.com - Uffizzi's ephemeral Registry + uses: docker/build-push-action@v3 + with: + context: . + file: ./util/container/Dockerfile.uffizzi + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + cache-from: type=gha + cache-to: type=gha, mode=max + + render-compose-file: + name: Render Docker Compose File + # Pass output of this workflow to another triggered by `workflow_run` event. + runs-on: ubuntu-latest + needs: + - build-opentitan + outputs: + compose-file-cache-key: ${{ steps.hash.outputs.hash }} + steps: + - name: Checkout git repo + uses: actions/checkout@v3 + - name: Render Compose File + run: | + OPENTITAN_IMAGE=${{ needs.build-opentitan.outputs.tags }} + export OPENTITAN_IMAGE + export UFFIZZI_URL=\$UFFIZZI_URL + GHA_ACTOR=${{github.actor}} + GHA_REPO=${{github.event.repository.name}} + GHA_BRANCH=${{github.head_ref}} + export GHA_ACTOR GHA_REPO GHA_BRANCH + # Render simple template from environment variables. + envsubst < docker-compose.uffizzi.yml > docker-compose.rendered.yml + cat docker-compose.rendered.yml + - name: Upload Rendered Compose File as Artifact + uses: actions/upload-artifact@v3 + with: + name: preview-spec + path: docker-compose.rendered.yml + retention-days: 2 + - name: Serialize PR Event to File + run: | + cat << EOF > event.json + ${{ toJSON(github.event) }} + + EOF + - name: Upload PR Event as Artifact + uses: actions/upload-artifact@v3 + with: + name: preview-spec + path: event.json + retention-days: 2 + + delete-preview: + name: Call for Preview Deletion + runs-on: ubuntu-latest + if: ${{ github.event.action == 'closed' }} + steps: + # If this PR is closing, we will not render a compose file nor pass it to the next workflow. + - name: Serialize PR Event to File + run: | + cat << EOF > event.json + ${{ toJSON(github.event) }} + + EOF + - name: Upload PR Event as Artifact + uses: actions/upload-artifact@v3 + with: + name: preview-spec + path: event.json + retention-days: 2 diff --git a/.github/workflows/uffizzi-preview.yml b/.github/workflows/uffizzi-preview.yml new file mode 100644 index 0000000000000..ded249260d8f2 --- /dev/null +++ b/.github/workflows/uffizzi-preview.yml @@ -0,0 +1,88 @@ +name: Deploy Uffizzi Preview + +# Workflow run — runs only when the Build PR/ uffizzi-build.yml completes successfully. +on: + workflow_run: + workflows: + - "Build PR Image" + types: + - completed + +jobs: + cache-compose-file: + name: Cache Compose File + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + outputs: + compose-file-cache-key: ${{ env.HASH }} + pr-number: ${{ env.PR_NUMBER }} + steps: + - name: 'Download artifacts' + # Fetch output (zip archive) from the workflow run that triggered this workflow. + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "preview-spec" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data)); + + - name: 'Unzip artifact' + run: unzip preview-spec.zip + - name: Read Event into ENV + run: | + echo 'EVENT_JSON<> $GITHUB_ENV + cat event.json >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + + - name: Hash Rendered Compose File + id: hash + # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact. + if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }} + run: echo "HASH=$(md5sum docker-compose.rendered.yml | awk '{ print $1 }')" >> $GITHUB_ENV + - name: Cache Rendered Compose File + if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }} + uses: actions/cache@v3 + with: + path: docker-compose.rendered.yml + key: ${{ env.HASH }} + + - name: Read PR Number From Event Object + id: pr + run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV + - name: DEBUG - Print Job Outputs + if: ${{ runner.debug }} + run: | + echo "PR number: ${{ env.PR_NUMBER }}" + echo "Compose file hash: ${{ env.HASH }}" + cat event.json + + deploy-uffizzi-preview: + name: Use Remote Workflow to Preview on Uffizzi + needs: + - cache-compose-file + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@v2 + with: + # If this workflow was triggered by a PR close event, cache-key will be an empty string + # and this reusable workflow will delete the preview deployment. + compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }} + compose-file-cache-path: docker-compose.rendered.yml + server: https://app.qa-gke.uffizzi.com + pr-number: ${{ needs.cache-compose-file.outputs.pr-number }} + permissions: + contents: read + pull-requests: write + id-token: write diff --git a/docker-compose.uffizzi.yml b/docker-compose.uffizzi.yml new file mode 100644 index 0000000000000..265dcffdbb0c5 --- /dev/null +++ b/docker-compose.uffizzi.yml @@ -0,0 +1,33 @@ +version: "3" + +x-uffizzi: + ingress: + service: opentitan + port: 7681 + +services: + + opentitan: + image: "${OPENTITAN_IMAGE}" + ports: + - "7700:7700" + - "7681:7681" + # entrypoint: ["/bin/bash", "-c"] + # command: ["cd /home/dev/src && ttyd bash"] + entrypoint: ["/bin/bash"] + command: + - "-c" + - "apt-get update && \ + apt-get install neovim -y && \ + apt-get install unzip -y && \ + apt-get install wget -y && \ + wget 'https://github.com/$GHA_ACTOR/$GHA_REPO/archive/refs/heads/$GHA_BRANCH.zip' && \ + unzip $GHA_BRANCH.zip -d . && \ + mv $GHA_REPO-$GHA_BRANCH /home/dev/src && \ + cd /home/dev/src && \ + ttyd bash + " + deploy: + resources: + limits: + memory: 4000M diff --git a/python-requirements.txt b/python-requirements.txt index 991fa5a7965ac..43bcd22859a89 100644 --- a/python-requirements.txt +++ b/python-requirements.txt @@ -6,7 +6,7 @@ hjson==3.1.0 jsonschema==4.17.3; python_version >= "3.7" libcst==0.4.1 -mako==1.1.6 +mako==1.2.0 pluralizer==1.2.0 pycryptodome==3.15.0 pyelftools==0.29 diff --git a/util/container/Dockerfile.uffizzi b/util/container/Dockerfile.uffizzi new file mode 100644 index 0000000000000..672d1c388a705 --- /dev/null +++ b/util/container/Dockerfile.uffizzi @@ -0,0 +1,212 @@ +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Docker container containing various hardware and software development tools +# for OpenTitan. + +# Global configuration options. +ARG VERILATOR_VERSION=4.210 +ARG VERIBLE_VERSION=v0.0-2135-gb534c1fe +# The RISCV toolchain version should match the release tag used in GitHub. +ARG RISCV_TOOLCHAIN_TAR_VERSION=20220210-1 +ARG RUST_VERSION=1.60.0 +# This should match the version in bazelish.sh. +ARG BAZELISK_VERSION=v1.11.0 +# This should match the version in ci/install-package-dependencies.sh +ARG GCC_VERSION=9 +# This should match the version of the lowRISC RISC-V toolchain. +ARG CLANG_VERSION=13 + +# Main container image. +FROM ubuntu:20.04 AS opentitan +ARG VERILATOR_VERSION +ARG VERIBLE_VERSION +ARG RISCV_TOOLCHAIN_TAR_VERSION +ARG RUST_VERSION +ARG BAZELISK_VERSION +ARG GCC_VERSION +ARG CLANG_VERSION + +LABEL version="1.0" +LABEL description="OpenTitan development container." +LABEL maintainer="opentitan-dev@opentitan.org" + +# Use bash as default shell. +RUN ln -sf /bin/bash /bin/sh + +# Add OBS repository to apt sources. +RUN OBS_URL="https://download.opensuse.org/repositories"; \ + OBS_PATH="/home:/phiwag:/edatools/xUbuntu_20.04"; \ + REPO_URL="${OBS_URL}${OBS_PATH}"; \ + \ + EDATOOLS_REPO_KEY="${REPO_URL}/Release.key"; \ + EDATOOLS_REPO="deb ${REPO_URL}/ /"; \ + \ + apt-get update && \ + apt-get install -y curl && \ + \ + curl -f -sL -o "$TMPDIR/obs.asc" "$EDATOOLS_REPO_KEY" || { \ + error "Failed to download repository key from ${REPO_URL}"; \ + } && \ + echo "$EDATOOLS_REPO" > "$TMPDIR/obs.list" && \ + mv "$TMPDIR/obs.asc" /etc/apt/trusted.gpg.d/obs.asc && \ + mv "$TMPDIR/obs.list" /etc/apt/sources.list.d/edatools.list + +# Install system packages +# +# Install (and cleanup) required packages (from apt-requirements.txt). +# Also add some additional packages for the use within this container and for +# developer convenience: +# - gosu and sudo are used by the scripting to make the image more convenient +# to use. +# - locales and locales-all are required to set the locale. +# - minicom and screen are useful to see UART communication. +# - dc and time are requirements of Synopsys VCS. +# - software-properties-common is required to be able to install newer gcc versions. + +# Necessary to avoid user interaction with tzdata during install +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=UTC + +COPY apt-requirements.txt /tmp/apt-requirements.txt +RUN echo "verilator-${VERILATOR_VERSION}" >>/tmp/apt-requirements.txt \ + && sed -i -e '/^$/d' -e '/^#/d' -e 's/#.*//' /tmp/apt-requirements.txt \ + && apt-get update \ + && xargs apt-get install -y /dev/null" + +ENTRYPOINT [ "/start.sh" ] diff --git a/util/container/start.sh b/util/container/start.sh index daf0a0847bde0..8df0e2ae90ab0 100755 --- a/util/container/start.sh +++ b/util/container/start.sh @@ -14,4 +14,4 @@ usermod -o -u "$DEV_UID" dev >/dev/null 2>&1 test -f "${USER_CONFIG}" && export BASH_ENV=${USER_CONFIG} cd /home/dev || exit -exec gosu dev:dev /bin/bash -c "$@" +exec gosu dev:dev /bin/bash -c ttyd bash From fcadc211821d806dcde1dcdcac3a585939834a32 Mon Sep 17 00:00:00 2001 From: Shruti Chaturvedi Date: Fri, 10 Feb 2023 19:19:26 +0530 Subject: [PATCH 2/4] Integrate uffizzi --- .bazelrc | 2 +- python-requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index d62f3f058834a..7d2b3155c5a52 100644 --- a/.bazelrc +++ b/.bazelrc @@ -20,7 +20,7 @@ build --incompatible_enable_cc_toolchain_resolution # This lets us generate key/value pairs for the workspace which can be # accessed like we do in util/BUILD -build --workspace_status_command=util/get_workspace_status.sh +build # This enables convenient building for opentitan targets with the argument # --config=riscv32 diff --git a/python-requirements.txt b/python-requirements.txt index 43bcd22859a89..6682fafbe004b 100644 --- a/python-requirements.txt +++ b/python-requirements.txt @@ -42,7 +42,6 @@ types-dataclasses==0.6.6 types-pkg_resources==0.1.3 types-pyyaml==6.0.11 types-tabulate==0.8.11 - # Dependency of sw/host/vendor/google_verible_verilog_syntax_py anytree==2.8.0 From 1d36120995fe55d64f95d3e86c4300028e7a4b46 Mon Sep 17 00:00:00 2001 From: Shruti Chaturvedi <66940685+ShrutiC-git@users.noreply.github.com> Date: Sat, 25 Feb 2023 20:21:37 +0530 Subject: [PATCH 3/4] GHA Context --- .github/workflows/uffizzi-build.yml | 1 + docker-compose.uffizzi.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/uffizzi-build.yml b/.github/workflows/uffizzi-build.yml index 48ef97f52ebab..f0b29bd008f1c 100644 --- a/.github/workflows/uffizzi-build.yml +++ b/.github/workflows/uffizzi-build.yml @@ -59,6 +59,7 @@ jobs: GHA_ACTOR=${{github.actor}} GHA_REPO=${{github.event.repository.name}} GHA_BRANCH=${{github.head_ref}} + GHA_REPOSITORY=${{github.event.repository.full_name}} export GHA_ACTOR GHA_REPO GHA_BRANCH # Render simple template from environment variables. envsubst < docker-compose.uffizzi.yml > docker-compose.rendered.yml diff --git a/docker-compose.uffizzi.yml b/docker-compose.uffizzi.yml index 265dcffdbb0c5..f831c8291cd23 100644 --- a/docker-compose.uffizzi.yml +++ b/docker-compose.uffizzi.yml @@ -21,7 +21,7 @@ services: apt-get install neovim -y && \ apt-get install unzip -y && \ apt-get install wget -y && \ - wget 'https://github.com/$GHA_ACTOR/$GHA_REPO/archive/refs/heads/$GHA_BRANCH.zip' && \ + wget 'https://github.com/$GHA_REPOSITORY/archive/refs/heads/$GHA_BRANCH.zip' && \ unzip $GHA_BRANCH.zip -d . && \ mv $GHA_REPO-$GHA_BRANCH /home/dev/src && \ cd /home/dev/src && \ From 6d792b8167b3b53c3594b770d6572b004ec2c4b3 Mon Sep 17 00:00:00 2001 From: Shruti Chaturvedi <66940685+ShrutiC-git@users.noreply.github.com> Date: Sat, 25 Feb 2023 21:22:26 +0530 Subject: [PATCH 4/4] Update uffizzi-build.yml --- .github/workflows/uffizzi-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/uffizzi-build.yml b/.github/workflows/uffizzi-build.yml index f0b29bd008f1c..86186b727d1bd 100644 --- a/.github/workflows/uffizzi-build.yml +++ b/.github/workflows/uffizzi-build.yml @@ -60,7 +60,7 @@ jobs: GHA_REPO=${{github.event.repository.name}} GHA_BRANCH=${{github.head_ref}} GHA_REPOSITORY=${{github.event.repository.full_name}} - export GHA_ACTOR GHA_REPO GHA_BRANCH + export GHA_ACTOR GHA_REPO GHA_BRANCH GHA_REPOSITORY # Render simple template from environment variables. envsubst < docker-compose.uffizzi.yml > docker-compose.rendered.yml cat docker-compose.rendered.yml