Add openGauss support to dataprep microservice and update related doc… #6389
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2024 Intel Corporation | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: MicroService-test | |
| permissions: read-all | |
| on: | |
| pull_request_target: | |
| branches: ["main", "*rc", "pre-ci", "**uv"] | |
| types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped | |
| paths: | |
| - comps/** | |
| - tests/** | |
| - "!**.md" | |
| - .github/workflows/pr-microservice-test.yml | |
| # If there is a new commit, the previous jobs will be canceled | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| job1: | |
| uses: ./.github/workflows/_get-test-matrix.yml | |
| Microservice-test: | |
| needs: job1 | |
| if: ${{ needs.job1.outputs.is_empty == 'false' }} | |
| strategy: | |
| matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }} | |
| runs-on: ${{ matrix.hardware }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| continue-on-error: true | |
| steps: | |
| - name: Clean Up Working Directory | |
| run: | | |
| # clean up workspace | |
| sudo rm -rf ${{github.workspace}}/* | |
| echo "Cleaning up test-comps-xxx containers ..." | |
| cid=$(docker ps -aq --filter "name=test-comps-*") | |
| if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi | |
| echo "Cleaning up containers using ports..." | |
| cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}') | |
| if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi | |
| # clean up images | |
| docker images --filter reference="*/*:comps" -q | xargs -r docker rmi && sleep 1s | |
| docker images --filter reference="opea/*:*" -q | xargs -r docker rmi && sleep 1s | |
| docker system prune -f | |
| docker ps | |
| - name: Checkout out Repo | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: "refs/pull/${{ github.event.number }}/merge" | |
| - name: Get AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 | |
| if: ${{ contains(matrix.service, 'bedrock') }} | |
| with: | |
| role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} | |
| aws-region: us-east-1 | |
| - name: Run microservice test | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }} | |
| PINECONE_KEY: ${{ secrets.PINECONE_KEY }} | |
| PREDICTIONGUARD_API_KEY: ${{ secrets.PREDICTIONGUARD_API_KEY }} | |
| service: ${{ matrix.service }} | |
| hardware: ${{ matrix.hardware }} | |
| run: | | |
| export PATH=${HOME}/miniforge3/bin/:$PATH | |
| cd tests | |
| echo "log_name=${service}" >> $GITHUB_ENV | |
| if [ -d "/data2/hf_model" ]; then | |
| export model_cache="/data2/hf_model" | |
| else | |
| echo "Model cache directory /data2/hf_model does not exist" | |
| export model_cache="~/.cache/huggingface/hub" | |
| fi | |
| timeout 60m bash $(find . -type f -name test_${service}.sh) | |
| - name: Clean up container | |
| run: | | |
| echo "Cleaning up test-comps-xxx containers ..." | |
| cid=$(docker ps -aq --filter "name=test-comps-*") | |
| if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi | |
| echo "Cleaning up containers using ports..." | |
| cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}') | |
| if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi | |
| docker ps | |
| echo "Cleaning up images ..." | |
| if [[ "${{ matrix.hardware }}" == "intel-cpu" ]]; then | |
| docker system prune -a -f | |
| else | |
| docker images --filter reference="*/*:comps" -q | xargs -r docker rmi && sleep 1s | |
| docker images --filter reference="opea/*:*" -q | xargs -r docker rmi && sleep 1s | |
| docker system prune -f | |
| fi | |
| docker images | |
| - name: Publish pipeline artifact | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.log_name }} | |
| path: ${{ github.workspace }}/tests/*.log |