upload nightly worker prom metrics to S3

Author: THardy98

.github/workflows/nightly-throughput-stress.yml

@@ -6,8 +6,7 @@ on:
     - cron: '00 11 * * *'
   push:
     branches:
-      - add-nightly-throughput-stress-workflow
-      - cleanup-nightly-tps-workflow
+      - upload-nightly-metrics-to-s3
   workflow_dispatch:
     inputs:
       duration:
@@ -34,6 +33,9 @@ env:
   TEST_DURATION: ${{ inputs.duration || vars.NIGHTLY_TEST_DURATION || '5h' }}
   TEST_TIMEOUT: ${{ inputs.timeout || vars.NIGHTLY_TEST_TIMEOUT || '5h30m' }}
 
+  # AWS S3 metrics upload ARN
+  AWS_S3_METRICS_UPLOAD_ROLE_ARN: ${{ vars.AWS_S3_METRICS_UPLOAD_ROLE_ARN }}
+
   # Logging and artifacts
   WORKER_LOG_DIR: /tmp/throughput-stress-logs
 
@@ -49,6 +51,7 @@ jobs:
     permissions:
       contents: read
       actions: write
+      id-token: write
 
     steps:
       - name: Print test configuration
@@ -118,13 +121,37 @@ jobs:
             --duration $TEST_DURATION \
             --timeout $TEST_TIMEOUT \
             --max-concurrent 10 \
+            --prom-listen-address 127.0.0.1:9091 \
+            --worker-prom-listen-address 127.0.0.1:9092 \
+            --prom-instance-addr 127.0.0.1:9090 \
+            --prom-instance-config \
+            --prom-export-worker-metrics $RUN_ID.parquet \
             --option internal-iterations=10 \
             --option continue-as-new-after-iterations=3 \
             --option sleep-time=1s \
             --option visibility-count-timeout=5m \
             --option min-throughput-per-hour=1000 \
             2>&1 | tee $WORKER_LOG_DIR/scenario.log
 
+      - name: Configure AWS credentials
+        if: always()
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ env.AWS_S3_METRICS_UPLOAD_ROLE_ARN }}
+          aws-region: us-west-2
+  
+      - name: Upload metrics to S3
+        if: always()
+        run: |
+          DATE=$(date +%Y-%m-%d)
+          # Use test/ prefix on non-main branches
+          PREFIX="language=go/date=$DATE"
+          if [[ "${{ github.ref }}" != "refs/heads/main" ]]; then
+            PREFIX="test/$PREFIX"
+          fi
+          aws s3 cp omes/$RUN_ID.parquet \
+            "s3://cloud-data-ingest-prod/github/sdk_load_test/$PREFIX/$RUN_ID.parquet"
+
       - name: Upload logs on failure/cancellation
         if: failure() || cancelled()
         uses: actions/upload-artifact@v4