Added job for cleaning up key value rows (#718)

ref https://linear.app/ghost/issue/PROD-685

We can now use `yarn run cleanup-db` to cleanup expired rows from the key_value
table locally. This image will also be used as a scheduled Cloud Run Job and will be
updated on changes in the cleanup-expired-key-value-records directory

Author: allouis

.github/workflows/cicd.yml

@@ -386,3 +386,55 @@ jobs:
           status: ${{ job.status }}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
+  build-cleanup-job:
+    name: Build & Push cleanup job
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check for changes in cleanup job
+        id: changes
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            cleanup:
+              - 'jobs/cleanup-expired-key-value-records/**'
+
+      - name: Authenticate with GCP
+        if: steps.changes.outputs.cleanup == 'true'
+        id: gcp-auth
+        uses: google-github-actions/auth@v2
+        with:
+          token_format: access_token
+          workload_identity_provider: projects/687476608778/locations/global/workloadIdentityPools/github-oidc-activitypub/providers/github-provider-activitypub
+          service_account: [email protected]
+
+      - name: Login to GCP Artifact Registry
+        if: steps.changes.outputs.cleanup == 'true'
+        uses: docker/login-action@v3
+        with:
+          registry: europe-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.gcp-auth.outputs.access_token }}
+
+      - name: Docker metadata
+        if: steps.changes.outputs.cleanup == 'true'
+        id: cleanup-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: europe-docker.pkg.dev/ghost-activitypub/activitypub/cleanup-expired-key-value-records
+          tags: |
+            type=sha
+            type=edge,branch=main
+
+      - name: Build & Push cleanup job image
+        if: steps.changes.outputs.cleanup == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: jobs/cleanup-expired-key-value-records
+          push: true
+          tags: ${{ steps.cleanup-meta.outputs.tags }}

docker-compose.yml

@@ -55,6 +55,15 @@ services:
       mysql:
         condition: service_healthy
 
+  cleanup-expired-key-value-records:
+    build: jobs/cleanup-expired-key-value-records
+    environment:
+      - MYSQL_USER=ghost
+      - MYSQL_PASSWORD=password
+      - MYSQL_HOST=mysql
+      - MYSQL_PORT=3306
+      - MYSQL_DATABASE=activitypub
+
   scripts:
     build: scripts
     depends_on:
@@ -157,6 +166,15 @@ services:
       mysql-testing:
         condition: service_healthy
 
+  cleanup-expired-key-value-records-testing:
+    build: jobs/cleanup-expired-key-value-records
+    environment:
+      - MYSQL_USER=ghost
+      - MYSQL_PASSWORD=password
+      - MYSQL_HOST=mysql-testing
+      - MYSQL_PORT=3306
+      - MYSQL_DATABASE=activitypub
+
   cucumber-tests:
     networks:
       - test_network

jobs/cleanup-expired-key-value-records/Dockerfile

@@ -0,0 +1,5 @@
+FROM mysql:8.3
+
+COPY cleanup-expired-key-value-records /bin
+
+ENTRYPOINT ["cleanup-expired-key-value-records"]

jobs/cleanup-expired-key-value-records/cleanup-expired-key-value-records

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -e
+
+mysql \
+  --host="$MYSQL_HOST" \
+  --port="$MYSQL_PORT" \
+  --user="$MYSQL_USER" \
+  --password="$MYSQL_PASSWORD" \
+  --database="$MYSQL_DATABASE" \
+  -e "DELETE FROM key_value WHERE expires IS NOT NULL AND expires < UTC_TIMESTAMP();"

package.json

@@ -15,6 +15,7 @@
     "fix": "docker compose down --rmi all activitypub activitypub-testing cucumber-tests nginx",
     "logs": "docker compose logs activitypub -f",
     "wipe-db": "docker compose down -v mysql && docker compose up -d activitypub nginx && echo 'ActivityPub DB wiped. You must restart Ghost'",
+    "cleanup-db": "docker compose run --rm cleanup-expired-key-value-records cleanup-expired-key-value-records-testing",
     "migration": "docker compose run migrate create",
     "migrate": "docker compose up migrate",
     "test:cucumber": "./docker/cucumber-tests",