Skip to content

[Don't merge] Added staging tests to CI/CD #687

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 42 additions & 2 deletions .github/workflows/cicd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ jobs:
load: true
tags: ${{ steps.migrations-docker-metadata.outputs.tags }}

- name: "Run Tests"
run: yarn test
#- name: "Run Tests"
# run: yarn test

- name: "Authenticate with GCP"
if: github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize' || github.event.action == 'reopened' || github.event.action == 'labeled' || github.event.action == 'unlabeled'))
Expand Down Expand Up @@ -240,6 +240,26 @@ jobs:
labels: |-
commit-sha=${{ github.sha }}

- name: "Deploy Tests to Cloud Run"
if: ${{ steps.check-labels.outputs.is_ephemeral_staging == 'true' }}
uses: google-github-actions/deploy-cloudrun@v2
with:
image: europe-docker.pkg.dev/ghost-activitypub/activitypub/activitypub:${{ needs.build-test-push.outputs.migrations_docker_version }}
region: europe-west4
job: stg-pr-${{ github.event.pull_request.number }}-tests
flags: --command="yarn" --args "_test:single" --wait --execute-now
skip_default_labels: true
labels: |-
commit-sha=${{ github.sha }}

- name: "Destroy Tests databases"
if: ${{ steps.check-labels.outputs.is_ephemeral_staging == 'true' }}
run: |
TEST_DATABASES=$(gcloud sql databases list --instance=stg-netherlands-activitypub --filter="name~pr_${{ github.event.pull_request.number }}_test*" --format="value(name)" --project ${GCP_PROJECT})
for TEST_DATABASE in ${TEST_DATABASES}; do
gcloud sql databases delete ${TEST_DATABASE} --instance=stg-netherlands-activitypub --quiet --project ${GCP_PROJECT}
done

- name: "Add route to GCP Load Balancer"
if: ${{ steps.check-labels.outputs.is_ephemeral_staging == 'true' }}
env:
Expand Down Expand Up @@ -306,6 +326,26 @@ jobs:
labels: |-
commit-sha=${{ github.sha }}

- name: "Deploy Tests to Cloud Run"
if: ${{ matrix.region == 'europe-west4' }}
uses: google-github-actions/deploy-cloudrun@v2
with:
image: europe-docker.pkg.dev/ghost-activitypub/activitypub/activitypub:${{ needs.build-test-push.outputs.migrations_docker_version }}
region: ${{ matrix.region }}
job: stg-${{ matrix.region_name }}-activitypub-tests
flags: --command="yarn _test:single" --wait --execute-now
skip_default_labels: true
labels: |-
commit-sha=${{ github.sha }}

- name: "Destroy Tests databases"
if: ${{ matrix.region == 'europe-west4' }}
run: |
TEST_DATABASES=$(gcloud sql databases list --instance=stg-netherlands-activitypub --filter="name~pr_${{ github.event.pull_request.number }}_test*" --format="value(name)" --project ${GCP_PROJECT})
for TEST_DATABASE in ${TEST_DATABASES}; do
gcloud sql databases delete ${TEST_DATABASE} --instance=stg-netherlands-activitypub --quiet --project ${GCP_PROJECT}
done

- name: "Deploy ActivityPub Queue to Cloud Run"
uses: google-github-actions/deploy-cloudrun@v2
with:
Expand Down
1 change: 1 addition & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ RUN yarn && \
COPY tsconfig.json .

COPY src ./src
COPY vitest.config.ts vitest.config.ts

ENV NODE_ENV=production
RUN yarn build
Expand Down
Binary file added src/storage/gcloud-storage/assets/dog.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
153 changes: 153 additions & 0 deletions src/storage/gcloud-storage/gcp-storage.service.integration.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
import { readFileSync } from 'node:fs';
import path from 'node:path';
import { getError, getValue, isError } from 'core/result';
import { File as NodeFile } from 'fetch-blob/file.js';
import { beforeAll, describe, expect, it } from 'vitest';
import { GCPStorageService } from './gcp-storage.service';

const logger = {
info: console.log,
error: console.error,
warn: console.warn,
} as unknown as import('@logtape/logtape').Logger;

const TEST_IMAGE_PATH = path.join(__dirname, 'assets/dog.jpg');
const TEST_ACCOUNT_UUID = 'integration-tests';

describe('GCPStorageService Integration', () => {
let service: GCPStorageService;

beforeAll(async () => {
service = new GCPStorageService(logger);
await service.init();
});

describe('saveFile', () => {
it('should save an image file to the bucket and return a valid URL', async () => {
const buffer = readFileSync(TEST_IMAGE_PATH);
const file = new NodeFile([buffer], 'dog.jpg', {
type: 'image/jpeg',
});

const result = await service.saveFile(
file as unknown as File,
TEST_ACCOUNT_UUID,
);

expect(isError(result)).toBe(false);
if (!isError(result)) {
const url = getValue(result);
expect(url).toBeTruthy();
expect(() => new URL(url)).not.toThrow();
expect(url).toContain(TEST_ACCOUNT_UUID);

if (process.env.GCP_STORAGE_EMULATOR_HOST) {
expect(url).toContain(
'localhost:4443/storage/v1/b/activitypub/o/',
);
expect(url).toContain('?alt=media');
} else {
const res = await fetch(url);
expect(res.status).toBe(200);
}
}
});

it('should reject files larger than 5MB', async () => {
// Create a 6MB buffer
const largeBuffer = Buffer.alloc(6 * 1024 * 1024);
const file = new NodeFile([largeBuffer], 'large.jpg', {
type: 'image/jpeg',
});

const result = await service.saveFile(
file as unknown as File,
TEST_ACCOUNT_UUID,
);

expect(isError(result)).toBe(true);
if (isError(result)) {
expect(getError(result)).toBe('file-too-large');
}
});

it('should reject unsupported file types', async () => {
const buffer = readFileSync(TEST_IMAGE_PATH);
const file = new NodeFile([buffer], 'test.gif', {
type: 'image/gif',
});

const result = await service.saveFile(
file as unknown as File,
TEST_ACCOUNT_UUID,
);

expect(isError(result)).toBe(true);
if (isError(result)) {
expect(getError(result)).toBe('file-type-not-supported');
}
});
});

describe('verifyImageUrl', () => {
it('should verify a valid image URL', async () => {
const buffer = readFileSync(TEST_IMAGE_PATH);
const file = new NodeFile([buffer], 'dog.jpg', {
type: 'image/jpeg',
});

const saveResult = await service.saveFile(
file as unknown as File,
TEST_ACCOUNT_UUID,
);

expect(isError(saveResult)).toBe(false);
if (!isError(saveResult)) {
const url = new URL(getValue(saveResult));
const verifyResult = await service.verifyImageUrl(url);
expect(isError(verifyResult)).toBe(false);
if (!isError(verifyResult)) {
expect(getValue(verifyResult)).toBe(true);
}
}
});

it('should reject invalid URLs', async () => {
const invalidUrl = new URL('https://example.com/invalid.jpg');
const result = await service.verifyImageUrl(invalidUrl);

expect(isError(result)).toBe(true);
if (isError(result)) {
expect(getError(result)).toBe('invalid-url');
}
});

it('should reject URLs with invalid file paths', async () => {
const invalidPathUrl = new URL(
'https://storage.googleapis.com/activitypub/invalid/path.jpg',
);
const result = await service.verifyImageUrl(invalidPathUrl);

expect(isError(result)).toBe(true);
if (isError(result)) {
process.env.GCP_STORAGE_EMULATOR_HOST
? expect(getError(result)).toBe('invalid-url')
: expect(getError(result)).toBe('invalid-file-path');
}
});

it('should reject non-existent files', async () => {
const nonExistentUrl = new URL(
`https://storage.googleapis.com/${process.env.GCP_BUCKET_NAME}/images/nonexistent.jpg`,
);
const result = await service.verifyImageUrl(nonExistentUrl);

expect(isError(result)).toBe(true);
if (isError(result)) {
process.env.GCP_STORAGE_EMULATOR_HOST
? expect(getError(result)).toBe('invalid-url')
: expect(getError(result)).toBe('file-not-found');
}
});
});
});
85 changes: 58 additions & 27 deletions src/test/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,25 @@ import { afterAll } from 'vitest';
export async function createTestDb() {
const systemClient = knex({
client: 'mysql2',
connection: {
host: process.env.MYSQL_HOST,
port: Number.parseInt(process.env.MYSQL_PORT!),
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: 'mysql',
timezone: '+00:00',
},
connection: process.env.MYSQL_SOCKET_PATH
? {
socketPath: process.env.MYSQL_SOCKET_PATH,
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: 'mysql',
timezone: '+00:00',
}
: {
host: process.env.MYSQL_HOST,
port: Number.parseInt(process.env.MYSQL_PORT!),
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: 'mysql',
timezone: '+00:00',
},
});

const dbName = `test_${randomBytes(16).toString('hex')}`;
const dbName = `${process.env.MYSQL_DATABASE?.includes('pr-') ? `${process.env.MYSQL_DATABASE.replace(/-/g, '_')}_` : ''}test_${randomBytes(16).toString('hex')}`;

await systemClient.raw(`CREATE DATABASE ${dbName}`);

Expand All @@ -29,37 +37,60 @@ export async function createTestDb() {

// Clone each table structure
for (const { TABLE_NAME } of tables[0]) {
await systemClient.raw(
`CREATE TABLE ${dbName}.${TABLE_NAME} LIKE ${process.env.MYSQL_DATABASE}.${TABLE_NAME}`,
const [createTableResult] = await systemClient.raw(
`SHOW CREATE TABLE \`${process.env.MYSQL_DATABASE}\`.\`${TABLE_NAME}\``,
);
const createTableSql = createTableResult[0]['Create Table']
.replace('CREATE TABLE ', `CREATE TABLE \`${dbName}\`.`)
.split('\n')
.filter((line: string) => !line.trim().startsWith('CONSTRAINT'))
.join('\n')
.replace(/,\n\)/, '\n)'); // clean up trailing comma
await systemClient.raw(createTableSql);
}

await systemClient.destroy();

const dbClient = knex({
client: 'mysql2',
connection: {
host: process.env.MYSQL_HOST,
port: Number.parseInt(process.env.MYSQL_PORT!),
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: dbName,
timezone: '+00:00',
},
connection: process.env.MYSQL_SOCKET_PATH
? {
socketPath: process.env.MYSQL_SOCKET_PATH,
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: dbName,
timezone: '+00:00',
}
: {
host: process.env.MYSQL_HOST,
port: Number.parseInt(process.env.MYSQL_PORT!),
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: dbName,
timezone: '+00:00',
},
});

afterAll(async () => {
await dbClient.destroy();
const systemClient = knex({
client: 'mysql2',
connection: {
host: process.env.MYSQL_HOST,
port: Number.parseInt(process.env.MYSQL_PORT!),
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: 'mysql',
timezone: '+00:00',
},
connection: process.env.MYSQL_SOCKET_PATH
? {
socketPath: process.env.MYSQL_SOCKET_PATH,
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: 'mysql',
timezone: '+00:00',
}
: {
host: process.env.MYSQL_HOST,
port: Number.parseInt(process.env.MYSQL_PORT!),
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: 'mysql',
timezone: '+00:00',
},
});
await systemClient.raw(`DROP DATABASE ${dbName}`);
await systemClient.destroy();
Expand Down
Loading