Skip to content

Commit 7328214

Browse files
geroplgeropl
authored
[server] Introduce job CapGitStatus (#19814)
* [server} Introduce job CapGitStatus * [server] Have runners report unitsOfWork * fix query
1 parent 4fb0c38 commit 7328214

15 files changed

+153
-24
lines changed

‎components/gitpod-db/src/periodic-deleter.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ export class PeriodicDbDeleter {
1515
@inject(GitpodTableDescriptionProvider) protected readonly tableProvider: GitpodTableDescriptionProvider;
1616
@inject(TypeORM) protected readonly typeORM: TypeORM;
1717

18-
async runOnce() {
18+
async runOnce(): Promise<number> {
1919
const tickID = new Date().toISOString();
2020
log.info("[PeriodicDbDeleter] Starting to collect deleted rows.", {
2121
periodicDeleterTickId: tickID,
@@ -52,6 +52,7 @@ export class PeriodicDbDeleter {
5252
log.info("[PeriodicDbDeleter] Finished deleting records.", {
5353
periodicDeleterTickId: tickID,
5454
});
55+
return pendingDeletions.length;
5556
}
5657

5758
protected async collectRowsToBeDeleted(table: TableDescription): Promise<{ table: string; deletions: string[] }> {

‎components/server/src/authorization/relationship-updater-job.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export class RelationshipUpdateJob implements Job {
2020
public name = "relationship-update-job";
2121
public frequencyMs = 1000 * 60 * 3; // 3m
2222

23-
public async run(): Promise<void> {
23+
public async run(): Promise<number | undefined> {
2424
try {
2525
const ids = await this.userDB.findUserIdsNotYetMigratedToFgaVersion(RelationshipUpdater.version, 50);
2626
const now = Date.now();
@@ -40,6 +40,7 @@ export class RelationshipUpdateJob implements Job {
4040
}
4141
}
4242
log.info(this.name + ": updated " + migrated + " users in " + (Date.now() - now) + "ms");
43+
return migrated;
4344
} catch (error) {
4445
log.error(this.name + ": error running relationship update job", error);
4546
}

‎components/server/src/container-module.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ import { ContextService } from "./workspace/context-service";
133133
import { RateLimitter } from "./rate-limitter";
134134
import { AnalyticsController } from "./analytics-controller";
135135
import { InstallationAdminCleanup } from "./jobs/installation-admin-cleanup";
136+
import { CapGitStatus } from "./jobs/cap-git-status";
136137

137138
export const productionContainerModule = new ContainerModule(
138139
(bind, unbind, isBound, rebind, unbindAsync, onActivation, onDeactivation) => {
@@ -373,6 +374,7 @@ export const productionContainerModule = new ContainerModule(
373374
bind(JobRunner).toSelf().inSingletonScope();
374375
bind(RelationshipUpdateJob).toSelf().inSingletonScope();
375376
bind(InstallationAdminCleanup).toSelf().inSingletonScope();
377+
bind(CapGitStatus).toSelf().inSingletonScope();
376378

377379
// Redis
378380
bind(Redis).toDynamicValue((ctx) => {

‎components/server/src/jobs/cap-git-status.ts

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/**
2+
* Copyright (c) 2024 Gitpod GmbH. All rights reserved.
3+
* Licensed under the GNU Affero General Public License (AGPL).
4+
* See License.AGPL.txt in the project root for license information.
5+
*/
6+
7+
import { DBWorkspaceInstance, WorkspaceDB } from "@gitpod/gitpod-db/lib";
8+
import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
9+
import { inject, injectable } from "inversify";
10+
import { Job } from "./runner";
11+
import { Config } from "../config";
12+
import { GIT_STATUS_LENGTH_CAP_BYTES } from "../workspace/workspace-service";
13+
import { Repository } from "typeorm";
14+
import { WorkspaceInstance, WorkspaceInstanceRepoStatus } from "@gitpod/gitpod-protocol";
15+
import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
16+
17+
@injectable()
18+
export class CapGitStatus implements Job {
19+
@inject(Config) protected readonly config: Config;
20+
@inject(WorkspaceDB) protected readonly workspaceDb: WorkspaceDB;
21+
22+
public name = "git-status-capper";
23+
public frequencyMs = 2 * 60 * 1000; // every 2 minutes
24+
25+
public async run(): Promise<number | undefined> {
26+
log.info("git-status: we're leading the quorum.");
27+
28+
const validateGitStatusLength = await getExperimentsClientForBackend().getValueAsync(
29+
"api_validate_git_status_length",
30+
false,
31+
{},
32+
);
33+
if (!validateGitStatusLength) {
34+
log.info("git-status: feature flag is not enabled.");
35+
return;
36+
}
37+
38+
const limit = 100;
39+
const instancesCapped = await this.workspaceDb.transaction(async (db) => {
40+
const repo = await ((db as any).getWorkspaceInstanceRepo() as Promise<Repository<DBWorkspaceInstance>>);
41+
const instances = await this.findInstancesWithLengthyGitStatus(repo, GIT_STATUS_LENGTH_CAP_BYTES, limit);
42+
if (instances.length === 0) {
43+
return 0;
44+
}
45+
46+
// Cap the git status (incl. status.repo, the old place where we stored it before)
47+
const MARGIN = 200;
48+
instances.forEach((i) => {
49+
if (i.gitStatus) {
50+
i.gitStatus = capGitStatus(i.gitStatus, GIT_STATUS_LENGTH_CAP_BYTES - MARGIN);
51+
}
52+
if (i.status) {
53+
delete (i.status as any).repo;
54+
}
55+
});
56+
57+
// In order to effectively cap the storage size, we have to delete and re-inser the instance.
58+
// Thank you, MySQL! -.-
59+
await repo.delete(instances.map((i) => i.id));
60+
await repo.save(instances);
61+
62+
return instances.length;
63+
});
64+
65+
log.info(`git-status: capped ${instancesCapped} instances.`);
66+
return instancesCapped;
67+
}
68+
69+
async findInstancesWithLengthyGitStatus(
70+
repo: Repository<DBWorkspaceInstance>,
71+
byteLimit: number,
72+
limit: number = 1000,
73+
): Promise<WorkspaceInstance[]> {
74+
const qb = repo
75+
.createQueryBuilder("wsi")
76+
.where("JSON_STORAGE_SIZE(wsi.gitStatus) > :byteLimit", { byteLimit })
77+
.orWhere("JSON_STORAGE_SIZE(wsi.status) > :byteLimit", { byteLimit })
78+
.limit(limit);
79+
return qb.getMany();
80+
}
81+
}
82+
83+
function capGitStatus(gitStatus: WorkspaceInstanceRepoStatus, maxLength: number): WorkspaceInstanceRepoStatus {
84+
let bytesUsed = 0;
85+
function capStr(str: string | undefined): string | undefined {
86+
if (str === undefined) {
87+
return undefined;
88+
}
89+
90+
const len = Buffer.byteLength(str, "utf8");
91+
if (bytesUsed + len > maxLength) {
92+
return "";
93+
}
94+
bytesUsed = bytesUsed + len;
95+
return str;
96+
}
97+
function filterStr(str: string | undefined): boolean {
98+
return !!capStr(str);
99+
}
100+
101+
gitStatus.branch = capStr(gitStatus.branch);
102+
gitStatus.latestCommit = capStr(gitStatus.latestCommit);
103+
gitStatus.uncommitedFiles = gitStatus.uncommitedFiles?.filter(filterStr);
104+
gitStatus.untrackedFiles = gitStatus.untrackedFiles?.filter(filterStr);
105+
gitStatus.unpushedCommits = gitStatus.unpushedCommits?.filter(filterStr);
106+
107+
return gitStatus;
108+
}

‎components/server/src/jobs/database-gc.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@ export class DatabaseGarbageCollector implements Job {
1818
public name = "database-gc";
1919
public frequencyMs = 30000; // every 30 seconds
2020

21-
public async run(): Promise<void> {
21+
public async run(): Promise<number | undefined> {
2222
if (!this.config.runDbDeleter) {
2323
log.info("database-gc: deleter is disabled");
2424
return;
2525
}
2626

2727
try {
28-
await this.periodicDbDeleter.runOnce();
28+
return await this.periodicDbDeleter.runOnce();
2929
} catch (err) {
3030
log.error("database-gc: error during run", err);
3131
throw err;

‎components/server/src/jobs/installation-admin-cleanup.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ export class InstallationAdminCleanup implements Job {
1616
public name = "installation-admin-cleanup";
1717
public frequencyMs = 5 * 60 * 1000; // every 5 minutes
1818

19-
public async run(): Promise<void> {
19+
public async run(): Promise<number | undefined> {
2020
try {
2121
const installationAdmin = await this.userDb.findUserById(BUILTIN_INSTLLATION_ADMIN_USER_ID);
2222
if (!installationAdmin) {
@@ -33,6 +33,8 @@ export class InstallationAdminCleanup implements Job {
3333
await this.userDb.storeUser(installationAdmin);
3434
log.info("Cleaned up SCM connections of installation admin.");
3535
}
36+
37+
return undefined;
3638
} catch (err) {
3739
log.error("Failed to clean up SCM connections of installation admin.", err);
3840
throw err;

‎components/server/src/jobs/ots-gc.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@ export class OTSGarbageCollector implements Job {
1616
public name = "ots-gc";
1717
public frequencyMs = 5 * 60 * 1000; // every 5 minutes
1818

19-
public async run(): Promise<void> {
19+
public async run(): Promise<number | undefined> {
2020
try {
2121
await this.oneTimeSecretDB.trace({}).pruneExpired();
22+
23+
return undefined;
2224
} catch (err) {
2325
log.error("Failed to garbage collect OTS", err);
2426
throw err;

‎components/server/src/jobs/runner.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,15 @@ import { WorkspaceStartController } from "../workspace/workspace-start-controlle
2121
import { runWithRequestContext } from "../util/request-context";
2222
import { SYSTEM_USER } from "../authorization/authorizer";
2323
import { InstallationAdminCleanup } from "./installation-admin-cleanup";
24+
import { CapGitStatus } from "./cap-git-status";
2425

2526
export const Job = Symbol("Job");
2627

2728
export interface Job {
2829
readonly name: string;
2930
readonly frequencyMs: number;
3031
readonly lockedResources?: string[];
31-
run: () => Promise<void>;
32+
run: () => Promise<number | undefined>;
3233
}
3334

3435
@injectable()
@@ -44,6 +45,7 @@ export class JobRunner {
4445
@inject(RelationshipUpdateJob) private readonly relationshipUpdateJob: RelationshipUpdateJob,
4546
@inject(WorkspaceStartController) private readonly workspaceStartController: WorkspaceStartController,
4647
@inject(InstallationAdminCleanup) private readonly installationAdminCleanup: InstallationAdminCleanup,
48+
@inject(CapGitStatus) private readonly capGitStatus: CapGitStatus,
4749
) {}
4850

4951
public start(): DisposableCollection {
@@ -59,6 +61,7 @@ export class JobRunner {
5961
this.relationshipUpdateJob,
6062
this.workspaceStartController,
6163
this.installationAdminCleanup,
64+
this.capGitStatus,
6265
];
6366

6467
for (const job of jobs) {
@@ -99,12 +102,12 @@ export class JobRunner {
99102
reportJobStarted(job.name);
100103
const now = new Date().getTime();
101104
try {
102-
await job.run();
105+
const unitsOfWork = await job.run();
103106
log.debug(`Successfully finished job ${job.name}`, {
104107
...logCtx,
105108
jobTookSec: `${(new Date().getTime() - now) / 1000}s`,
106109
});
107-
reportJobCompleted(job.name, true);
110+
reportJobCompleted(job.name, true, unitsOfWork);
108111
} catch (err) {
109112
log.error(`Error while running job ${job.name}`, err, {
110113
...logCtx,

‎components/server/src/jobs/snapshots.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export class SnapshotsJob implements Job {
2020
public name = "snapshots";
2121
public frequencyMs = 5 * 60 * 1000; // every 5 minutes
2222

23-
public async run(): Promise<void> {
23+
public async run(): Promise<number | undefined> {
2424
if (this.config.completeSnapshotJob?.disabled) {
2525
log.info("snapshots: Snapshot completion job is disabled.");
2626
return;
@@ -52,5 +52,7 @@ export class SnapshotsJob implements Job {
5252
.driveSnapshot({ workspaceOwner: workspace.ownerId, snapshot })
5353
.catch((err) => log.error("driveSnapshot", err));
5454
}
55+
56+
return pendingSnapshots.length;
5557
}
5658
}

‎components/server/src/jobs/token-gc.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@ export class TokenGarbageCollector implements Job {
2222
public name = "token-gc";
2323
public frequencyMs = 5 * 60 * 1000; // every 5 minutes
2424

25-
public async run(): Promise<void> {
25+
public async run(): Promise<number | undefined> {
2626
const span = opentracing.globalTracer().startSpan("collectExpiredTokenEntries");
2727
log.debug("token-gc: start collecting...");
2828
try {
2929
await this.userDb.deleteExpiredTokenEntries(new Date().toISOString());
3030
log.debug("token-gc: done collecting.");
31+
32+
return undefined;
3133
} catch (err) {
3234
TraceContext.setError({ span }, err);
3335
log.error("token-gc: error collecting expired tokens: ", err);

‎components/server/src/jobs/webhook-gc.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,15 @@ export class WebhookEventGarbageCollector implements Job {
1919
public name = "webhook-gc";
2020
public frequencyMs = 4 * 60 * 1000; // every 4 minutes
2121

22-
public async run(): Promise<void> {
22+
public async run(): Promise<number | undefined> {
2323
const span = opentracing.globalTracer().startSpan("collectObsoleteWebhookEvents");
2424
log.debug("webhook-event-gc: start collecting...");
2525

2626
try {
2727
await this.db.deleteOldEvents(10 /* days */, 600 /* limit per run */);
2828
log.debug("webhook-event-gc: done collecting.");
29+
30+
return undefined;
2931
} catch (err) {
3032
TraceContext.setError({ span }, err);
3133
log.error("webhook-event-gc: error collecting webhook events: ", err);

‎components/server/src/jobs/workspace-gc.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ export class WorkspaceGarbageCollector implements Job {
4343
this.frequencyMs = this.config.workspaceGarbageCollection.intervalSeconds * 1000;
4444
}
4545

46-
public async run(): Promise<void> {
46+
public async run(): Promise<number | undefined> {
4747
if (this.config.workspaceGarbageCollection.disabled) {
4848
log.info("workspace-gc: Garbage collection disabled.");
4949
return;
@@ -69,6 +69,8 @@ export class WorkspaceGarbageCollector implements Job {
6969
} catch (err) {
7070
log.error("workspace-gc: error during prebuild deletion", err);
7171
}
72+
73+
return undefined;
7274
}
7375

7476
/**

‎components/server/src/prometheus-metrics.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -280,11 +280,11 @@ export function reportJobStarted(name: string) {
280280
export const jobsCompletedTotal = new prometheusClient.Counter({
281281
name: "gitpod_server_jobs_completed_total",
282282
help: "Total number of errors caught by an error boundary in the dashboard",
283-
labelNames: ["name", "success"],
283+
labelNames: ["name", "success", "unitsOfWork"],
284284
});
285285

286-
export function reportJobCompleted(name: string, success: boolean) {
287-
jobsCompletedTotal.inc({ name, success: String(success) });
286+
export function reportJobCompleted(name: string, success: boolean, unitsOfWork?: number | undefined) {
287+
jobsCompletedTotal.inc({ name, success: String(success), unitsOfWork: String(unitsOfWork) });
288288
}
289289

290290
export const jobsDurationSeconds = new prometheusClient.Histogram({

‎components/server/src/workspace/workspace-service.ts

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ import { InstallationService } from "../auth/installation-service";
7676
import { PublicAPIConverter } from "@gitpod/public-api-common/lib/public-api-converter";
7777
import { WatchWorkspaceStatusResponse } from "@gitpod/public-api/lib/gitpod/v1/workspace_pb";
7878

79+
export const GIT_STATUS_LENGTH_CAP_BYTES = 4096;
80+
7981
export interface StartWorkspaceOptions extends StarterStartWorkspaceOptions {
8082
/**
8183
* This field is used to guess the workspace location using the RegionService
@@ -726,7 +728,7 @@ export class WorkspaceService {
726728
},
727729
);
728730
if (validateGitStatusLength) {
729-
this.validateGitStatusLength(gitStatus);
731+
this.validateGitStatusLength(gitStatus, GIT_STATUS_LENGTH_CAP_BYTES);
730732
}
731733
}
732734

@@ -744,16 +746,13 @@ export class WorkspaceService {
744746
});
745747
}
746748

747-
protected validateGitStatusLength(gitStatus: Required<WorkspaceInstanceRepoStatus>) {
748-
/** [bytes] */
749-
const MAX_GIT_STATUS_LENGTH = 4096;
750-
749+
protected validateGitStatusLength(gitStatus: Required<WorkspaceInstanceRepoStatus>, maxByteLength: number) {
751750
try {
752751
const s = JSON.stringify(gitStatus);
753-
if (Buffer.byteLength(s, "utf8") > MAX_GIT_STATUS_LENGTH) {
752+
if (Buffer.byteLength(s, "utf8") > maxByteLength) {
754753
throw new ApplicationError(
755754
ErrorCodes.BAD_REQUEST,
756-
`gitStatus too long, maximum is ${MAX_GIT_STATUS_LENGTH} bytes`,
755+
`gitStatus too long, maximum is ${maxByteLength} bytes`,
757756
);
758757
}
759758
} catch (err) {

0 commit comments

Comments
 (0)