Skip to content

Commit 9b574a9

Browse files
filiptronicekgeropl
filiptronicek
and
geropl
authored
[server] make gRPC clients viable in non-HTTP/2-compatible environments (#20565)
* [server] make gRPC clients viable in non-HTTP/2-compatible environments Tool: gitpod/catfood.gitpod.cloud * Address review comments Co-authored-by: Gero Posmyk-Leinemann <[email protected]> Tool: gitpod/catfood.gitpod.cloud --------- Co-authored-by: Gero Posmyk-Leinemann <[email protected]>
1 parent 77f3fde commit 9b574a9

File tree

9 files changed

+62
-123
lines changed

9 files changed

+62
-123
lines changed

‎components/dashboard/src/service/public-api.ts

Lines changed: 7 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,8 @@ import { CallOptions, Code, ConnectError, PromiseClient, createPromiseClient } f
1010
import { createConnectTransport } from "@connectrpc/connect-web";
1111
import { Disposable } from "@gitpod/gitpod-protocol";
1212
import { PublicAPIConverter } from "@gitpod/public-api-common/lib/public-api-converter";
13-
import { Project as ProtocolProject } from "@gitpod/gitpod-protocol/lib/teams-projects-protocol";
1413
import { HelloService } from "@gitpod/public-api/lib/gitpod/experimental/v1/dummy_connect";
1514
import { OIDCService } from "@gitpod/public-api/lib/gitpod/experimental/v1/oidc_connect";
16-
import { ProjectsService } from "@gitpod/public-api/lib/gitpod/experimental/v1/projects_connect";
17-
import { Project } from "@gitpod/public-api/lib/gitpod/experimental/v1/projects_pb";
1815
import { TokensService } from "@gitpod/public-api/lib/gitpod/experimental/v1/tokens_connect";
1916
import { OrganizationService } from "@gitpod/public-api/lib/gitpod/v1/organization_connect";
2017
import { WorkspaceService } from "@gitpod/public-api/lib/gitpod/v1/workspace_connect";
@@ -52,10 +49,6 @@ export const converter = new PublicAPIConverter();
5249

5350
export const helloService = createServiceClient(HelloService);
5451
export const personalAccessTokensService = createPromiseClient(TokensService, transport);
55-
/**
56-
* @deprecated use configurationClient instead
57-
*/
58-
export const projectsService = createPromiseClient(ProjectsService, transport);
5952

6053
export const oidcService = createPromiseClient(OIDCService, transport);
6154

@@ -68,7 +61,7 @@ export const organizationClient = createServiceClient(OrganizationService, {
6861
featureFlagSuffix: "organization",
6962
});
7063

71-
// No jsonrcp client for the configuration service as it's only used in new UI of the dashboard
64+
// No jsonrpc client for the configuration service as it's only used in new UI of the dashboard
7265
export const configurationClient = createServiceClient(ConfigurationService);
7366
export const prebuildClient = createServiceClient(PrebuildService, {
7467
client: new JsonRpcPrebuildClient(),
@@ -110,56 +103,6 @@ export const installationClient = createServiceClient(InstallationService, {
110103
featureFlagSuffix: "installation",
111104
});
112105

113-
export async function listAllProjects(opts: { orgId: string }): Promise<ProtocolProject[]> {
114-
let pagination = {
115-
page: 1,
116-
pageSize: 100,
117-
};
118-
119-
const response = await projectsService.listProjects({
120-
teamId: opts.orgId,
121-
pagination,
122-
});
123-
const results = response.projects;
124-
125-
while (results.length < response.totalResults) {
126-
pagination = {
127-
pageSize: 100,
128-
page: 1 + pagination.page,
129-
};
130-
const response = await projectsService.listProjects({
131-
teamId: opts.orgId,
132-
pagination,
133-
});
134-
results.push(...response.projects);
135-
}
136-
137-
return results.map(projectToProtocol);
138-
}
139-
140-
export function projectToProtocol(project: Project): ProtocolProject {
141-
return {
142-
id: project.id,
143-
name: project.name,
144-
cloneUrl: project.cloneUrl,
145-
creationTime: project.creationTime?.toDate().toISOString() || "",
146-
teamId: project.teamId,
147-
appInstallationId: "undefined",
148-
settings: {
149-
workspaceClasses: {
150-
regular: project.settings?.workspace?.workspaceClass?.regular || "",
151-
},
152-
prebuilds: {
153-
enable: project.settings?.prebuild?.enablePrebuilds,
154-
branchStrategy: project.settings?.prebuild?.branchStrategy as any,
155-
branchMatchingPattern: project.settings?.prebuild?.branchMatchingPattern,
156-
prebuildInterval: project.settings?.prebuild?.prebuildInterval,
157-
workspaceClass: project.settings?.prebuild?.workspaceClass,
158-
},
159-
},
160-
};
161-
}
162-
163106
let user: { id: string; email?: string } | undefined;
164107
export function updateUserForExperiments(newUser?: { id: string; email?: string }) {
165108
user = newUser;
@@ -176,10 +119,13 @@ function createServiceClient<T extends ServiceType>(
176119
get(grpcClient, prop) {
177120
const experimentsClient = getExperimentsClient();
178121
// TODO(ak) remove after migration
179-
async function resolveClient(): Promise<PromiseClient<T>> {
122+
async function resolveClient(preferJsonRpc?: boolean): Promise<PromiseClient<T>> {
180123
if (!jsonRpcOptions) {
181124
return grpcClient;
182125
}
126+
if (preferJsonRpc) {
127+
return jsonRpcOptions.client;
128+
}
183129
const featureFlags = [`dashboard_public_api_${jsonRpcOptions.featureFlagSuffix}_enabled`];
184130
const resolvedFlags = await Promise.all(
185131
featureFlags.map((ff) =>
@@ -241,7 +187,8 @@ function createServiceClient<T extends ServiceType>(
241187
}
242188
return (async function* () {
243189
try {
244-
const client = await resolveClient();
190+
// for server streaming, we prefer jsonRPC
191+
const client = await resolveClient(true);
245192
const generator = Reflect.apply(client[prop as any], client, args) as AsyncGenerator<any>;
246193
for await (const item of generator) {
247194
yield item;

‎components/dashboard/src/service/service.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import { sendTrackEvent } from "../Analytics";
3333
export const gitpodHostUrl = new GitpodHostUrl(window.location.toString());
3434

3535
function createGitpodService<C extends GitpodClient, S extends GitpodServer>() {
36-
let host = gitpodHostUrl.asWebsocket().with({ pathname: GitpodServerPath }).withApi();
36+
const host = gitpodHostUrl.asWebsocket().with({ pathname: GitpodServerPath }).withApi();
3737

3838
const connectionProvider = new WebSocketConnectionProvider();
3939
instrumentWebSocketConnection(connectionProvider);

‎components/gitpod-db/src/redis/publisher.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,13 @@ export class RedisPublisher {
2323
constructor(@inject(Redis) private readonly redis: Redis) {}
2424

2525
async publishPrebuildUpdate(update: RedisPrebuildUpdate): Promise<void> {
26-
log.debug("[redis] Publish prebuild udpate invoked.");
26+
log.debug("[redis] Publish prebuild update invoked.");
2727

2828
let err: Error | undefined;
2929
try {
3030
const serialized = JSON.stringify(update);
3131
await this.redis.publish(PrebuildUpdatesChannel, serialized);
32-
log.debug("[redis] Succesfully published prebuild update.", update);
32+
log.debug("[redis] Successfully published prebuild update.", update);
3333
} catch (e) {
3434
err = e;
3535
log.error("[redis] Failed to publish prebuild update.", e, update);
@@ -43,7 +43,7 @@ export class RedisPublisher {
4343
try {
4444
const serialized = JSON.stringify(update);
4545
await this.redis.publish(WorkspaceInstanceUpdatesChannel, serialized);
46-
log.debug("[redis] Succesfully published instance update.", update);
46+
log.debug("[redis] Successfully published instance update.", update);
4747
} catch (e) {
4848
err = e;
4949
log.error("[redis] Failed to publish instance update.", e, update);
@@ -53,13 +53,13 @@ export class RedisPublisher {
5353
}
5454

5555
async publishHeadlessUpdate(update: RedisHeadlessUpdate): Promise<void> {
56-
log.debug("[redis] Publish headless udpate invoked.");
56+
log.debug("[redis] Publish headless update invoked.");
5757

5858
let err: Error | undefined;
5959
try {
6060
const serialized = JSON.stringify(update);
6161
await this.redis.publish(HeadlessUpdatesChannel, serialized);
62-
log.debug("[redis] Succesfully published headless update.", update);
62+
log.debug("[redis] Successfully published headless update.", update);
6363
} catch (e) {
6464
err = e;
6565
log.error("[redis] Failed to publish headless update.", e, update);

‎components/gitpod-protocol/src/redis.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export type RedisPrebuildUpdate = {
2222
prebuildID: string;
2323
workspaceID: string;
2424
projectID: string;
25+
organizationID?: string;
2526
};
2627

2728
export type RedisHeadlessUpdate = {

‎components/server/src/messaging/redis-subscriber.ts

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ export class RedisSubscriber {
6767
let err: Error | undefined;
6868
try {
6969
await this.onMessage(channel, message);
70-
log.debug("[redis] Succesfully handled update", { channel, message });
70+
log.debug("[redis] Successfully handled update", { channel, message });
7171
} catch (e) {
7272
err = e;
7373
log.error("[redis] Failed to handle message from Pub/Sub", e, { channel, message });
@@ -132,7 +132,10 @@ export class RedisSubscriber {
132132
return;
133133
}
134134

135-
const listeners = this.prebuildUpdateListeners.get(update.projectID) || [];
135+
const listeners = this.prebuildUpdateListeners.get(update.projectID) ?? [];
136+
if (update.organizationID) {
137+
listeners.push(...(this.prebuildUpdateListeners.get(update.organizationID) ?? []));
138+
}
136139
if (listeners.length === 0) {
137140
return;
138141
}
@@ -182,10 +185,14 @@ export class RedisSubscriber {
182185
this.disposables.dispose();
183186
}
184187

185-
listenForPrebuildUpdates(projectId: string, listener: PrebuildUpdateListener): Disposable {
188+
listenForProjectPrebuildUpdates(projectId: string, listener: PrebuildUpdateListener): Disposable {
186189
return this.doRegister(projectId, listener, this.prebuildUpdateListeners, "prebuild");
187190
}
188191

192+
listenForOrganizationPrebuildUpdates(organizationId: string, listener: PrebuildUpdateListener): Disposable {
193+
return this.doRegister(organizationId, listener, this.prebuildUpdateListeners, "prebuild");
194+
}
195+
189196
listenForPrebuildUpdatableEvents(listener: HeadlessWorkspaceEventListener): Disposable {
190197
// we're being cheap here in re-using a map where it just needs to be a plain array.
191198
return this.doRegister(UNDEFINED_KEY, listener, this.headlessWorkspaceEventListeners, "prebuild-updatable");

‎components/server/src/prebuilds/prebuild-manager.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ export class PrebuildManager {
100100
await this.auth.checkPermissionOnProject(userId, "read_prebuild", configurationId);
101101
return generateAsyncGenerator<PrebuildWithStatus>((sink) => {
102102
try {
103-
const toDispose = this.subscriber.listenForPrebuildUpdates(configurationId, (_ctx, prebuild) => {
103+
const toDispose = this.subscriber.listenForProjectPrebuildUpdates(configurationId, (_ctx, prebuild) => {
104104
sink.push(prebuild);
105105
});
106106
return () => {

‎components/server/src/workspace/gitpod-server-impl.ts

Lines changed: 34 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -236,30 +236,17 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
236236
log.debug({ userId: this.userID }, "initializeClient");
237237

238238
this.listenForWorkspaceInstanceUpdates();
239-
this.listenForPrebuildUpdates().catch((err) => log.error("error registering for prebuild updates", err));
239+
this.listenForPrebuildUpdates(connectionCtx).catch((err) =>
240+
log.error("error registering for prebuild updates", err),
241+
);
240242
}
241243

242-
private async listenForPrebuildUpdates() {
243-
if (!this.client) {
244-
return;
245-
}
246-
247-
// todo(ft) disable registering for all updates from all projects by default and only listen to updates when the client is explicity interested in them
248-
const disableWebsocketPrebuildUpdates = await getExperimentsClientForBackend().getValueAsync(
249-
"disableWebsocketPrebuildUpdates",
250-
false,
251-
{
252-
gitpodHost: this.config.hostUrl.url.host,
253-
},
254-
);
255-
if (disableWebsocketPrebuildUpdates) {
256-
log.info("ws prebuild updates disabled by feature flag");
244+
private async listenForPrebuildUpdates(ctx?: TraceContext) {
245+
const userId = this.userID;
246+
if (!this.client || !userId) {
257247
return;
258248
}
259249

260-
// 'registering for prebuild updates for all projects this user has access to
261-
const projects = await this.getAccessibleProjects();
262-
263250
const handler = (ctx: TraceContext, update: PrebuildWithStatus) =>
264251
TraceContext.withSpan(
265252
"forwardPrebuildUpdateToClient",
@@ -273,38 +260,30 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
273260
);
274261

275262
if (!this.disposables.disposed) {
276-
for (const project of projects) {
277-
this.disposables.push(this.subscriber.listenForPrebuildUpdates(project.id, handler));
278-
}
279-
}
280-
281-
// TODO(at) we need to keep the list of accessible project up to date
282-
}
283-
284-
private async getAccessibleProjects() {
285-
const userId = this.userID;
286-
if (!userId) {
287-
return [];
263+
await runWithRequestContext(
264+
{
265+
requestKind: "gitpod-server-impl-listener",
266+
requestMethod: "listenForPrebuildUpdates",
267+
signal: new AbortController().signal,
268+
subjectId: SubjectId.fromUserId(userId),
269+
},
270+
async () => {
271+
const organizations = await this.getTeams(ctx ?? {});
272+
for (const organization of organizations) {
273+
const hasPermission = await this.auth.hasPermissionOnOrganization(
274+
userId,
275+
"read_prebuild",
276+
organization.id,
277+
);
278+
if (hasPermission) {
279+
this.disposables.push(
280+
this.subscriber.listenForOrganizationPrebuildUpdates(organization.id, handler),
281+
);
282+
}
283+
}
284+
},
285+
);
288286
}
289-
290-
// update all project this user has access to
291-
// gpl: This call to runWithRequestContext is not nice, but it's only there to please the old impl for a limited time, so it's fine.
292-
return runWithRequestContext(
293-
{
294-
requestKind: "gitpod-server-impl-listener",
295-
requestMethod: "getAccessibleProjects",
296-
signal: new AbortController().signal,
297-
subjectId: SubjectId.fromUserId(userId),
298-
},
299-
async () => {
300-
const allProjects: Project[] = [];
301-
const teams = await this.organizationService.listOrganizationsByMember(userId, userId);
302-
for (const team of teams) {
303-
allProjects.push(...(await this.projectsService.getProjects(userId, team.id)));
304-
}
305-
return allProjects;
306-
},
307-
);
308287
}
309288

310289
private listenForWorkspaceInstanceUpdates(): void {
@@ -497,9 +476,9 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
497476

498477
/**
499478
* Returns the descriptions of auth providers. This also controls the visibility of
500-
* auth providers on the dashbard.
479+
* auth providers on the dashboard.
501480
*
502-
* If this call is unauthenticated (i.e. for anonumous users,) it returns only information
481+
* If this call is unauthenticated (i.e. for anonymous users,) it returns only information
503482
* necessary for the Login page.
504483
*
505484
* If there are built-in auth providers configured, only these are returned.
@@ -1701,7 +1680,9 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
17011680
ctx,
17021681
);
17031682

1704-
this.disposables.pushAll([this.subscriber.listenForPrebuildUpdates(project.id, prebuildUpdateHandler)]);
1683+
this.disposables.pushAll([
1684+
this.subscriber.listenForProjectPrebuildUpdates(project.id, prebuildUpdateHandler),
1685+
]);
17051686
}
17061687

17071688
return project;

‎components/server/src/workspace/workspace-starter.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -914,6 +914,7 @@ export class WorkspaceStarter {
914914
prebuildID: prebuild.id,
915915
projectID: prebuild.projectId,
916916
workspaceID: workspace.id,
917+
organizationID: workspace.organizationId,
917918
});
918919
}
919920
}

‎components/ws-manager-bridge/src/prebuild-updater.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ export class PrebuildUpdater {
9898
prebuildID: updatedPrebuild.id,
9999
status: updatedPrebuild.state,
100100
workspaceID: workspaceId,
101+
organizationID: info.teamId,
101102
});
102103
}
103104
}
@@ -127,6 +128,7 @@ export class PrebuildUpdater {
127128
prebuildID: prebuild.id,
128129
status: prebuild.state,
129130
workspaceID: instance.workspaceId,
131+
organizationID: info.teamId,
130132
});
131133
}
132134
}

0 commit comments

Comments
 (0)