Skip to content

Commit c6f5e9a

Browse files
fix: ensure that we capture service worker requests (#28517)
* fix: ensure that we capture service worker requests * add changelog * fix changelog * fix tests * PR comments * PR comments * PR comment * PR comment * update changelog * Update cli/CHANGELOG.md Co-authored-by: Mike McCready <[email protected]> * enable builds on all archs * fix permission issue * PR comments * Update smoke.js * Update cli/CHANGELOG.md * attempt to fix smoke tests * bump ci cache * Update smoke.js * Update smoke.js * Update example.json * fix multiple specs * fix tests * Update CHANGELOG.md --------- Co-authored-by: Mike McCready <[email protected]>
1 parent 7a9e3a4 commit c6f5e9a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1668
-98
lines changed

.circleci/workflows.yml

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,7 @@ macWorkflowFilters: &darwin-workflow-filters
4343
# use the following branch as well to ensure that v8 snapshot cache updates are fully tested
4444
- equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
4545
- equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
46-
- equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
47-
- equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
46+
- equal: [ 'ryanm/fix/service-worker-capture', << pipeline.git.branch >> ]
4847
- matches:
4948
pattern: /^release\/\d+\.\d+\.\d+$/
5049
value: << pipeline.git.branch >>
@@ -56,8 +55,7 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
5655
# use the following branch as well to ensure that v8 snapshot cache updates are fully tested
5756
- equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
5857
- equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
59-
- equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
60-
- equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
58+
- equal: [ 'ryanm/fix/service-worker-capture', << pipeline.git.branch >> ]
6159
- matches:
6260
pattern: /^release\/\d+\.\d+\.\d+$/
6361
value: << pipeline.git.branch >>
@@ -81,10 +79,7 @@ windowsWorkflowFilters: &windows-workflow-filters
8179
# use the following branch as well to ensure that v8 snapshot cache updates are fully tested
8280
- equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
8381
- equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
84-
- equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
85-
- equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
86-
- equal: [ 'em/shallow-checkout', << pipeline.git.branch >> ]
87-
- equal: [ 'mschile/mochaEvents_win_sep', << pipeline.git.branch >> ]
82+
- equal: [ 'ryanm/fix/service-worker-capture', << pipeline.git.branch >> ]
8883
- matches:
8984
pattern: /^release\/\d+\.\d+\.\d+$/
9085
value: << pipeline.git.branch >>
@@ -154,7 +149,7 @@ commands:
154149
name: Set environment variable to determine whether or not to persist artifacts
155150
command: |
156151
echo "Setting SHOULD_PERSIST_ARTIFACTS variable"
157-
echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "em/protocol-log-false" ]]; then
152+
echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "ryanm/fix/service-worker-capture" ]]; then
158153
export SHOULD_PERSIST_ARTIFACTS=true
159154
fi' >> "$BASH_ENV"
160155
# You must run `setup_should_persist_artifacts` command and be using bash before running this command

cli/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ _Released 1/16/2024 (PENDING)_
1010
- When generating assertions via Cypress Studio, the preview of the generated assertions now correctly displays the past tense of 'expected' instead of 'expect'. Fixed in [#28593](https://github.com/cypress-io/cypress/pull/28593).
1111
- Fixed a regression in [`13.6.2`](https://docs.cypress.io/guides/references/changelog/13.6.2) where the `body` element was not highlighted correctly in Test Replay. Fixed in [#28627](https://github.com/cypress-io/cypress/pull/28627).
1212
- Fixed an issue where some cross-origin logs, like assertions or cy.clock(), were getting too many dom snapshots. Fixes [#28609](https://github.com/cypress-io/cypress/issues/28609).
13+
- Fixed asset capture for Test Replay for requests that are routed through service workers. This addresses an issue where styles were not being applied properly in Test Replay and `cy.intercept` was not working properly for requests in this scenario. Fixes [#28516](https://github.com/cypress-io/cypress/issues/28516).
1314

1415
**Performance:**
1516

packages/proxy/lib/http/index.ts

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,19 @@ import type { RemoteStates } from '@packages/server/lib/remote_states'
2626
import type { CookieJar, SerializableAutomationCookie } from '@packages/server/lib/util/cookies'
2727
import type { ResourceTypeAndCredentialManager } from '@packages/server/lib/util/resourceTypeAndCredentialManager'
2828
import type { ProtocolManagerShape } from '@packages/types'
29+
import type Protocol from 'devtools-protocol'
30+
import { ServiceWorkerManager } from './util/service-worker-manager'
2931

3032
function getRandomColorFn () {
3133
return chalk.hex(`#${Number(
3234
Math.floor(Math.random() * 0xFFFFFF),
3335
).toString(16).padStart(6, 'F').toUpperCase()}`)
3436
}
3537

38+
const hasServiceWorkerHeader = (headers: Record<string, string | string[] | undefined>) => {
39+
return headers?.['service-worker'] === 'script' || headers?.['Service-Worker'] === 'script'
40+
}
41+
3642
export const isVerboseTelemetry = true
3743

3844
const isVerbose = isVerboseTelemetry
@@ -273,6 +279,7 @@ export class Http {
273279
autUrl?: string
274280
getCookieJar: () => CookieJar
275281
protocolManager?: ProtocolManagerShape
282+
serviceWorkerManager: ServiceWorkerManager = new ServiceWorkerManager()
276283

277284
constructor (opts: ServerCtx & { middleware?: HttpMiddlewareStacks }) {
278285
this.buffers = new HttpBuffers()
@@ -332,6 +339,18 @@ export class Http {
332339
getAUTUrl: this.getAUTUrl,
333340
setAUTUrl: this.setAUTUrl,
334341
getPreRequest: (cb) => {
342+
// The initial request that loads the service worker does not always get sent to CDP. Thus, we need to explicitly ignore it. We determine
343+
// it's the service worker request via the `service-worker` header
344+
if (hasServiceWorkerHeader(req.headers)) {
345+
ctx.debug('Ignoring service worker script since we are not guaranteed to receive it', req.proxiedUrl)
346+
347+
cb({
348+
noPreRequestExpected: true,
349+
})
350+
351+
return
352+
}
353+
335354
return this.preRequests.get(ctx.req, ctx.debug, cb)
336355
},
337356
addPendingUrlWithoutPreRequest: (url) => {
@@ -429,20 +448,28 @@ export class Http {
429448
}
430449
}
431450

432-
reset (options: { resetPreRequests: boolean }) {
451+
reset (options: { resetPreRequests: boolean, resetBetweenSpecs: boolean }) {
433452
this.buffers.reset()
434453
this.setAUTUrl(undefined)
435454

436455
if (options.resetPreRequests) {
437456
this.preRequests.reset()
438457
}
458+
459+
if (options.resetBetweenSpecs) {
460+
this.serviceWorkerManager = new ServiceWorkerManager()
461+
}
439462
}
440463

441464
setBuffer (buffer) {
442465
return this.buffers.set(buffer)
443466
}
444467

445468
addPendingBrowserPreRequest (browserPreRequest: BrowserPreRequest) {
469+
if (this.shouldIgnorePendingRequest(browserPreRequest)) {
470+
return
471+
}
472+
446473
this.preRequests.addPending(browserPreRequest)
447474
}
448475

@@ -454,6 +481,18 @@ export class Http {
454481
this.preRequests.addPendingUrlWithoutPreRequest(url)
455482
}
456483

484+
updateServiceWorkerRegistrations (data: Protocol.ServiceWorker.WorkerRegistrationUpdatedEvent) {
485+
this.serviceWorkerManager.updateServiceWorkerRegistrations(data)
486+
}
487+
488+
updateServiceWorkerVersions (data: Protocol.ServiceWorker.WorkerVersionUpdatedEvent) {
489+
this.serviceWorkerManager.updateServiceWorkerVersions(data)
490+
}
491+
492+
updateServiceWorkerClientSideRegistrations (data: { scriptURL: string, initiatorURL: string }) {
493+
this.serviceWorkerManager.addInitiatorToServiceWorker({ scriptURL: data.scriptURL, initiatorURL: data.initiatorURL })
494+
}
495+
457496
setProtocolManager (protocolManager: ProtocolManagerShape) {
458497
this.protocolManager = protocolManager
459498
this.preRequests.setProtocolManager(protocolManager)
@@ -462,4 +501,23 @@ export class Http {
462501
setPreRequestTimeout (timeout: number) {
463502
this.preRequests.setPreRequestTimeout(timeout)
464503
}
504+
505+
private shouldIgnorePendingRequest (browserPreRequest: BrowserPreRequest) {
506+
// The initial request that loads the service worker does not always get sent to CDP. If it does, we want it to not clog up either the prerequests
507+
// or pending requests. Thus, we need to explicitly ignore it here and in `get`. We determine it's the service worker request via the
508+
// `service-worker` header
509+
if (hasServiceWorkerHeader(browserPreRequest.headers)) {
510+
debugVerbose('Ignoring service worker script since we are not guaranteed to receive it: %o', browserPreRequest)
511+
512+
return true
513+
}
514+
515+
if (this.serviceWorkerManager.processBrowserPreRequest(browserPreRequest)) {
516+
debugVerbose('Not correlating request since it is fully controlled by the service worker and the correlation will happen within the service worker: %o', browserPreRequest)
517+
518+
return true
519+
}
520+
521+
return false
522+
}
465523
}

packages/proxy/lib/http/util/prerequests.ts

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,9 @@ export class PreRequests {
157157
}
158158

159159
addPending (browserPreRequest: BrowserPreRequest) {
160-
metrics.browserPreRequestsReceived++
161160
const key = `${browserPreRequest.method}-${tryDecodeURI(browserPreRequest.url)}`
161+
162+
metrics.browserPreRequestsReceived++
162163
const pendingRequest = this.pendingRequests.shift(key)
163164

164165
if (pendingRequest) {
@@ -230,19 +231,6 @@ export class PreRequests {
230231
}
231232

232233
get (req: CypressIncomingRequest, ctxDebug, callback: GetPreRequestCb) {
233-
// The initial request that loads the service worker does not get sent to CDP and it happens prior
234-
// to the service worker target being added. Thus, we need to explicitly ignore it. We determine
235-
// it's the service worker request via the `sec-fetch-dest` header
236-
if (req.headers['sec-fetch-dest'] === 'serviceworker') {
237-
ctxDebug('Ignoring request with sec-fetch-dest: serviceworker', req.proxiedUrl)
238-
239-
callback({
240-
noPreRequestExpected: true,
241-
})
242-
243-
return
244-
}
245-
246234
const proxyRequestReceivedTimestamp = performance.now() + performance.timeOrigin
247235

248236
metrics.proxyRequestsReceived++
@@ -252,6 +240,7 @@ export class PreRequests {
252240
if (pendingPreRequest) {
253241
metrics.immediatelyMatchedRequests++
254242
ctxDebug('Incoming request %s matches known pre-request: %o', key, pendingPreRequest)
243+
255244
callback({
256245
browserPreRequest: {
257246
...pendingPreRequest.browserPreRequest,
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
import Debug from 'debug'
2+
import type { BrowserPreRequest } from '../../types'
3+
import type Protocol from 'devtools-protocol'
4+
5+
const debug = Debug('cypress:proxy:service-worker-manager')
6+
7+
type ServiceWorkerRegistration = {
8+
registrationId: string
9+
scopeURL: string
10+
activatedServiceWorker?: ServiceWorker
11+
}
12+
13+
type ServiceWorker = {
14+
registrationId: string
15+
scriptURL: string
16+
initiatorURL?: string
17+
controlledURLs: Set<string>
18+
}
19+
20+
type RegisterServiceWorkerOptions = {
21+
registrationId: string
22+
scopeURL: string
23+
}
24+
25+
type UnregisterServiceWorkerOptions = {
26+
registrationId: string
27+
}
28+
29+
type AddActivatedServiceWorkerOptions = {
30+
registrationId: string
31+
scriptURL: string
32+
}
33+
34+
type AddInitiatorToServiceWorkerOptions = {
35+
scriptURL: string
36+
initiatorURL: string
37+
}
38+
39+
/**
40+
* Manages service worker registrations and their controlled URLs.
41+
*
42+
* The basic lifecycle is as follows:
43+
*
44+
* 1. A service worker is registered via `registerServiceWorker`.
45+
* 2. The service worker is activated via `addActivatedServiceWorker`.
46+
*
47+
* At some point while 1 and 2 are happening:
48+
*
49+
* 3. We receive a message from the browser that a service worker has been initiated with the `addInitiatorToServiceWorker` method.
50+
*
51+
* At this point, when the manager tries to process a browser pre-request, it will check if the request is controlled by a service worker.
52+
* It determines it is controlled by a service worker if:
53+
*
54+
* 1. The document URL for the browser pre-request matches the initiator URL for the service worker.
55+
* 2. The request URL is within the scope of the service worker or the request URL's initiator is controlled by the service worker.
56+
*/
57+
export class ServiceWorkerManager {
58+
private serviceWorkerRegistrations: Map<string, ServiceWorkerRegistration> = new Map<string, ServiceWorkerRegistration>()
59+
private pendingInitiators: Map<string, string> = new Map<string, string>()
60+
61+
/**
62+
* Goes through the list of service worker registrations and adds or removes them from the manager.
63+
*/
64+
updateServiceWorkerRegistrations (data: Protocol.ServiceWorker.WorkerRegistrationUpdatedEvent) {
65+
data.registrations.forEach((registration) => {
66+
if (registration.isDeleted) {
67+
this.unregisterServiceWorker({ registrationId: registration.registrationId })
68+
} else {
69+
this.registerServiceWorker({ registrationId: registration.registrationId, scopeURL: registration.scopeURL })
70+
}
71+
})
72+
}
73+
74+
/**
75+
* Goes through the list of service worker versions and adds any that are activated to the manager.
76+
*/
77+
updateServiceWorkerVersions (data: Protocol.ServiceWorker.WorkerVersionUpdatedEvent) {
78+
data.versions.forEach((version) => {
79+
if (version.status === 'activated') {
80+
this.addActivatedServiceWorker({ registrationId: version.registrationId, scriptURL: version.scriptURL })
81+
}
82+
})
83+
}
84+
85+
/**
86+
* Adds an initiator URL to a service worker. If the service worker has not yet been activated, the initiator URL is added to a pending list and will
87+
* be added to the service worker when it is activated.
88+
*/
89+
addInitiatorToServiceWorker ({ scriptURL, initiatorURL }: AddInitiatorToServiceWorkerOptions) {
90+
let initiatorAdded = false
91+
92+
for (const registration of this.serviceWorkerRegistrations.values()) {
93+
if (registration.activatedServiceWorker?.scriptURL === scriptURL) {
94+
registration.activatedServiceWorker.initiatorURL = initiatorURL
95+
96+
initiatorAdded = true
97+
break
98+
}
99+
}
100+
101+
if (!initiatorAdded) {
102+
this.pendingInitiators.set(scriptURL, initiatorURL)
103+
}
104+
}
105+
106+
/**
107+
* Processes a browser pre-request to determine if it is controlled by a service worker. If it is, the service worker's controlled URLs are updated with the given request URL.
108+
*
109+
* @param browserPreRequest The browser pre-request to process.
110+
* @returns `true` if the request is controlled by a service worker, `false` otherwise.
111+
*/
112+
processBrowserPreRequest (browserPreRequest: BrowserPreRequest) {
113+
if (browserPreRequest.initiator?.type === 'preload') {
114+
return false
115+
}
116+
117+
let requestControlledByServiceWorker = false
118+
119+
this.serviceWorkerRegistrations.forEach((registration) => {
120+
const activatedServiceWorker = registration.activatedServiceWorker
121+
const paramlessDocumentURL = browserPreRequest.documentURL.split('?')[0]
122+
123+
if (!activatedServiceWorker || activatedServiceWorker.initiatorURL !== paramlessDocumentURL) {
124+
return
125+
}
126+
127+
const paramlessURL = browserPreRequest.url.split('?')[0]
128+
const paramlessInitiatorURL = browserPreRequest.initiator?.url?.split('?')[0]
129+
const paramlessCallStackURL = browserPreRequest.initiator?.stack?.callFrames[0]?.url?.split('?')[0]
130+
const urlIsControlled = paramlessURL.startsWith(registration.scopeURL)
131+
const initiatorUrlIsControlled = paramlessInitiatorURL && activatedServiceWorker.controlledURLs?.has(paramlessInitiatorURL)
132+
const topStackUrlIsControlled = paramlessCallStackURL && activatedServiceWorker.controlledURLs?.has(paramlessCallStackURL)
133+
134+
if (urlIsControlled || initiatorUrlIsControlled || topStackUrlIsControlled) {
135+
activatedServiceWorker.controlledURLs.add(paramlessURL)
136+
requestControlledByServiceWorker = true
137+
}
138+
})
139+
140+
return requestControlledByServiceWorker
141+
}
142+
143+
/**
144+
* Registers the given service worker with the given scope. Will not overwrite an existing registration.
145+
*/
146+
private registerServiceWorker ({ registrationId, scopeURL }: RegisterServiceWorkerOptions) {
147+
// Only register service workers if they haven't already been registered
148+
if (this.serviceWorkerRegistrations.get(registrationId)?.scopeURL === scopeURL) {
149+
return
150+
}
151+
152+
this.serviceWorkerRegistrations.set(registrationId, {
153+
registrationId,
154+
scopeURL,
155+
})
156+
}
157+
158+
/**
159+
* Unregisters the service worker with the given registration ID.
160+
*/
161+
private unregisterServiceWorker ({ registrationId }: UnregisterServiceWorkerOptions) {
162+
this.serviceWorkerRegistrations.delete(registrationId)
163+
}
164+
165+
/**
166+
* Adds an activated service worker to the manager.
167+
*/
168+
private addActivatedServiceWorker ({ registrationId, scriptURL }: AddActivatedServiceWorkerOptions) {
169+
const registration = this.serviceWorkerRegistrations.get(registrationId)
170+
171+
if (registration) {
172+
const initiatorURL = this.pendingInitiators.get(scriptURL)
173+
174+
registration.activatedServiceWorker = {
175+
registrationId,
176+
scriptURL,
177+
controlledURLs: new Set<string>(),
178+
initiatorURL: initiatorURL || registration.activatedServiceWorker?.initiatorURL,
179+
}
180+
181+
this.pendingInitiators.delete(scriptURL)
182+
} else {
183+
debug('Could not find service worker registration for registration ID %s', registrationId)
184+
}
185+
}
186+
}

0 commit comments

Comments
 (0)