Skip to content

Commit 17209ef

Browse files
pauldambrapauldambra
committed
feat: use yielding to main thread in more places
1 parent 8fc7045 commit 17209ef

File tree

14 files changed

+711
-108
lines changed

14 files changed

+711
-108
lines changed

‎.changeset/smooth-wolves-mix.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'posthog-js': minor
3+
---
4+
5+
feat: introduce a task queue that will yield to the main thread periodically reducing the impact of long operations

‎packages/browser/jest.config.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
module.exports = {
2-
testPathIgnorePatterns: ['/node_modules/', '/cypress/', '/react/', '/test_data/', '/testcafe/'],
2+
testPathIgnorePatterns: ['/node_modules/', '/cypress/', '/react/', '/test_data/', '/testcafe/', '\\.d\\.ts$'],
33
moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
44
setupFilesAfterEnv: ['./src/__tests__/setup.js'],
55
modulePathIgnorePatterns: ['src/__tests__/setup.js', 'src/__tests__/helpers/'],

‎packages/browser/src/__tests__/retry-queue.test.ts‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ describe('RetryQueue', () => {
1818
jest.spyOn(assignableWindow.console, 'warn').mockImplementation()
1919
})
2020

21-
const fastForwardTimeAndRunTimer = (time = 3500) => {
21+
const fastForwardTimeAndRunTimer = async (time = 3500) => {
2222
now += time
2323
jest.setSystemTime(now)
24-
jest.runOnlyPendingTimers()
24+
await jest.runOnlyPendingTimersAsync()
2525
}
2626

2727
const enqueueRequests = () => {
@@ -55,7 +55,7 @@ describe('RetryQueue', () => {
5555
mockPosthog._send_request.mockClear()
5656
}
5757

58-
it('processes retry requests', () => {
58+
it('processes retry requests', async () => {
5959
enqueueRequests()
6060

6161
expect(retryQueue.length).toEqual(4)
@@ -95,7 +95,7 @@ describe('RetryQueue', () => {
9595
])
9696

9797
// Fast forward enough time to clear the jitter
98-
fastForwardTimeAndRunTimer(3500)
98+
await fastForwardTimeAndRunTimer(3500)
9999

100100
// clears queue
101101
expect(retryQueue.length).toEqual(0)
@@ -109,9 +109,9 @@ describe('RetryQueue', () => {
109109
])
110110
})
111111

112-
it('adds the retry_count to the url', () => {
112+
it('adds the retry_count to the url', async () => {
113113
enqueueRequests()
114-
fastForwardTimeAndRunTimer(3500)
114+
await fastForwardTimeAndRunTimer(3500)
115115

116116
expect(mockPosthog._send_request.mock.calls.map(([arg1]) => arg1.url)).toEqual([
117117
'/e?retry_count=1',
@@ -136,12 +136,12 @@ describe('RetryQueue', () => {
136136
])
137137
})
138138

139-
it('enqueues requests when offline and flushes immediately when online again', () => {
139+
it('enqueues requests when offline and flushes immediately when online again', async () => {
140140
retryQueue['_areWeOnline'] = false
141141
expect(retryQueue['_areWeOnline']).toEqual(false)
142142

143143
enqueueRequests()
144-
fastForwardTimeAndRunTimer()
144+
await fastForwardTimeAndRunTimer()
145145

146146
// requests aren't attempted when we're offline
147147
expect(mockPosthog._send_request).toHaveBeenCalledTimes(0)
@@ -166,7 +166,7 @@ describe('RetryQueue', () => {
166166
expect(retryQueue.length).toEqual(0)
167167
})
168168

169-
it('only calls the callback when successful', () => {
169+
it('only calls the callback when successful', async () => {
170170
const cb = jest.fn()
171171
mockPosthog._send_request.mockImplementation(({ callback }) => {
172172
callback?.({ statusCode: 500 })
@@ -182,7 +182,7 @@ describe('RetryQueue', () => {
182182
callback?.({ statusCode: 200, text: 'it worked!' })
183183
})
184184

185-
fastForwardTimeAndRunTimer()
185+
await fastForwardTimeAndRunTimer()
186186

187187
expect(retryQueue.length).toEqual(0)
188188
expect(cb).toHaveBeenCalledTimes(1)
@@ -254,23 +254,23 @@ describe('RetryQueue', () => {
254254
})
255255

256256
describe('memory management', () => {
257-
it('stops polling when queue becomes empty', () => {
257+
it('stops polling when queue becomes empty', async () => {
258258
enqueueRequests()
259259

260260
expect(retryQueue['_isPolling']).toBe(true)
261261
expect(retryQueue['_poller']).toBeDefined()
262262
expect(retryQueue.length).toEqual(4)
263263

264-
fastForwardTimeAndRunTimer(3500)
264+
await fastForwardTimeAndRunTimer(3500)
265265

266266
expect(retryQueue.length).toEqual(0)
267267
expect(retryQueue['_isPolling']).toBe(false)
268268
expect(retryQueue['_poller']).toBeUndefined()
269269
})
270270

271-
it('restarts polling when items are added after stopping', () => {
271+
it('restarts polling when items are added after stopping', async () => {
272272
enqueueRequests()
273-
fastForwardTimeAndRunTimer(3500)
273+
await fastForwardTimeAndRunTimer(3500)
274274

275275
expect(retryQueue['_isPolling']).toBe(false)
276276
expect(retryQueue['_poller']).toBeUndefined()

‎packages/browser/src/extensions/replay/external/lazy-loaded-session-recorder.ts‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import { assignableWindow, LazyLoadedSessionRecordingInterface, window, document
3535
import { addEventListener } from '../../../utils'
3636
import { MutationThrottler } from './mutation-throttler'
3737
import { createLogger } from '../../../utils/logger'
38+
import { processWithYield } from '../../../utils/task-queue'
3839
import {
3940
clampToRange,
4041
includes,
@@ -1068,17 +1069,21 @@ export class LazyLoadedSessionRecording implements LazyLoadedSessionRecordingInt
10681069

10691070
if (this._buffer.data.length > 0) {
10701071
const snapshotEvents = splitBuffer(this._buffer)
1071-
snapshotEvents.forEach((snapshotBuffer) => {
1072-
this._flushedSizeTracker?.trackSize(snapshotBuffer.size)
1073-
this._captureSnapshot({
1074-
$snapshot_bytes: snapshotBuffer.size,
1075-
$snapshot_data: snapshotBuffer.data,
1076-
$session_id: snapshotBuffer.sessionId,
1077-
$window_id: snapshotBuffer.windowId,
1078-
$lib: 'web',
1079-
$lib_version: Config.LIB_VERSION,
1080-
})
1081-
})
1072+
void processWithYield(
1073+
snapshotEvents,
1074+
(snapshotBuffer) => {
1075+
this._flushedSizeTracker?.trackSize(snapshotBuffer.size)
1076+
this._captureSnapshot({
1077+
$snapshot_bytes: snapshotBuffer.size,
1078+
$snapshot_data: snapshotBuffer.data,
1079+
$session_id: snapshotBuffer.sessionId,
1080+
$window_id: snapshotBuffer.windowId,
1081+
$lib: 'web',
1082+
$lib_version: Config.LIB_VERSION,
1083+
})
1084+
},
1085+
{ timeBudgetMs: 30 }
1086+
)
10821087
}
10831088

10841089
// buffer is empty, we clear it in case the session id has changed

‎packages/browser/src/extensions/replay/external/network-plugin.ts‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import { formDataToQuery } from '../../../utils/request-utils'
1818
import { patch } from '../rrweb-plugins/patch'
1919
import { isHostOnDenyList } from '../../../extensions/replay/external/denylist'
2020
import { defaultNetworkOptions } from './config'
21+
import { processWithYield } from '../../../utils/task-queue'
2122

2223
const logger = createLogger('[Recorder]')
2324

@@ -61,11 +62,18 @@ function initPerformanceObserver(cb: networkCallback, win: IWindow, options: Req
6162
isNavigationTiming(entry) ||
6263
(isResourceTiming(entry) && options.initiatorTypes.includes(entry.initiatorType as InitiatorType))
6364
)
64-
cb({
65-
requests: initialPerformanceEntries.flatMap((entry) =>
66-
prepareRequest({ entry, method: undefined, status: undefined, networkRequest: {}, isInitial: true })
67-
),
68-
isInitial: true,
65+
66+
// Process initial performance entries with yielding for large sets
67+
processWithYield(
68+
initialPerformanceEntries,
69+
(entry) =>
70+
prepareRequest({ entry, method: undefined, status: undefined, networkRequest: {}, isInitial: true }),
71+
{ timeBudgetMs: 30 }
72+
).then((requests) => {
73+
cb({
74+
requests: requests.flat(),
75+
isInitial: true,
76+
})
6977
})
7078
}
7179
const observer = new win.PerformanceObserver((entries) => {

‎packages/browser/src/posthog-core.ts‎

Lines changed: 29 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
SURVEYS_REQUEST_TIMEOUT_MS,
1212
USER_STATE,
1313
} from './constants'
14+
import { TaskQueue } from './utils/task-queue'
1415
import { DeadClicksAutocapture, isDeadClicksEnabledForAutocapture } from './extensions/dead-clicks-autocapture'
1516
import { ExceptionObserver } from './extensions/exception-autocapture'
1617
import { HistoryAutocapture } from './extensions/history-autocapture'
@@ -663,10 +664,6 @@ export class PostHog {
663664
}
664665

665666
private _initExtensions(startInCookielessMode: boolean): void {
666-
// we don't support IE11 anymore, so performance.now is safe
667-
// eslint-disable-next-line compat/compat
668-
const initStartTime = performance.now()
669-
670667
this.historyAutocapture = new HistoryAutocapture(this)
671668
this.historyAutocapture.startIfEnabled()
672669

@@ -733,52 +730,39 @@ export class PostHog {
733730
})
734731

735732
// Process tasks with time-slicing to avoid blocking
736-
this._processInitTaskQueue(initTasks, initStartTime)
737-
}
738-
739-
private _processInitTaskQueue(queue: Array<() => void>, initStartTime: number): void {
740-
const TIME_BUDGET_MS = 30 // Respect frame budget (~60fps = 16ms, but we're already deferred)
741-
742-
while (queue.length > 0) {
743-
// Only time-slice if deferred init is enabled, otherwise run synchronously
744-
if (this.config.__preview_deferred_init_extensions) {
745-
// we don't support IE11 anymore, so performance.now is safe
746-
// eslint-disable-next-line compat/compat
747-
const elapsed = performance.now() - initStartTime
748-
749-
// Check if we've exceeded our time budget
750-
if (elapsed >= TIME_BUDGET_MS && queue.length > 0) {
751-
// Yield to browser, then continue processing
752-
setTimeout(() => {
753-
this._processInitTaskQueue(queue, initStartTime)
754-
}, 0)
755-
return
756-
}
757-
}
758-
759-
// Process next task
760-
const task = queue.shift()
761-
if (task) {
733+
// Only use time-slicing if deferred init is enabled, otherwise process synchronously
734+
if (this.config.__preview_deferred_init_extensions) {
735+
const taskQueue = new TaskQueue({
736+
timeBudgetMs: 30,
737+
onComplete: (totalTimeMs) => {
738+
this.register_for_session({
739+
$sdk_debug_extensions_init_method: 'deferred',
740+
$sdk_debug_extensions_init_time_ms: totalTimeMs,
741+
})
742+
logger.info(`PostHog extensions initialized (${totalTimeMs}ms)`)
743+
},
744+
onError: (error) => {
745+
logger.error('Error initializing extension:', error)
746+
},
747+
})
748+
taskQueue.enqueueAll(initTasks)
749+
} else {
750+
// we don't support IE11 anymore, so performance.now is safe
751+
// eslint-disable-next-line compat/compat
752+
const startTime = performance.now()
753+
initTasks.forEach((task) => {
762754
try {
763755
task()
764756
} catch (error) {
765757
logger.error('Error initializing extension:', error)
766758
}
767-
}
768-
}
769-
770-
// All tasks complete - record timing for both sync and deferred modes
771-
// we don't support IE11 anymore, so performance.now is safe
772-
// eslint-disable-next-line compat/compat
773-
const taskInitTiming = Math.round(performance.now() - initStartTime)
774-
this.register_for_session({
775-
$sdk_debug_extensions_init_method: this.config.__preview_deferred_init_extensions
776-
? 'deferred'
777-
: 'synchronous',
778-
$sdk_debug_extensions_init_time_ms: taskInitTiming,
779-
})
780-
if (this.config.__preview_deferred_init_extensions) {
781-
logger.info(`PostHog extensions initialized (${taskInitTiming}ms)`)
759+
})
760+
// eslint-disable-next-line compat/compat
761+
const totalTimeMs = Math.round(performance.now() - startTime)
762+
this.register_for_session({
763+
$sdk_debug_extensions_init_method: 'synchronous',
764+
$sdk_debug_extensions_init_time_ms: totalTimeMs,
765+
})
782766
}
783767
}
784768

‎packages/browser/src/request-queue.ts‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { each } from './utils'
33

44
import { isArray, isUndefined, clampToRange } from '@posthog/core'
55
import { logger } from './utils/logger'
6+
import { processWithYield } from './utils/task-queue'
67

78
export const DEFAULT_FLUSH_INTERVAL_MS = 3000
89

@@ -57,22 +58,27 @@ export class RequestQueue {
5758
if (this._isPaused) {
5859
return
5960
}
60-
this._flushTimeout = setTimeout(() => {
61+
this._flushTimeout = setTimeout(async () => {
6162
this._clearFlushTimeout()
6263
if (this._queue.length > 0) {
6364
const requests = this._formatQueue()
64-
for (const key in requests) {
65-
const req = requests[key]
66-
const now = new Date().getTime()
65+
const requestEntries = Object.entries(requests)
66+
const now = new Date().getTime()
6767

68-
if (req.data && isArray(req.data)) {
69-
each(req.data, (data) => {
70-
data['offset'] = Math.abs(data['timestamp'] - now)
71-
delete data['timestamp']
72-
})
73-
}
74-
this._sendRequest(req)
75-
}
68+
// Process timestamp updates with yielding for large batches
69+
await processWithYield(
70+
requestEntries,
71+
([, req]) => {
72+
if (req.data && isArray(req.data)) {
73+
each(req.data, (data) => {
74+
data['offset'] = Math.abs(data['timestamp'] - now)
75+
delete data['timestamp']
76+
})
77+
}
78+
this._sendRequest(req)
79+
},
80+
{ timeBudgetMs: 30 }
81+
)
7682
}
7783
}, this._flushTimeoutMs)
7884
}

‎packages/browser/src/retry-queue.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { window } from './utils/globals'
66
import { PostHog } from './posthog-core'
77
import { extendURLParams } from './request'
88
import { addEventListener } from './utils'
9+
import { processWithYield } from './utils/task-queue'
910

1011
const thirtyMinutes = 30 * 60 * 1000
1112

@@ -120,15 +121,15 @@ export class RetryQueue {
120121
return
121122
}
122123

123-
this._poller = setTimeout(() => {
124+
this._poller = setTimeout(async () => {
124125
if (this._areWeOnline && this._queue.length > 0) {
125-
this._flush()
126+
await this._flush()
126127
}
127128
this._poll()
128129
}, this._pollIntervalMs) as any as number
129130
}
130131

131-
private _flush(): void {
132+
private async _flush(): Promise<void> {
132133
const now = Date.now()
133134
const notToFlush: RetryQueueElement[] = []
134135
const toFlush = this._queue.filter((item) => {
@@ -142,9 +143,9 @@ export class RetryQueue {
142143
this._queue = notToFlush
143144

144145
if (toFlush.length > 0) {
145-
for (const { requestOptions } of toFlush) {
146-
this.retriableRequest(requestOptions)
147-
}
146+
await processWithYield(toFlush, ({ requestOptions }) => this.retriableRequest(requestOptions), {
147+
timeBudgetMs: 30,
148+
})
148149
}
149150
}
150151

0 commit comments

Comments
 (0)