feat: use yielding to main thread in more places

.changeset/smooth-wolves-mix.md

@@ -0,0 +1,5 @@
+---
+'posthog-js': minor
+---
+
+feat: introduce a task queue that will yield to the main thread periodically reducing the impact of long operations

packages/browser/README.md

@@ -84,7 +84,7 @@ We have 2 options for linking this project to your local version: via [pnpm link
 - run `pnpm build` and `pnpm package` in the root of this repo to generate a tarball of this project.
 - run `pnpm -r update posthog-js@file:[ABSOLUTE_PATH_TO_POSTHOG_JS_REPO]/target/posthog-js.tgz` in the root of the repo that you want to link to (e.g. the posthog main repo).
 - run `pnpm install` in that same repo
-- run `cd frontend && pnpm run copy-scripts` if the repo that you want to link to is the posthog main repo.
+- run `pnpm --filter=frontend copy-scripts` if the repo that you want to link to is the posthog main repo.
 
 Then, once this link has been created, any time you need to make a change to `posthog-js`, you can run `pnpm build && pnpm package` from the `posthog-js` root and the changes will appear in the other repo.
 

packages/browser/jest.config.js

@@ -1,5 +1,5 @@
 module.exports = {
-    testPathIgnorePatterns: ['/node_modules/', '/cypress/', '/react/', '/test_data/', '/testcafe/'],
+    testPathIgnorePatterns: ['/node_modules/', '/cypress/', '/react/', '/test_data/', '/testcafe/', '\\.d\\.ts$'],
     moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
     setupFilesAfterEnv: ['./src/__tests__/setup.js'],
     modulePathIgnorePatterns: ['src/__tests__/setup.js', 'src/__tests__/helpers/'],

packages/browser/src/__tests__/deferred-init-extensions.test.ts

@@ -1,6 +1,7 @@
 import { createPosthogInstance } from './helpers/posthog-instance'
 import { uuidv7 } from '../uuidv7'
 import { RemoteConfig } from '../types'
+import { scheduler } from '../utils/scheduler'
 
 jest.mock('../utils/globals', () => {
     const orig = jest.requireActual('../utils/globals')
@@ -39,6 +40,7 @@ describe('deferred extension initialization', () => {
         console.error = jest.fn()
         mockReferrerGetter.mockReturnValue('https://referrer.com')
         mockURLGetter.mockReturnValue('https://example.com')
+        scheduler._reset()
     })
 
     describe('race condition handling', () => {
@@ -121,7 +123,7 @@ describe('deferred extension initialization', () => {
 
             const posthog = await createPosthogInstance(token, {
                 __preview_deferred_init_extensions: true,
-                advanced_disable_decide: false,
+                advanced_disable_decide: true,
                 capture_pageview: false,
                 disable_session_recording: true,
             })

packages/browser/src/__tests__/extensions/replay/lazy-sessionrecording.test.ts

@@ -41,6 +41,7 @@ import {
 } from '../../../extensions/replay/types/rrweb-types'
 import { ConsentManager } from '../../../consent'
 import { SimpleEventEmitter } from '../../../utils/simple-event-emitter'
+import { scheduler } from '../../../utils/scheduler'
 import Mock = jest.Mock
 import { SessionRecording } from '../../../extensions/replay/session-recording'
 import {
@@ -211,6 +212,9 @@ describe('Lazy SessionRecording', () => {
     }
 
     beforeEach(() => {
+        jest.useFakeTimers()
+        scheduler._reset()
+
         removePageviewCaptureHookMock = jest.fn()
         sessionId = 'sessionId' + uuidv7()
 
@@ -295,6 +299,7 @@ describe('Lazy SessionRecording', () => {
     })
 
     afterEach(() => {
+        jest.useRealTimers()
         // @ts-expect-error this is a test, it's safe to write to location like this
         window!.location = originalLocation
     })
@@ -903,6 +908,8 @@ describe('Lazy SessionRecording', () => {
 
                 // this triggers idle state and isn't a user interaction so does not take a full snapshot
                 emitInactiveEvent(thirdActivityTimestamp, true)
+                // Run only the scheduler's setTimeout(0), not all timers which would advance system time
+                jest.advanceTimersByTime(1)
 
                 // event was not active so activity timestamp is not updated
                 expect(sessionRecording['_lazyLoadedSessionRecording']['_lastActivityTimestamp']).toEqual(
@@ -1000,6 +1007,7 @@ describe('Lazy SessionRecording', () => {
                 // this triggers idle state and isn't a user interaction so does not take a full snapshot
 
                 emitInactiveEvent(thirdActivityTimestamp, true)
+                jest.runAllTimers()
 
                 // event was not active so activity timestamp is not updated
                 expect(sessionRecording['_lazyLoadedSessionRecording']['_lastActivityTimestamp']).toEqual(
@@ -1039,7 +1047,7 @@ describe('Lazy SessionRecording', () => {
                 // this triggers exit from idle state as it is a user interaction
                 // this will restart the session so the activity timestamp won't match
                 // restarting the session checks the id with "now" so we need to freeze that, or we'll start a second new session
-                jest.useFakeTimers().setSystemTime(new Date(fourthActivityTimestamp))
+                jest.setSystemTime(new Date(fourthActivityTimestamp))
                 const fourthSnapshot = emitActiveEvent(fourthActivityTimestamp, false)
                 expect(sessionIdGeneratorMock).toHaveBeenCalledTimes(1)
                 const endingSessionId = sessionRecording['_lazyLoadedSessionRecording']['_sessionId']
@@ -1207,6 +1215,7 @@ describe('Lazy SessionRecording', () => {
                     })
                 )
                 sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+                jest.runAllTimers()
 
                 expect(posthog.capture).toHaveBeenCalledWith(
                     '$snapshot',
@@ -1231,6 +1240,7 @@ describe('Lazy SessionRecording', () => {
             it('compresses incremental snapshot mutation data', () => {
                 _emit(createIncrementalMutationEvent({ texts: [Array(30).fill(uuidv7()).join('')] }))
                 sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+                jest.runAllTimers()
 
                 expect(posthog.capture).toHaveBeenCalledWith(
                     '$snapshot',
@@ -1262,6 +1272,7 @@ describe('Lazy SessionRecording', () => {
             it('compresses incremental snapshot style data', () => {
                 _emit(createIncrementalStyleSheetEvent({ adds: [Array(30).fill(uuidv7()).join('')] }))
                 sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+                jest.runAllTimers()
 
                 expect(posthog.capture).toHaveBeenCalledWith(
                     '$snapshot',
@@ -1297,6 +1308,7 @@ describe('Lazy SessionRecording', () => {
                 const mouseEvent = createIncrementalMouseEvent()
                 _emit(mouseEvent)
                 sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+                jest.runAllTimers()
 
                 expect(posthog.capture).toHaveBeenCalledWith(
                     '$snapshot',
@@ -1315,6 +1327,7 @@ describe('Lazy SessionRecording', () => {
             it('does not compress custom events', () => {
                 _emit(createCustomSnapshot(undefined, { tag: 'wat' }))
                 sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+                jest.runAllTimers()
 
                 expect(posthog.capture).toHaveBeenCalledWith(
                     '$snapshot',
@@ -1341,6 +1354,7 @@ describe('Lazy SessionRecording', () => {
             it('does not compress meta events', () => {
                 _emit(createMetaSnapshot())
                 sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+                jest.runAllTimers()
 
                 expect(posthog.capture).toHaveBeenCalledWith(
                     '$snapshot',
@@ -1430,6 +1444,7 @@ describe('Lazy SessionRecording', () => {
 
             // access private method 🤯so we don't need to wait for the timer
             sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+            jest.runAllTimers()
             expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer'].data.length).toEqual(0)
 
             expect(posthog.capture).toHaveBeenCalledTimes(1)
@@ -1473,6 +1488,7 @@ describe('Lazy SessionRecording', () => {
             expect(sessionRecording['_lazyLoadedSessionRecording']['_flushBufferTimer']).not.toBeUndefined()
 
             sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+            jest.runAllTimers()
             expect(sessionRecording['_lazyLoadedSessionRecording']['_flushBufferTimer']).toBeUndefined()
 
             expect(posthog.capture).toHaveBeenCalledTimes(1)
@@ -1519,6 +1535,8 @@ describe('Lazy SessionRecording', () => {
 
             // Another big event means the old data will be flushed
             _emit(createIncrementalSnapshot({ data: { source: 1, payload: bigData } }))
+            // Run only the scheduler's setTimeout(0), not the full flush timer
+            jest.advanceTimersByTime(1)
             expect(posthog.capture).toHaveBeenCalled()
             expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer'].data.length).toEqual(1) // The new event
             expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer']).toMatchObject({ size: 755017 })
@@ -1583,6 +1601,8 @@ describe('Lazy SessionRecording', () => {
             // i.e. the data in the buffer should be sent with 'otherSessionId'
             sessionRecording['_lazyLoadedSessionRecording']['_buffer']!.sessionId = 'otherSessionId'
             _emit(createIncrementalSnapshot({ emit: 2 }))
+            // Run only the scheduler's setTimeout(0), not the full flush timer
+            jest.advanceTimersByTime(1)
 
             expect(posthog.capture).toHaveBeenCalledWith(
                 '$snapshot',
@@ -2196,6 +2216,8 @@ describe('Lazy SessionRecording', () => {
             _emit(createIncrementalSnapshot({ data: { source: 2 } }))
 
             simpleEventEmitter.emit('eventCaptured', { event: '$exception' })
+            // Run only the scheduler's setTimeout(0), not all timers which would advance system time
+            jest.advanceTimersByTime(1)
             expect(sessionRecording.status).toBe('active')
             expect(posthog.capture).toHaveBeenCalled()
         })
@@ -2469,6 +2491,7 @@ describe('Lazy SessionRecording', () => {
 
             // don't wait two seconds for the flush timer
             sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+            jest.runAllTimers()
 
             _emit(createIncrementalSnapshot({ data: { source: 1 } }))
             expect(posthog.capture).toHaveBeenCalled()
@@ -2909,6 +2932,7 @@ describe('Lazy SessionRecording', () => {
             expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer'].data.length).toBe(1) // the emitted incremental event
             // call the private method to avoid waiting for the timer
             sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+            jest.runAllTimers()
 
             expect(posthog.capture).toHaveBeenCalled()
         })
@@ -2929,6 +2953,8 @@ describe('Lazy SessionRecording', () => {
             expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer'].data.length).toBe(1) // the emitted incremental event
             // call the private method to avoid waiting for the timer
             sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+            // Run only the scheduler's setTimeout(0), not all timers which would advance system time
+            jest.advanceTimersByTime(1)
 
             expect(posthog.capture).not.toHaveBeenCalled()
 
@@ -2937,6 +2963,8 @@ describe('Lazy SessionRecording', () => {
             expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer'].data.length).toBe(2) // two emitted incremental events
             // call the private method to avoid waiting for the timer
             sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+            // Run only the scheduler's setTimeout(0), not all timers which would advance system time
+            jest.advanceTimersByTime(1)
 
             expect(posthog.capture).toHaveBeenCalled()
             expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer'].data.length).toBe(0)
@@ -2946,6 +2974,7 @@ describe('Lazy SessionRecording', () => {
             expect(sessionRecording['_lazyLoadedSessionRecording']['_sessionDuration']).toBe(1502)
             // call the private method to avoid waiting for the timer
             sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+            jest.runAllTimers()
 
             expect(posthog.capture).toHaveBeenCalled()
             expect(sessionRecording['_lazyLoadedSessionRecording']['_buffer'].data.length).toBe(0)
@@ -3125,6 +3154,7 @@ describe('Lazy SessionRecording', () => {
 
             // manually flush the buffer to simulate data being sent
             sessionRecording['_lazyLoadedSessionRecording']['_flushBuffer']()
+            jest.runAllTimers()
 
             // verify data was tracked
             const flushedSize =

packages/browser/src/__tests__/request-queue.test.ts

@@ -1,6 +1,7 @@
 import { DEFAULT_FLUSH_INTERVAL_MS, RequestQueue } from '../request-queue'
 import { QueuedRequestWithOptions } from '../types'
 import { createPosthogInstance } from './helpers/posthog-instance'
+import { scheduler } from '../utils/scheduler'
 
 const EPOCH = 1_600_000_000
 
@@ -48,6 +49,7 @@ describe('RequestQueue', () => {
             jest.useFakeTimers()
             jest.setSystemTime(EPOCH - 3000) // Running the timers will add 3 seconds
             jest.spyOn(console, 'warn').mockImplementation(() => {})
+            scheduler._reset()
         })
 
         it('handles poll after enqueueing requests', () => {
@@ -74,7 +76,7 @@ describe('RequestQueue', () => {
 
             expect(sendRequest).toHaveBeenCalledTimes(0)
 
-            jest.runOnlyPendingTimers()
+            jest.runAllTimers()
 
             expect(sendRequest).toHaveBeenCalledTimes(3)
             expect(jest.mocked(sendRequest).mock.calls).toEqual([

packages/browser/src/__tests__/retry-queue.test.ts

@@ -2,6 +2,7 @@
 
 import { pickNextRetryDelay, RetryQueue } from '../retry-queue'
 import { assignableWindow } from '../utils/globals'
+import { scheduler } from '../utils/scheduler'
 
 describe('RetryQueue', () => {
     const mockPosthog = {
@@ -16,12 +17,15 @@ describe('RetryQueue', () => {
         jest.useFakeTimers()
         jest.setSystemTime(now)
         jest.spyOn(assignableWindow.console, 'warn').mockImplementation()
+        scheduler._reset()
     })
 
-    const fastForwardTimeAndRunTimer = (time = 3500) => {
+    const fastForwardTimeAndRunTimer = async (time = 3500) => {
         now += time
         jest.setSystemTime(now)
-        jest.runOnlyPendingTimers()
+        jest.runAllTimers()
+        // eslint-disable-next-line compat/compat
+        await Promise.resolve()
     }
 
     const enqueueRequests = () => {
@@ -55,7 +59,7 @@ describe('RetryQueue', () => {
         mockPosthog._send_request.mockClear()
     }
 
-    it('processes retry requests', () => {
+    it('processes retry requests', async () => {
         enqueueRequests()
 
         expect(retryQueue.length).toEqual(4)
@@ -95,7 +99,7 @@ describe('RetryQueue', () => {
         ])
 
         // Fast forward enough time to clear the jitter
-        fastForwardTimeAndRunTimer(3500)
+        await fastForwardTimeAndRunTimer(3500)
 
         // clears queue
         expect(retryQueue.length).toEqual(0)
@@ -109,9 +113,9 @@ describe('RetryQueue', () => {
         ])
     })
 
-    it('adds the retry_count to the url', () => {
+    it('adds the retry_count to the url', async () => {
         enqueueRequests()
-        fastForwardTimeAndRunTimer(3500)
+        await fastForwardTimeAndRunTimer(3500)
 
         expect(mockPosthog._send_request.mock.calls.map(([arg1]) => arg1.url)).toEqual([
             '/e?retry_count=1',
@@ -136,12 +140,15 @@ describe('RetryQueue', () => {
         ])
     })
 
-    it('enqueues requests when offline and flushes immediately when online again', () => {
+    it('enqueues requests when offline and flushes immediately when online again', async () => {
         retryQueue['_areWeOnline'] = false
         expect(retryQueue['_areWeOnline']).toEqual(false)
 
         enqueueRequests()
-        fastForwardTimeAndRunTimer()
+        // Only advance time, don't run all timers as the queue polls indefinitely when offline
+        now += 3500
+        jest.setSystemTime(now)
+        jest.advanceTimersByTime(3500)
 
         // requests aren't attempted when we're offline
         expect(mockPosthog._send_request).toHaveBeenCalledTimes(0)
@@ -150,6 +157,9 @@ describe('RetryQueue', () => {
         expect(retryQueue.length).toEqual(4)
 
         window.dispatchEvent(new Event('online'))
+        jest.runAllTimers()
+        // eslint-disable-next-line compat/compat
+        await Promise.resolve()
 
         expect(retryQueue['_areWeOnline']).toEqual(true)
         expect(retryQueue.length).toEqual(0)
@@ -166,7 +176,7 @@ describe('RetryQueue', () => {
         expect(retryQueue.length).toEqual(0)
     })
 
-    it('only calls the callback when successful', () => {
+    it('only calls the callback when successful', async () => {
         const cb = jest.fn()
         mockPosthog._send_request.mockImplementation(({ callback }) => {
             callback?.({ statusCode: 500 })
@@ -182,7 +192,7 @@ describe('RetryQueue', () => {
             callback?.({ statusCode: 200, text: 'it worked!' })
         })
 
-        fastForwardTimeAndRunTimer()
+        await fastForwardTimeAndRunTimer()
 
         expect(retryQueue.length).toEqual(0)
         expect(cb).toHaveBeenCalledTimes(1)
@@ -254,23 +264,23 @@ describe('RetryQueue', () => {
     })
 
     describe('memory management', () => {
-        it('stops polling when queue becomes empty', () => {
+        it('stops polling when queue becomes empty', async () => {
             enqueueRequests()
 
             expect(retryQueue['_isPolling']).toBe(true)
             expect(retryQueue['_poller']).toBeDefined()
             expect(retryQueue.length).toEqual(4)
 
-            fastForwardTimeAndRunTimer(3500)
+            await fastForwardTimeAndRunTimer(3500)
 
             expect(retryQueue.length).toEqual(0)
             expect(retryQueue['_isPolling']).toBe(false)
             expect(retryQueue['_poller']).toBeUndefined()
         })
 
-        it('restarts polling when items are added after stopping', () => {
+        it('restarts polling when items are added after stopping', async () => {
             enqueueRequests()
-            fastForwardTimeAndRunTimer(3500)
+            await fastForwardTimeAndRunTimer(3500)
 
             expect(retryQueue['_isPolling']).toBe(false)
             expect(retryQueue['_poller']).toBeUndefined()