fix(reporters): check the test result again when tests are rerunning (#8063)

Author: sheremet-va

packages/vitest/src/node/core.ts

@@ -548,10 +548,6 @@ export class Vitest {
       await this._reportFileTask(file)
     }
 
-    if (hasFailed(files)) {
-      process.exitCode = 1
-    }
-
     this._checkUnhandledErrors(errors)
     await this._testRun.end(specifications, errors).catch(noop)
     await this.initCoverageProvider()
@@ -632,22 +628,14 @@ export class Vitest {
 
     // if run with --changed, don't exit if no tests are found
     if (!files.length) {
-      const throwAnError = !this.config.watch || !(this.config.changed || this.config.related?.length)
-
       await this._testRun.start([])
       const coverage = await this.coverageProvider?.generateCoverage?.({ allTestsRun: true })
 
-      // set exit code before calling `onTestRunEnd` so the lifecycle is consistent
-      if (throwAnError) {
-        const exitCode = this.config.passWithNoTests ? 0 : 1
-        process.exitCode = exitCode
-      }
-
       await this._testRun.end([], [], coverage)
       // Report coverage for uncovered files
       await this.reportCoverage(coverage, true)
 
-      if (throwAnError) {
+      if (!this.config.watch || !(this.config.changed || this.config.related?.length)) {
         throw new FilesNotFoundError(this.mode)
       }
     }
@@ -784,10 +772,6 @@ export class Vitest {
 
         const files = this.state.getFiles()
 
-        if (hasFailed(files)) {
-          process.exitCode = 1
-        }
-
         this.cache.results.updateResults(files)
         try {
           await this.cache.results.writeToCache()

packages/vitest/src/node/test-run.ts

@@ -13,6 +13,7 @@ import type { Vitest } from './core'
 import type { TestProject } from './project'
 import type { ReportedHookContext, TestCase, TestCollection, TestModule } from './reporters/reported-tasks'
 import type { TestSpecification } from './spec'
+import type { TestRunEndReason } from './types/reporter'
 import assert from 'node:assert'
 import { createHash } from 'node:crypto'
 import { copyFile, mkdir } from 'node:fs/promises'
@@ -89,14 +90,18 @@ export class TestRun {
     const modules = specifications.map(spec => spec.testModule).filter(s => s != null)
     const files = modules.map(m => m.task)
 
-    const state = this.vitest.isCancelling
+    const state: TestRunEndReason = this.vitest.isCancelling
       ? 'interrupted'
       // by this point, the run will be marked as failed if there are any errors,
       // should it be done by testRun.end?
-      : process.exitCode
+      : this.hasFailed(modules)
         ? 'failed'
         : 'passed'
 
+    if (state !== 'passed') {
+      process.exitCode = 1
+    }
+
     try {
       await Promise.all([
         this.vitest.report('onTestRunEnd', modules, [...errors] as SerializedError[], state),
@@ -111,6 +116,14 @@ export class TestRun {
     }
   }
 
+  private hasFailed(modules: TestModule[]) {
+    if (!modules.length) {
+      return !this.vitest.config.passWithNoTests
+    }
+
+    return modules.some(m => !m.ok())
+  }
+
   private async reportEvent(id: string, event: TaskUpdateEvent, data: TaskEventData | undefined) {
     const task = this.vitest.state.idMap.get(id)
     const entity = task && this.vitest.state.getReportedEntity(task)

test/reporters/tests/test-run.test.ts

@@ -8,6 +8,7 @@ import type {
   TestSpecification,
   TestSuite,
   UserConfig,
+  Vitest,
 } from 'vitest/node'
 import { rmSync } from 'node:fs'
 import { resolve, sep } from 'node:path'
@@ -1011,6 +1012,88 @@ describe('type checking', () => {
   })
 })
 
+describe('test run result', () => {
+  test('test run is interrupted', async () => {
+    let vitest: Vitest
+    let reason: TestRunEndReason | undefined
+
+    await runInlineTests({
+      'example.test.js': `
+        test('basic', () => new Promise(() => {}))
+      `,
+    }, {
+      globals: true,
+      reporters: [
+        {
+          onInit(ctx) {
+            vitest = ctx
+          },
+          async onTestModuleCollected() {
+            await vitest.cancelCurrentRun('keyboard-input')
+          },
+          onTestRunEnd(_, __, reason_) {
+            reason = reason_
+          },
+        },
+      ],
+    })
+
+    expect(reason).toBe('interrupted')
+  })
+
+  test('test run failed, but passed afterwards', async () => {
+    let reason: TestRunEndReason | undefined
+
+    const { fs } = await runInlineTests({
+      'example.test.js': `
+        test('basic', () => {
+          expect(1).toBe(2)
+        })
+      `,
+    }, {
+      globals: true,
+      watch: true,
+      reporters: [
+        {
+          onTestRunEnd(_, __, reason_) {
+            reason = reason_
+          },
+        },
+      ],
+    })
+
+    expect(reason).toBe('failed')
+
+    fs.editFile('./example.test.js', c => c.replace('toBe(2)', 'toBe(1)'))
+
+    await expect.poll(() => reason).toBe('passed')
+  })
+
+  test('test run passed', async () => {
+    let reason: TestRunEndReason | undefined
+
+    await runInlineTests({
+      'example.test.js': `
+        test('basic', () => {
+          expect(1).toBe(1)
+        })
+      `,
+    }, {
+      globals: true,
+      watch: true,
+      reporters: [
+        {
+          onTestRunEnd(_, __, reason_) {
+            reason = reason_
+          },
+        },
+      ],
+    })
+
+    expect(reason).toBe('passed')
+  })
+})
+
 interface ReporterOptions {
   printTestRunEvents?: boolean
   roots?: string[]