diff --git a/.github/workflows/ai.yml b/.github/workflows/ai.yml index c10f76a889..4cd3c2419c 100644 --- a/.github/workflows/ai.yml +++ b/.github/workflows/ai.yml @@ -113,11 +113,27 @@ jobs: path: ${{ github.workspace }}/packages/web-integration/midscene_run/report if-no-files-found: ignore + - name: Run apps/report e2e tests + run: cd apps/report && pnpm run e2e + id: e2e-tests-app-report + continue-on-error: true + + - name: Upload apps/report e2e report + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-app-report-output + path: | + ${{ github.workspace }}/midscene_run/report + ${{ github.workspace }}/apps/report/midscene_run/report + if-no-files-found: ignore + - name: Check if tests failed - if: steps.e2e-tests.outcome == 'failure' || steps.e2e-tests-cache.outcome == 'failure' || steps.e2e-tests-report.outcome == 'failure' + if: steps.e2e-tests.outcome == 'failure' || steps.e2e-tests-cache.outcome == 'failure' || steps.e2e-tests-report.outcome == 'failure' || steps.e2e-tests-app-report.outcome == 'failure' run: | echo "::error::The following e2e tests failed:" if [ "${{ steps.e2e-tests.outcome }}" == "failure" ]; then echo " - e2e (basic)"; fi if [ "${{ steps.e2e-tests-cache.outcome }}" == "failure" ]; then echo " - e2e:cache"; fi if [ "${{ steps.e2e-tests-report.outcome }}" == "failure" ]; then echo " - e2e:report"; fi + if [ "${{ steps.e2e-tests-app-report.outcome }}" == "failure" ]; then echo " - e2e:app-report"; fi exit 1 diff --git a/apps/report/e2e/check-html.yaml b/apps/report/e2e/check-html.yaml index 06926f00ee..5d030d5288 100755 --- a/apps/report/e2e/check-html.yaml +++ b/apps/report/e2e/check-html.yaml @@ -2,8 +2,8 @@ target: serve: ./dist url: demo.html tasks: - - name: check console html + - name: check report loads and displays tasks flow: - - ai: Click the 'Insight / Locate' on Left - sleep: 3000 - - aiAssert: There is a 'Open in Playground' button on the page + - aiAssert: There is a sidebar on the left showing a task execution list with task names and time costs + - aiAssert: There is a 'Record' timeline section with screenshot thumbnails diff --git a/apps/report/e2e/player-autoplay.yaml b/apps/report/e2e/player-autoplay.yaml new file mode 100644 index 0000000000..e74e32e3dc --- /dev/null +++ b/apps/report/e2e/player-autoplay.yaml @@ -0,0 +1,15 @@ +target: + serve: ./dist + url: demo.html +tasks: + - name: player autoplay starts automatically + flow: + - sleep: 3000 + - aiAssert: There is a video player area with a progress bar showing playback time + - aiAssert: There is a pause button (two vertical bars icon) visible, indicating the player is currently playing + + - name: pause and resume works + flow: + - ai: Click the pause button on the player controls bar + - sleep: 1000 + - aiAssert: There is a play button (triangle icon) visible, indicating the player is paused diff --git a/apps/report/e2e/player-completion.yaml b/apps/report/e2e/player-completion.yaml new file mode 100644 index 0000000000..75f42c8b62 --- /dev/null +++ b/apps/report/e2e/player-completion.yaml @@ -0,0 +1,16 @@ +target: + serve: ./dist + url: demo.html +tasks: + - name: after playback completes player shows full view at last step + flow: + # Speed up playback to 2x and wait for completion + - ai: Click the settings gear icon on the player controls bar + - sleep: 1000 + - ai: Select the '2x' playback speed option + - sleep: 500 + - ai: Click somewhere on the player video area to close the settings popup + - sleep: 30000 + # After playback completes, verify the state + - aiAssert: The play button (triangle icon) is visible on the player controls bar, indicating playback has stopped + - aiAssert: The player is showing a screenshot image of a web page diff --git a/apps/report/e2e/player-settings.yaml b/apps/report/e2e/player-settings.yaml new file mode 100644 index 0000000000..f7ce9b8222 --- /dev/null +++ b/apps/report/e2e/player-settings.yaml @@ -0,0 +1,15 @@ +target: + serve: ./dist + url: demo.html +tasks: + - name: subtitle toggle works + flow: + - sleep: 3000 + - aiAssert: There is a subtitle text overlay at the bottom area of the player + - ai: Click the settings gear icon on the player controls bar + - sleep: 1000 + - ai: Toggle off the subtitle switch in the settings popup + - sleep: 1000 + - ai: Click somewhere on the player video area to close the settings popup + - sleep: 1000 + - aiAssert: There is no subtitle text overlay visible at the bottom area of the player diff --git a/apps/report/e2e/sidebar-interaction.yaml b/apps/report/e2e/sidebar-interaction.yaml new file mode 100644 index 0000000000..940e679170 --- /dev/null +++ b/apps/report/e2e/sidebar-interaction.yaml @@ -0,0 +1,17 @@ +target: + serve: ./dist + url: demo.html +tasks: + - name: clicking a task in sidebar exits replay mode and shows detail + flow: + - sleep: 3000 + - ai: Click on the first task row in the left sidebar execution list + - sleep: 2000 + - aiAssert: The right side shows a detail panel with task information, not the video player + - aiAssert: There is a screenshot image displayed in the main content area + + - name: clicking play button re-enters replay mode + flow: + - ai: Click the play icon button near the top of the sidebar to start replay + - sleep: 3000 + - aiAssert: The video player is visible with playback controls at the bottom diff --git a/apps/report/package.json b/apps/report/package.json index abc3946f64..268728ba22 100644 --- a/apps/report/package.json +++ b/apps/report/package.json @@ -8,7 +8,8 @@ "dev:rsdoctor": "RSDOCTOR=true rsbuild dev", "build:rsdoctor": "RSDOCTOR=true rsbuild build", "preview": "rsbuild preview", - "e2e": "node ../../packages/cli/bin/midscene ./e2e/" + "generate-demo": "node scripts/generate-demo-report.mjs", + "e2e": "node scripts/generate-demo-report.mjs && node ../../packages/cli/bin/midscene ./e2e/" }, "dependencies": { "@ant-design/icons": "^5.3.1", diff --git a/apps/report/scripts/generate-demo-report.mjs b/apps/report/scripts/generate-demo-report.mjs new file mode 100644 index 0000000000..fea0d13642 --- /dev/null +++ b/apps/report/scripts/generate-demo-report.mjs @@ -0,0 +1,70 @@ +/** + * Run a real Midscene YAML test to generate a report, then copy it as demo.html. + * This ensures the e2e tests validate against a genuinely generated report. + * + * Usage: node scripts/generate-demo-report.mjs + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const rootDir = path.join(__dirname, '..'); +const repoRoot = path.join(rootDir, '..', '..'); +const distDir = path.join(rootDir, 'dist'); + +// Resolve report directory the same way the runtime does (respects MIDSCENE_RUN_DIR) +const runDir = process.env.MIDSCENE_RUN_DIR || 'midscene_run'; +const reportDir = path.resolve(repoRoot, runDir, 'report'); + +// Record the latest report file before running +const reportsBefore = new Set( + fs.existsSync(reportDir) + ? fs.readdirSync(reportDir).filter((f) => f.endsWith('.html')) + : [], +); + +// Run the generation YAML which produces a real report +const yamlPath = path.join(rootDir, 'scripts', 'generate-report.yaml'); +const cliPath = path.join(repoRoot, 'packages', 'cli', 'bin', 'midscene'); + +console.log('Running Midscene test to generate report...'); +try { + execFileSync('node', [cliPath, yamlPath], { + cwd: repoRoot, + stdio: 'inherit', + env: { ...process.env }, + }); +} catch { + // Report is still generated even if some assertions fail + console.log( + 'Test exited with errors, but report may still have been generated.', + ); +} + +// Find the newly generated report +const reportsAfter = fs + .readdirSync(reportDir) + .filter((f) => f.endsWith('.html')); +const newReports = reportsAfter.filter((f) => !reportsBefore.has(f)); + +if (newReports.length === 0) { + console.error('No new report generated. Check if the test ran successfully.'); + process.exit(1); +} + +// Pick the most recent one +const latestReport = newReports + .map((f) => ({ + name: f, + mtime: fs.statSync(path.join(reportDir, f)).mtimeMs, + })) + .sort((a, b) => b.mtime - a.mtime)[0].name; + +const reportPath = path.join(reportDir, latestReport); +const demoPath = path.join(distDir, 'demo.html'); + +fs.mkdirSync(distDir, { recursive: true }); +fs.copyFileSync(reportPath, demoPath); +console.log(`Copied ${latestReport} -> dist/demo.html`); diff --git a/apps/report/scripts/generate-report.yaml b/apps/report/scripts/generate-report.yaml new file mode 100644 index 0000000000..4f528a805e --- /dev/null +++ b/apps/report/scripts/generate-report.yaml @@ -0,0 +1,13 @@ +# This YAML generates a real report for e2e testing. +# Keep tasks simple and stable to avoid flaky failures. +web: + url: https://www.saucedemo.com/ + +tasks: + - name: login to sauce demo + flow: + - ai: type 'standard_user' in user name input, type 'secret_sauce' in password, click 'Login' + + - name: verify inventory page + flow: + - aiAssert: The page shows a list of products with prices