diff --git a/pubspec.yaml b/pubspec.yaml index cfd5e1333d..01c281ba43 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -27,7 +27,10 @@ dev_dependencies: dart_style: ^3.0.0 lints: ">=5.0.0 <7.0.0" matcher: ^0.12.15 + puppeteer: ^3.18.0 sass: ^1.87.0 + shelf: ^1.4.2 + shelf_static: ^1.1.3 test: ^1.24.2 test_descriptor: ^2.0.1 test_process: ^2.0.3 @@ -36,3 +39,6 @@ dev_dependencies: executables: dartdoc: null + +dependency_overrides: + archive: ^4.0.0 diff --git a/test/end2end/dartdoc_test.dart b/test/end2end/dartdoc_test.dart index 6f0f14b8b5..8aeea3af40 100644 --- a/test/end2end/dartdoc_test.dart +++ b/test/end2end/dartdoc_test.dart @@ -2,6 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:io'; + import 'package:analyzer/file_system/file_system.dart'; import 'package:dartdoc/src/dartdoc.dart' show Dartdoc, DartdocResults; import 'package:dartdoc/src/dartdoc_options.dart'; @@ -15,9 +17,13 @@ import 'package:dartdoc/src/package_config_provider.dart'; import 'package:dartdoc/src/package_meta.dart'; import 'package:dartdoc/src/warnings.dart'; import 'package:path/path.dart' as path; +import 'package:shelf/shelf_io.dart'; +import 'package:shelf_static/shelf_static.dart'; import 'package:test/test.dart'; import '../src/utils.dart'; +import 'screenshot_utils.dart'; +import 'test_browser.dart'; final _resourceProvider = pubPackageMetaProvider.resourceProvider; final _pathContext = _resourceProvider.pathContext; @@ -51,6 +57,8 @@ void main() { // Set up the pub metadata for our test packages. runPubGet(testPackageToolError.path); runPubGet(_testSkyEnginePackage.path); + + if (isScreenshotDirSet) {} }); setUp(() async { @@ -203,6 +211,41 @@ void main() { expect(level2.exists, isTrue); expect(level2.readAsStringSync(), contains('')); + + if (isScreenshotDirSet) { + final server = await _setupStaticHttpServer(tempDir.path); + final browser = TestBrowser(origin: 'http://localhost:${server.port}'); + final allPaths = Directory(tempDir.path) + .listSync(recursive: true, followLinks: true) + .map((f) => path.relative(f.path, from: tempDir.path)) + .where((p) => p.endsWith('.html')) + .toList(); + final paths = [ + 'index.html', + 'ex/index.html', + 'ex/HtmlInjection-class.html', + 'ex/IntSet/sum.html', + ]; + assert(paths.every(allPaths.contains)); + try { + await browser.startBrowser(); + final session = await browser.createSession(); + await session.withPage(fn: (page) async { + for (final p in paths) { + await page.gotoOrigin('/$p'); + final prefix = p.replaceAll('.html', '').replaceAll('.', '-'); + await page.takeScreenshots(selector: 'body', prefix: prefix); + } + }); + } finally { + await server.close(); + await browser.close(); + } + } }); }, timeout: Timeout.factor(12)); } + +Future _setupStaticHttpServer(String path) async { + return await serve(createStaticHandler(path), 'localhost', 0); +} diff --git a/test/end2end/screenshot_utils.dart b/test/end2end/screenshot_utils.dart new file mode 100644 index 0000000000..2a1e470029 --- /dev/null +++ b/test/end2end/screenshot_utils.dart @@ -0,0 +1,110 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:puppeteer/puppeteer.dart'; + +// Default screen with 16:10 ratio. +final desktopDeviceViewport = DeviceViewport(width: 1280, height: 800); + +final _screenshotDir = Platform.environment['SCREENSHOT_DIR']; +final isScreenshotDirSet = _screenshotDir != null && _screenshotDir!.isNotEmpty; + +// Set this variable to enable screenshot files to be updated with new takes. +// The default is to throw an exception to prevent accidental overrides from +// separate tests. +final _allowScreeshotUpdates = Platform.environment['SCREENSHOT_UPDATE'] == '1'; + +// Note: The default values are the last, so we don't need reset +// the original values after taking the screenshots. +final _themes = ['dark', 'light']; +final _viewports = { + 'mobile': DeviceViewport(width: 400, height: 800), + 'tablet': DeviceViewport(width: 768, height: 1024), + 'desktop': desktopDeviceViewport, +}; + +extension ScreenshotPageExt on Page { + Future writeScreenshotToFile(String path) async { + await File(path).writeAsBytes(await screenshot()); + } + + /// Takes screenshots **if** `SCREENSHOT_DIR` environment variable is set. + /// + /// Iterates over viewports and themes, and generates screenshot files with the + /// following pattern: + /// - `SCREENSHOT_DIR/$prefix-desktop-dark.png` + /// - `SCREENSHOT_DIR/$prefix-desktop-light.png` + /// - `SCREENSHOT_DIR/$prefix-mobile-dark.png` + /// - `SCREENSHOT_DIR/$prefix-mobile-light.png` + /// - `SCREENSHOT_DIR/$prefix-tablet-dark.png` + /// - `SCREENSHOT_DIR/$prefix-tablet-light.png` + Future takeScreenshots({ + required String selector, + required String prefix, + }) async { + final handle = await $(selector); + await handle.takeScreenshots(prefix); + } +} + +extension ScreenshotElementHandleExt on ElementHandle { + /// Takes screenshots **if** `SCREENSHOT_DIR` environment variable is set. + /// + /// Iterates over viewports and themes, and generates screenshot files with the + /// following pattern: + /// - `SCREENSHOT_DIR/$prefix-desktop-dark.png` + /// - `SCREENSHOT_DIR/$prefix-desktop-light.png` + /// - `SCREENSHOT_DIR/$prefix-mobile-dark.png` + /// - `SCREENSHOT_DIR/$prefix-mobile-light.png` + /// - `SCREENSHOT_DIR/$prefix-tablet-dark.png` + /// - `SCREENSHOT_DIR/$prefix-tablet-light.png` + Future takeScreenshots(String prefix) async { + final body = await page.$('body'); + final bodyClassAttr = (await body + .evaluate('el => el.getAttribute("class")')) as String; + final bodyClasses = [ + ...bodyClassAttr.split(' '), + '--ongoing-screenshot', + ]; + + for (final vp in _viewports.entries) { + await page.setViewport(vp.value); + + for (final theme in _themes) { + final newClasses = [ + ...bodyClasses.where((c) => !c.endsWith('-theme')), + '$theme-theme', + ]; + await body.evaluate('(el, v) => el.setAttribute("class", v)', + args: [newClasses.join(' ')]); + + // The presence of the element is verified, continue only if screenshots are enabled. + if (!isScreenshotDirSet) continue; + + // Arbitrary delay in the hope that potential ongoing updates complete. + await Future.delayed(Duration(milliseconds: 500)); + + final path = p.join(_screenshotDir!, '$prefix-${vp.key}-$theme.png'); + await _writeScreenshotToFile(path); + } + } + + // restore the original body class attributes + await body.evaluate('(el, v) => el.setAttribute("class", v)', + args: [bodyClassAttr]); + } + + Future _writeScreenshotToFile(String path) async { + final file = File(path); + final exists = file.existsSync(); + if (exists && !_allowScreeshotUpdates) { + throw Exception('Screenshot update is detected in: $path'); + } + await file.parent.create(recursive: true); + await File(path).writeAsBytes(await screenshot()); + } +} diff --git a/test/end2end/test_browser.dart b/test/end2end/test_browser.dart new file mode 100644 index 0000000000..0c4bd8b97e --- /dev/null +++ b/test/end2end/test_browser.dart @@ -0,0 +1,360 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:puppeteer/puppeteer.dart'; + +import 'screenshot_utils.dart'; + +/// Creates and tracks the headless Chrome environment, its temp directories and +/// and uncaught exceptions. +class TestBrowser { + final String? _testName; + final String _origin; + final String? _coverageDir; + final Directory _tempDir; + final bool _displayBrowser; + Browser? _browser; + late final _trackCoverage = + _coverageDir != null || Platform.environment.containsKey('COVERAGE'); + final _trackedSessions = []; + + /// The coverage report of JavaScript files. + final _jsCoverages = {}; + + /// The coverage report of CSS files. + final _cssCoverages = {}; + + TestBrowser({ + required String origin, + String? testName, + String? coverageDir, + bool displayBrowser = false, + }) : _displayBrowser = displayBrowser, + _testName = testName, + _origin = origin, + _coverageDir = coverageDir ?? Platform.environment['COVERAGE_DIR'], + _tempDir = Directory.systemTemp.createTempSync('puppeteer'); + + Future _detectChromeBinary() async { + // TODO: scan $PATH + // check hardcoded values + final binaries = [ + '/usr/bin/google-chrome', + ]; + + for (final binary in binaries) { + if (!File(binary).existsSync()) { + continue; + } + try { + // Get the local chrome's main version + final pr = await Process.run(binary, ['--version']) + .timeout(Duration(seconds: 5)); + final output = pr.stdout.toString(); + final mainVersion = output + .split(' ') + .expand((p) => p.split('.')) + .map(int.tryParse) + .whereType() + .firstOrNull; + // No version string, better not running it. + if (mainVersion == null) continue; + // Main version is after the currently supported version, will fail. + // https://github.com/xvrh/puppeteer-dart/issues/364 + if (mainVersion >= 132) continue; + + return binary; + } on TimeoutException catch (_) { + print('Unable to detect chrome version with $binary'); + continue; + } + } + + // Otherwise let puppeteer download a chrome in the local .dart_tool directory: + final r = await downloadChrome(cachePath: '.dart_tool/puppeteer/chromium'); + return r.executablePath; + } + + Future startBrowser() async { + if (_browser != null) return; + final chromeBin = await _detectChromeBinary(); + final userDataDir = await _tempDir.createTemp('user'); + _browser = await puppeteer.launch( + executablePath: chromeBin, + args: [ + '--lang=en-US,en', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-accelerated-2d-canvas', + '--disable-gpu', + ], + noSandboxFlag: true, + userDataDir: userDataDir.path, + headless: !_displayBrowser, + devTools: false, + defaultViewport: desktopDeviceViewport, + ); + + // Update the default permissions like clipboard access. + await _browser!.defaultBrowserContext + .overridePermissions(_origin, [PermissionType.clipboardReadWrite]); + } + + Future createSession() async { + final incognito = await _browser!.createIncognitoBrowserContext(); + await incognito + .overridePermissions(_origin, [PermissionType.clipboardReadWrite]); + final session = TestBrowserSession(this, incognito); + _trackedSessions.add(session); + return session; + } + + Future close() async { + for (final s in _trackedSessions) { + await s.close(); + } + await _browser!.close(); + + _printCoverage(); + if (_coverageDir != null) { + await _saveCoverage(p.join(_coverageDir, 'puppeteer')); + } + await _tempDir.delete(recursive: true); + } + + void _printCoverage() { + for (final c in _jsCoverages.values) { + print('${c.url}: ${c.percent.toStringAsFixed(2)}%'); + } + for (final c in _cssCoverages.values) { + print('${c.url}: ${c.percent.toStringAsFixed(2)}%'); + } + } + + Future _saveCoverage(String outputDir) async { + Future saveToFile(Map map, String path) async { + if (map.isNotEmpty) { + final file = File(path); + await file.parent.create(recursive: true); + await file.writeAsString(json.encode(map.map( + (k, v) => MapEntry( + v.url, + { + 'textLength': v.textLength, + 'ranges': v._coveredRanges.map((r) => r.toJson()).toList(), + }, + ), + ))); + } + } + + final outputFileName = _testName ?? _generateTestName(); + await saveToFile(_jsCoverages, '$outputDir/$outputFileName.js.json'); + await saveToFile(_cssCoverages, '$outputDir/$outputFileName.css.json'); + } +} + +class TestBrowserSession { + final TestBrowser _browser; + final BrowserContext _context; + final _trackedPages = []; + + TestBrowserSession(this._browser, this._context); + + /// Creates a new page and setup overrides and tracking. + Future withPage({ + required Future Function(Page page) fn, + }) async { + final clientErrors = []; + final serverErrors = []; + final page = await _context.newPage(); + _pageOriginExpando[page] = _browser._origin; + await page.setRequestInterception(true); + if (_browser._trackCoverage) { + await page.coverage.startJSCoverage(resetOnNavigation: false); + await page.coverage.startCSSCoverage(resetOnNavigation: false); + } + + page.onRequest.listen((rq) async { + // soft-abort + if (rq.url.startsWith('https://www.google-analytics.com/') || + rq.url.startsWith('https://www.googletagmanager.com/') || + rq.url.startsWith('https://www.google.com/insights') || + rq.url.startsWith( + 'https://www.gstatic.com/brandstudio/kato/cookie_choice_component/')) { + // reduce log error by replying with empty JS content + if (rq.url.endsWith('.js') || rq.url.contains('.js?')) { + await rq.respond( + status: 200, + body: '{}', + contentType: 'application/javascript', + ); + } else { + await rq.abort(error: ErrorReason.failed); + } + return; + } + // ignore + if (rq.url.startsWith('data:')) { + await rq.continueRequest(); + return; + } + + final uri = Uri.parse(rq.url); + if (uri.path.contains('//')) { + serverErrors.add('Double-slash URL detected: "${rq.url}".'); + } + + await rq.continueRequest(); + }); + + page.onResponse.listen((rs) async { + if (rs.status >= 500) { + serverErrors + .add('${rs.status} ${rs.statusText} received on ${rs.request.url}'); + } else if (rs.status >= 400) { + serverErrors + .add('${rs.status} ${rs.statusText} received on ${rs.request.url}'); + } + + final contentType = rs.headers[HttpHeaders.contentTypeHeader]; + if (contentType == null || contentType.isEmpty) { + serverErrors + .add('Content type header is missing for ${rs.request.url}.'); + } + }); + + // print console messages + page.onConsole.listen(print); + + // print and store uncaught errors + page.onError.listen((e) { + print('Client error: $e'); + clientErrors.add(e); + }); + + _trackedPages.add(page); + + try { + final r = await fn(page); + if (clientErrors.isNotEmpty) { + throw Exception('Client errors detected: ${clientErrors.first}'); + } + if (serverErrors.isNotEmpty) { + throw Exception('Server errors detected: ${serverErrors.first}'); + } + return r; + } finally { + await _closePage(page); + } + } + + /// Gets tracking results of [page] and closes it. + Future _closePage(Page page) async { + if (_browser._trackCoverage) { + final jsEntries = await page.coverage.stopJSCoverage(); + for (final e in jsEntries) { + _browser._jsCoverages[e.url] ??= _Coverage(e.url); + _browser._jsCoverages[e.url]!.textLength = e.text.length; + _browser._jsCoverages[e.url]!.addRanges(e.ranges); + } + + final cssEntries = await page.coverage.stopCSSCoverage(); + for (final e in cssEntries) { + _browser._cssCoverages[e.url] ??= _Coverage(e.url); + _browser._cssCoverages[e.url]!.textLength = e.text.length; + _browser._cssCoverages[e.url]!.addRanges(e.ranges); + } + } + + await page.close(); + _trackedPages.remove(page); + } + + Future close() async { + if (_trackedPages.isNotEmpty) { + throw StateError('There are tracked pages with pending coverage report.'); + } + await _context.close(); + } +} + +String _generateTestName() { + return [ + p.basenameWithoutExtension(Platform.script.path), + DateTime.now().microsecondsSinceEpoch, + ProcessInfo.currentRss, + ].join('-'); +} + +/// Stores the origin URL on the page. +final _pageOriginExpando = Expando(); + +extension PageExt on Page { + /// The base URL of the website. + String get origin => _pageOriginExpando[this]!; + + /// Visits the [path] relative to the origin. + Future gotoOrigin(String path) async { + return await goto('$origin$path', wait: Until.networkIdle); + } + + /// Returns the [property] value of the first element by [selector]. + Future propertyValue(String selector, String property) async { + final h = await $(selector); + return await h.propertyValue(property); + } +} + +extension ElementHandleExt on ElementHandle { + Future textContent() async { + return await propertyValue('textContent'); + } + + Future attributeValue(String name) async { + return await evaluate('el => el.getAttribute("$name")'); + } +} + +/// Track the covered ranges in the source file. +class _Coverage { + final String url; + int? textLength; + + /// List of start-end ranges that were covered in the source file during the + /// execution of the app. + List _coveredRanges = []; + + _Coverage(this.url); + + void addRanges(List ranges) { + final list = [..._coveredRanges, ...ranges]; + // sort by start position first, and if they are matching, sort by end position + list.sort((a, b) { + final x = a.start.compareTo(b.start); + return x == 0 ? a.end.compareTo(b.end) : x; + }); + // merge ranges + _coveredRanges = list.fold>([], (m, range) { + if (m.isEmpty || m.last.end < range.start) { + m.add(range); + } else { + final last = m.removeLast(); + m.add(Range(last.start, range.end)); + } + return m; + }); + } + + double get percent { + final coveredPosition = + _coveredRanges.fold(0, (sum, r) => sum + r.end - r.start); + return coveredPosition * 100 / textLength!; + } +}