feat: Local pub server (#2919)

* ci: Fix db_common tests on Windows

* Use powershell

* feat: Add `pub_server` package

Adds a minimal package for running a local, embedded pub server.

* chore: Improve license script

Running license script without `addlicense` installed should not leave a bunch of files around and should be runnable without sudo

* feat(pub_server): Add launcher

Adds a mechanism for seeding the local pub server with either the packages of a local directory or of a remote git repo.

* feat(aft): Add `serve` command

Adds a command for serving the repo's packages on a local pub server.

* chore(pub_server): Clean up

* ci: Add `pub_server` workflow

* chore(pub_server): Add more doc comments

* test(pub_server): Add e2e tests

Author: dnys1

.github/dependabot.yaml

@@ -74,6 +74,7 @@ updates:
           - "version-update:semver-patch"
       - dependency-name: "aws_common"
       - dependency-name: "amplify_lints"
+      - dependency-name: "pub_server"
       - dependency-name: "smithy"
       - dependency-name: "smithy_codegen"
       - dependency-name: "aws_signature_v4"
@@ -1414,6 +1415,17 @@ updates:
       - dependency-name: "aws_common"
       - dependency-name: "amplify_lints"
       - dependency-name: "aws_signature_v4"
+  - package-ecosystem: "pub"
+    directory: "packages/test/pub_server"
+    schedule:
+      interval: "daily"
+    ignore:
+      # Ignore patch version bumps
+      - dependency-name: "*"
+        update-types:
+          - "version-update:semver-patch"
+      - dependency-name: "aws_common"
+      - dependency-name: "amplify_lints"
   - package-ecosystem: "pub"
     directory: "packages/worker_bee/e2e"
     schedule:

.github/workflows/pub_server.yaml

@@ -0,0 +1,39 @@
+# Generated with aft. To update, run: `aft generate workflows`
+name: pub_server
+on:
+  push:
+    branches:
+      - main
+      - stable
+  pull_request:
+    paths:
+      - '.github/workflows/dart_native.yaml'
+      - '.github/workflows/dart_vm.yaml'
+      - '.github/workflows/pub_server.yaml'
+      - 'packages/amplify_lints/lib/**/*.yaml'
+      - 'packages/amplify_lints/pubspec.yaml'
+      - 'packages/aws_common/lib/**/*.dart'
+      - 'packages/aws_common/pubspec.yaml'
+      - 'packages/test/pub_server/**/*.dart'
+      - 'packages/test/pub_server/**/*.yaml'
+      - 'packages/test/pub_server/lib/**/*'
+      - 'packages/test/pub_server/test/**/*'
+  schedule:
+    - cron: "0 0 * * 0" # Every Sunday at 00:00
+defaults:
+  run:
+    shell: bash
+permissions: read-all
+
+jobs:
+  test:
+    uses: ./.github/workflows/dart_vm.yaml
+    with:
+      package-name: pub_server
+      working-directory: packages/test/pub_server
+  native_test:
+    needs: test
+    uses: ./.github/workflows/dart_native.yaml
+    with:
+      package-name: pub_server
+      working-directory: packages/test/pub_server

packages/aft/lib/aft.dart

@@ -15,6 +15,7 @@ export 'src/commands/link_command.dart';
 export 'src/commands/list_packages_command.dart';
 export 'src/commands/publish_command.dart';
 export 'src/commands/run_command.dart';
+export 'src/commands/serve_command.dart';
 export 'src/commands/version_bump_command.dart';
 export 'src/models.dart';
 export 'src/pub/pub_runner.dart';

packages/aft/lib/src/command_runner.dart

@@ -31,7 +31,8 @@ Future<void> run(List<String> args) async {
     ..addCommand(CreateCommand())
     ..addCommand(SaveRepoStateCommand())
     ..addCommand(RunCommand())
-    ..addCommand(DocsCommand());
+    ..addCommand(DocsCommand())
+    ..addCommand(ServeCommand());
 
   try {
     final argResults = runner.argParser.parse(args);

packages/aft/lib/src/commands/publish_command.dart

@@ -11,38 +11,39 @@ import 'package:collection/collection.dart';
 import 'package:graphs/graphs.dart';
 import 'package:path/path.dart' as p;
 
-/// Command to publish all Dart/Flutter packages in the repo.
-class PublishCommand extends AmplifyCommand with GlobOptions {
-  PublishCommand() {
-    argParser
-      ..addFlag(
-        'force',
-        abbr: 'f',
-        help: 'Ignores errors in pre-publishing commands and publishes '
-            'without prompt',
-        negatable: false,
-      )
-      ..addFlag(
-        'dry-run',
-        help: 'Passes `--dry-run` flag to `dart` or `flutter` publish command',
-        negatable: false,
-      );
-  }
+/// Helpers for commands which publish packages to `pub.dev`.
+mixin PublishHelpers on AmplifyCommand {
+  bool get dryRun;
+  bool get force;
 
-  late final bool force = argResults!['force'] as bool;
-  late final bool dryRun = argResults!['dry-run'] as bool;
+  /// Gathers the subset of packages which are publishable and whose latest
+  /// version is not already available on `pub.dev`.
+  Future<List<PackageInfo>> unpublishedPackages(
+    List<PackageInfo> publishablePackages,
+  ) async {
+    final unpublishedPackages = (await Future.wait([
+      for (final package in publishablePackages) checkPublishable(package),
+    ]))
+        .whereType<PackageInfo>()
+        .toList();
 
-  @override
-  String get description =>
-      'Publishes all packages in the Amplify Flutter repo which '
-      'need publishing.';
+    try {
+      sortPackagesTopologically<PackageInfo>(
+        unpublishedPackages,
+        (pkg) => pkg.pubspecInfo.pubspec,
+      );
+    } on CycleException<dynamic> {
+      if (!force) {
+        exitError('Cannot sort packages with inter-dependencies.');
+      }
+    }
 
-  @override
-  String get name => 'publish';
+    return unpublishedPackages;
+  }
 
   /// Checks if [package] can be published based on whether the local version
   /// is newer than the one published to `pub.dev`.
-  Future<PackageInfo?> _checkPublishable(PackageInfo package) async {
+  Future<PackageInfo?> checkPublishable(PackageInfo package) async {
     final publishTo = package.pubspecInfo.pubspec.publishTo;
     if (publishTo == 'none') {
       return null;
@@ -61,8 +62,8 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
       final publishedVersion = maxBy(
         [
           if (versionInfo?.latestPrerelease != null)
-            versionInfo?.latestPrerelease!,
-          if (versionInfo?.latestVersion != null) versionInfo?.latestVersion!,
+            versionInfo!.latestPrerelease!,
+          if (versionInfo?.latestVersion != null) versionInfo!.latestVersion!,
         ],
         (v) => v,
       );
@@ -85,7 +86,7 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
 
   /// Runs pre-publish operations for [package], most importantly any necessary
   /// `build_runner` tasks.
-  Future<void> _prePublish(PackageInfo package) async {
+  Future<void> prePublish(PackageInfo package) async {
     logger.info('Running pre-publish checks for ${package.name}...');
     if (!dryRun) {
       // Remove any overrides so that `pub` commands resolve against
@@ -136,7 +137,7 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
   static final _validationErrorRegex = RegExp(r'^\s*\*');
 
   /// Publishes the package using `pub`.
-  Future<void> _publish(PackageInfo package) async {
+  Future<void> publish(PackageInfo package) async {
     logger.info('Publishing ${package.name}${dryRun ? ' (dry run)' : ''}...');
     final publishCmd = await Process.start(
       package.flavor.entrypoint,
@@ -181,21 +182,49 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
       }
     }
   }
+}
+
+/// Command to publish all Dart/Flutter packages in the repo.
+class PublishCommand extends AmplifyCommand with GlobOptions, PublishHelpers {
+  PublishCommand() {
+    argParser
+      ..addFlag(
+        'force',
+        abbr: 'f',
+        help: 'Ignores errors in pre-publishing commands and publishes '
+            'without prompt',
+        negatable: false,
+      )
+      ..addFlag(
+        'dry-run',
+        help: 'Passes `--dry-run` flag to `dart` or `flutter` publish command',
+        negatable: false,
+      );
+  }
+
+  @override
+  late final bool force = argResults!['force'] as bool;
+
+  @override
+  late final bool dryRun = argResults!['dry-run'] as bool;
+
+  @override
+  String get description =>
+      'Publishes all packages in the Amplify Flutter repo which '
+      'need publishing.';
+
+  @override
+  String get name => 'publish';
 
   @override
   Future<void> run() async {
     await super.run();
     // Gather packages which can be published.
-    final publishablePackages = repo
-        .publishablePackages(commandPackages)
-        .where((pkg) => pkg.pubspecInfo.pubspec.publishTo != 'none');
+    final publishablePackages = repo.publishablePackages(commandPackages);
 
     // Gather packages which need to be published.
-    final packagesNeedingPublish = (await Future.wait([
-      for (final package in publishablePackages) _checkPublishable(package),
-    ]))
-        .whereType<PackageInfo>()
-        .toList();
+    final packagesNeedingPublish =
+        await unpublishedPackages(publishablePackages);
 
     // Publishable packages which are being held back.
     final unpublishablePackages = publishablePackages.where(
@@ -207,17 +236,6 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
       return;
     }
 
-    try {
-      sortPackagesTopologically<PackageInfo>(
-        packagesNeedingPublish,
-        (pkg) => pkg.pubspecInfo.pubspec,
-      );
-    } on CycleException<dynamic> {
-      if (!force) {
-        exitError('Cannot sort packages with inter-dependencies.');
-      }
-    }
-
     stdout
       ..writeln('Preparing to publish${dryRun ? ' (dry run)' : ''}: ')
       ..writeln(
@@ -247,8 +265,8 @@ class PublishCommand extends AmplifyCommand with GlobOptions {
     // some packages will not be published, it also means that the command
     // can be re-run to pick up where it left off.
     for (final package in packagesNeedingPublish) {
-      await _prePublish(package);
-      await _publish(package);
+      await prePublish(package);
+      await publish(package);
     }
 
     stdout.writeln(

packages/aft/lib/src/commands/serve_command.dart

@@ -0,0 +1,108 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import 'dart:io';
+
+import 'package:aft/aft.dart';
+import 'package:aft/src/options/glob_options.dart';
+import 'package:async/async.dart';
+import 'package:pub_server/pub_server.dart';
+import 'package:shelf/shelf_io.dart' as io;
+
+/// A command for serving all packages in the Amplify Flutter repo on a local
+/// pub server.
+class ServeCommand extends AmplifyCommand with GlobOptions, PublishHelpers {
+  ServeCommand() {
+    argParser.addOption(
+      'port',
+      abbr: 'p',
+      defaultsTo: '0',
+      help: 'The port to serve on.',
+    );
+  }
+
+  /// The port to serve on.
+  late final int port = int.parse(argResults!['port'] as String);
+
+  @override
+  bool get force => true;
+
+  @override
+  bool get dryRun => false;
+
+  @override
+  String get description => 'Serves all packages in the Amplify Flutter repo '
+      'on a local pub server.';
+
+  @override
+  String get name => 'serve';
+
+  @override
+  Future<PackageInfo?> checkPublishable(PackageInfo package) async {
+    final publishTo = package.pubspecInfo.pubspec.publishTo;
+    if (publishTo == 'none') {
+      return null;
+    }
+    return package;
+  }
+
+  @override
+  Future<void> run() async {
+    await super.run();
+    await linkPackages();
+
+    final pubServer = await io.serve(
+      PubServer.prod().handler,
+      InternetAddress.anyIPv4,
+      port,
+    );
+    final pubServerUri = Uri.parse('http://localhost:${pubServer.port}');
+
+    // Gather packages which can be published.
+    final publishablePackages = repo.publishablePackages(commandPackages);
+
+    // Publish packages to local pub server.
+    final packagesNeedingPublish =
+        await unpublishedPackages(publishablePackages);
+    final launcherPackages = packagesNeedingPublish
+        .map(
+          (pkg) => LocalPackage(
+            name: pkg.name,
+            path: pkg.path,
+            pubspec: pkg.pubspecInfo.pubspec,
+          ),
+        )
+        .toList();
+    final launcher = AmplifyPubLauncher(
+      pubServerUri,
+      launcherPackages,
+      prePublish: (LocalPackage pkg) {
+        final package = repo.allPackages[pkg.name]!;
+        return prePublish(package);
+      },
+    );
+    await launcher.run();
+
+    logger.info('Serving packages on $pubServerUri. Press Ctrl+C to exit.');
+
+    await StreamGroup.merge([
+      ProcessSignal.sigint.watch(),
+      ProcessSignal.sigterm.watch(),
+    ]).first;
+
+    await pubServer.close(force: true);
+  }
+}
+
+class AmplifyPubLauncher extends PubLauncher {
+  AmplifyPubLauncher(
+    super.pubServerUri,
+    super.packages, {
+    required Future<void> Function(LocalPackage) prePublish,
+  }) : _prePublish = prePublish;
+
+  final Future<void> Function(LocalPackage) _prePublish;
+
+  @override
+  Future<void> prePublish(LocalPackage package) => _prePublish(package);
+}

packages/aft/lib/src/models.dart

@@ -20,6 +20,7 @@ const falsePositiveExamples = [
   'amplify_auth_cognito_test',
   'amplify_secure_storage_test',
   'amplify_native_legacy_wrapper',
+  'pub_server',
 
   // Smithy Golden packages
   'aws_json1_0_v1',

packages/aft/pubspec.yaml

@@ -31,6 +31,8 @@ dependencies:
   path: any
   pub_api_client: ^2.4.0
   pub_semver: ^2.1.1
+  pub_server:
+    path: ../test/pub_server
   pubspec_parse: ^1.2.0
   shelf: ^1.4.0
   shelf_static: ^1.1.1