Allow tests to be loaded asynchronously (microsoft#119537)

* initial wip in the extension host

* wip

* wip

* wip

* continued progress

* update api from discussion

* update for new proposal

* wip

* update to lastest testing api, almost everything working

Refs microsoft#115089
Design https://gist.github.com/connor4312/73f1883d720654834b7fd40550d3b6e0

* re-wire retirement state

* update actions to new async test structure

* minor cleanup

* remove unused es2018 that failed build

Author: connor4312

.eslintrc.json

@@ -982,6 +982,7 @@
 						"allowed": [
 							"FileSystemProvider",
 							"TreeDataProvider",
+							"TestProvider",
 							"CustomEditorProvider",
 							"CustomReadonlyEditorProvider",
 							"TerminalLinkProvider",
@@ -1015,6 +1016,7 @@
 							"override",
 							"receive",
 							"register",
+							"remove",
 							"rename",
 							"save",
 							"send",

src/vs/base/browser/ui/tree/asyncDataTree.ts

@@ -616,7 +616,7 @@ export class AsyncDataTree<TInput, T, TFilterData = void> implements IDisposable
 		return this.tree.isCollapsible(this.getDataNode(element));
 	}
 
-	isCollapsed(element: T): boolean {
+	isCollapsed(element: TInput | T): boolean {
 		return this.tree.isCollapsed(this.getDataNode(element));
 	}
 

src/vs/base/common/types.ts

@@ -50,6 +50,13 @@ export function isNumber(obj: unknown): obj is number {
 	return (typeof obj === 'number' && !isNaN(obj));
 }
 
+/**
+ * @returns whether the provided parameter is an Iterable, casting to the given generic
+ */
+export function isIterable<T>(obj: unknown): obj is Iterable<T> {
+	return !!obj && typeof (obj as any)[Symbol.iterator] === 'function';
+}
+
 /**
  * @returns whether the provided parameter is a JavaScript Boolean or not.
  */

src/vs/vscode.proposed.d.ts

@@ -2185,33 +2185,6 @@ declare module 'vscode' {
 		 * in {@link onDidChangeTest} when a test is added or removed.
 		 */
 		readonly root: T;
-
-		/**
-		 * An event that fires when an existing test `root` changes.  This can be
-		 * a result of a property update, or an update to its children. Changes
-		 * made to tests will not be visible to {@link TestObserver} instances
-		 * until this event is fired.
-		 *
-		 * When a change is signalled, VS Code will check for any new or removed
-		 * direct children of the changed ite, For example, firing the event with
-		 * the {@link testRoot} will detect any new children in `root.children`.
-		 */
-		readonly onDidChangeTest: Event<T>;
-
-		/**
-		 * Promise that should be resolved when all tests that are initially
-		 * defined have been discovered. The provider should continue to watch for
-		 * changes and fire `onDidChangeTest` until the hierarchy is disposed.
-		 */
-		readonly discoveredInitialTests?: Thenable<unknown>;
-
-		/**
-		 * An event that fires when a test becomes outdated, as a result of
-		 * file changes, for example. In "auto run" mode, tests that are outdated
-		 * will be automatically re-run after a short delay. Firing a test
-		 * with children will mark the entire subtree as outdated.
-		 */
-		readonly onDidInvalidateTest?: Event<T>;
 	}
 
 	/**
@@ -2233,8 +2206,9 @@ declare module 'vscode' {
 		 *
 		 * @param workspace The workspace in which to observe tests
 		 * @param cancellationToken Token that signals the used asked to abort the test run.
+		 * @returns the root test item for the workspace
 		 */
-		provideWorkspaceTestHierarchy(workspace: WorkspaceFolder, token: CancellationToken): ProviderResult<TestHierarchy<T>>;
+		provideWorkspaceTestRoot(workspace: WorkspaceFolder, token: CancellationToken): ProviderResult<T>;
 
 		/**
 		 * Requests that tests be provided for the given document. This will be
@@ -2246,18 +2220,21 @@ declare module 'vscode' {
 		 * saved, if possible.
 		 *
 		 * If the test system is not able to provide or estimate for tests on a
-		 * per-file basis, this method may not be implemented. In that case, VS
-		 * Code will request and use the information from the workspace hierarchy.
+		 * per-file basis, this method may not be implemented. In that case, the
+		 * editor will request and use the information from the workspace tree.
 		 *
 		 * @param document The document in which to observe tests
 		 * @param cancellationToken Token that signals the used asked to abort the test run.
+		 * @returns the root test item for the workspace
 		 */
-		provideDocumentTestHierarchy?(document: TextDocument, token: CancellationToken): ProviderResult<TestHierarchy<T>>;
+		provideDocumentTestRoot?(document: TextDocument, token: CancellationToken): ProviderResult<T>;
 
 		/**
+		 * @todo this will move out of the provider soon
+		 * @todo this will eventually need to be able to return a summary report, coverage for example.
+		 *
 		 * Starts a test run. This should cause {@link onDidChangeTest} to
 		 * fire with update test states during the run.
-		 * @todo this will eventually need to be able to return a summary report, coverage for example.
 		 * @param options Options for this test run
 		 * @param cancellationToken Token that signals the used asked to abort the test run.
 		 */
@@ -2305,18 +2282,55 @@ declare module 'vscode' {
 		setState(test: T, state: TestState): void;
 	}
 
+	export interface TestChildrenCollection<T> extends Iterable<T> {
+		/**
+		 * Gets the number of children in the collection.
+		 */
+		readonly size: number;
+
+		/**
+		 * Gets an existing TestItem by its ID, if it exists.
+		 * @param id ID of the test.
+		 * @returns the TestItem instance if it exists.
+		 */
+		get(id: string): T | undefined;
+
+		/**
+		 * Adds a new child test item. No-ops if the test was already a child.
+		 * @param child The test item to add.
+		 */
+		add(child: T): void;
+
+		/**
+		 * Removes the child test item by reference or ID from the collection.
+		 * @param child Child ID or instance to remove.
+		 */
+		delete(child: T | string): void;
+
+		/**
+		 * Removes all children from the collection.
+		 */
+		clear(): void;
+	}
+
 	/**
 	 * A test item is an item shown in the "test explorer" view. It encompasses
 	 * both a suite and a test, since they have almost or identical capabilities.
 	 */
-	export class TestItem {
+	export class TestItem<TChildren = any> {
 		/**
 		 * Unique identifier for the TestItem. This is used to correlate
 		 * test results and tests in the document with those in the workspace
 		 * (test explorer). This must not change for the lifetime of the TestItem.
 		 */
 		readonly id: string;
 
+		/**
+		 * A set of children this item has. You can add new children to it, which
+		 * will propagate to the editor UI.
+		 */
+		readonly children: TestChildrenCollection<TChildren>;
+
 		/**
 		 * Display name describing the test case.
 		 */
@@ -2327,6 +2341,12 @@ declare module 'vscode' {
 		 */
 		description?: string;
 
+		/**
+		 * Location of the test in the workspace. This is used to show line
+		 * decorations and code lenses for the test.
+		 */
+		location?: Location;
+
 		/**
 		 * Whether this test item can be run individually, defaults to `true`.
 		 *
@@ -2344,22 +2364,53 @@ declare module 'vscode' {
 		debuggable: boolean;
 
 		/**
-		 * Location of the test in the workspace. This is used to show line
-		 * decorations and code lenses for the test.
-		 */
-		location?: Location;
-
-		/**
-		 * Optional list of nested tests for this item.
+		 * Whether this test item can be expanded in the tree view, implying it
+		 * has (or may have) children. If this is given, the item may be
+		 * passed to the {@link TestHierarchy.getChildren} method.
 		 */
-		children: TestItem[];
+		expandable: boolean;
 
 		/**
 		 * Creates a new TestItem instance.
 		 * @param id Value of the "id" property
 		 * @param label Value of the "label" property.
+		 * @param parent Parent of this item. This should only be defined for the
+		 * test root.
+		 */
+		constructor(id: string, label: string, expandable: boolean);
+
+		/**
+		 * Marks the test as outdated. This can happen as a result of file changes,
+		 * for example. In "auto run" mode, tests that are outdated will be
+		 * automatically re-run after a short delay. Invoking this on a
+		 * test with children will mark the entire subtree as outdated.
+		 *
+		 * Extensions should generally not override this method.
+		 */
+		invalidate(): void;
+
+		/**
+		 * Requests the children of the test item. Extensions should override this
+		 * method for any test that can discover children.
+		 *
+		 * When called, the item should discover tests and update its's `children`.
+		 * The provider will be marked as 'busy' when this method is called, and
+		 * the provider should report `{ busy: false }` to {@link Progress.report}
+		 * once discovery is complete.
+		 *
+		 * The item should continue watching for changes to the children and
+		 * firing updates until the token is cancelled. The process of watching
+		 * the tests may involve creating a file watcher, for example.
+		 *
+		 * The editor will only call this method when it's interested in refreshing
+		 * the children of the item, and will not call it again while there's an
+		 * existing, uncancelled discovery for an item.
+		 *
+		 * @param token Cancellation for the request. Cancellation will be
+		 * requested if the test changes before the previous call completes.
+		 * @returns a provider result of child test items
 		 */
-		constructor(id: string, label: string);
+		discoverChildren(progress: Progress<{ busy: boolean }>, token: CancellationToken): void;
 	}
 
 	/**
@@ -2483,23 +2534,46 @@ declare module 'vscode' {
 		 * List of test results. The items in this array are the items that
 		 * were passed in the {@link test.runTests} method.
 		 */
-		results: ReadonlyArray<Readonly<TestItemWithResults>>;
+		results: ReadonlyArray<Readonly<TestResultSnapshot>>;
 	}
 
 	/**
-	 * A {@link TestItem} with an associated result, which appear or can be
-	 * provided in {@link TestResult} interfaces.
+	 * A {@link TestItem}-like interface with an associated result, which appear
+	 * or can be provided in {@link TestResult} interfaces.
 	 */
-	export interface TestItemWithResults extends TestItem {
+	export interface TestResultSnapshot {
+		/**
+		 * Unique identifier that matches that of the associated TestItem.
+		 * This is used to correlate test results and tests in the document with
+		 * those in the workspace (test explorer).
+		 */
+		readonly id: string;
+
+		/**
+		 * Display name describing the test case.
+		 */
+		readonly label: string;
+
+		/**
+		 * Optional description that appears next to the label.
+		 */
+		readonly description?: string;
+
+		/**
+		 * Location of the test in the workspace. This is used to show line
+		 * decorations and code lenses for the test.
+		 */
+		readonly location?: Location;
+
 		/**
 		 * Current result of the test.
 		 */
-		result: TestState;
+		readonly result: TestState;
 
 		/**
 		 * Optional list of nested tests for this item.
 		 */
-		children: Readonly<TestItemWithResults>[];
+		readonly children: Readonly<TestResultSnapshot>[];
 	}
 
 	//#endregion

src/vs/workbench/api/browser/mainThreadTesting.ts

@@ -11,14 +11,14 @@ import { Range } from 'vs/editor/common/core/range';
 import { extHostNamedCustomer } from 'vs/workbench/api/common/extHostCustomers';
 import { getTestSubscriptionKey, ISerializedTestResults, ITestState, RunTestsRequest, TestDiffOpType, TestsDiff } from 'vs/workbench/contrib/testing/common/testCollection';
 import { HydratedTestResult, ITestResultService, LiveTestResult } from 'vs/workbench/contrib/testing/common/testResultService';
-import { ITestService } from 'vs/workbench/contrib/testing/common/testService';
+import { ITestRootProvider, ITestService } from 'vs/workbench/contrib/testing/common/testService';
 import { ExtHostContext, ExtHostTestingResource, ExtHostTestingShape, IExtHostContext, MainContext, MainThreadTestingShape } from '../common/extHost.protocol';
 
 const reviveDiff = (diff: TestsDiff) => {
 	for (const entry of diff) {
 		if (entry[0] === TestDiffOpType.Add || entry[0] === TestDiffOpType.Update) {
 			const item = entry[1];
-			if (item.item.location) {
+			if (item.item?.location) {
 				item.item.location.uri = URI.revive(item.item.location.uri);
 				item.item.location.range = Range.lift(item.item.location.range);
 			}
@@ -27,7 +27,7 @@ const reviveDiff = (diff: TestsDiff) => {
 };
 
 @extHostNamedCustomer(MainContext.MainThreadTesting)
-export class MainThreadTesting extends Disposable implements MainThreadTestingShape {
+export class MainThreadTesting extends Disposable implements MainThreadTestingShape, ITestRootProvider {
 	private readonly proxy: ExtHostTestingShape;
 	private readonly testSubscriptions = new Map<string, IDisposable>();
 	private readonly testProviderRegistrations = new Map<string, IDisposable>();
@@ -56,7 +56,7 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 			}
 		}));
 
-		testService.updateRootProviderCount(1);
+		this._register(testService.registerRootProvider(this));
 
 		for (const { resource, uri } of this.testService.subscriptions) {
 			this.proxy.$subscribeToTests(resource, uri);
@@ -66,25 +66,14 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 	/**
 	 * @inheritdoc
 	 */
-	$publishExtensionProvidedResults(results: ISerializedTestResults, persist: boolean): void {
+	public $publishExtensionProvidedResults(results: ISerializedTestResults, persist: boolean): void {
 		this.resultService.push(new HydratedTestResult(results, persist));
 	}
 
 	/**
 	 * @inheritdoc
 	 */
-	$retireTest(extId: string): void {
-		for (const result of this.resultService.results) {
-			if (result instanceof LiveTestResult) {
-				result.retire(extId);
-			}
-		}
-	}
-
-	/**
-	 * @inheritdoc
-	 */
-	$updateTestStateInRun(runId: string, testId: string, state: ITestState): void {
+	public $updateTestStateInRun(runId: string, testId: string, state: ITestState): void {
 		const r = this.resultService.getResult(runId);
 		if (r && r instanceof LiveTestResult) {
 			for (const message of state.messages) {
@@ -105,6 +94,7 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 		const disposable = this.testService.registerTestController(id, {
 			runTests: (req, token) => this.proxy.$runTestsForProvider(req, token),
 			lookupTest: test => this.proxy.$lookupTest(test),
+			expandTest: (src, levels) => this.proxy.$expandTest(src, isFinite(levels) ? levels : -1),
 		});
 
 		this.testProviderRegistrations.set(id, disposable);
@@ -121,7 +111,7 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 	/**
 	 * @inheritdoc
 	 */
-	$subscribeToDiffs(resource: ExtHostTestingResource, uriComponents: UriComponents): void {
+	public $subscribeToDiffs(resource: ExtHostTestingResource, uriComponents: UriComponents): void {
 		const uri = URI.revive(uriComponents);
 		const disposable = this.testService.subscribeToDiffs(resource, uri,
 			diff => this.proxy.$acceptDiff(resource, uriComponents, diff));
@@ -152,7 +142,6 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 
 	public dispose() {
 		super.dispose();
-		this.testService.updateRootProviderCount(-1);
 		for (const subscription of this.testSubscriptions.values()) {
 			subscription.dispose();
 		}