Skip to content

Commit 2f6cf99

Browse files
araujoguiaraujogui
committed
fix: remove thread pool loop
1 parent 2f1cb75 commit 2f6cf99

File tree

3 files changed

+163
-158
lines changed

3 files changed

+163
-158
lines changed

‎bin/commands/generate.mjs

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -139,20 +139,16 @@ export default {
139139

140140
const releases = await getAllMajors();
141141

142-
await Promise.all(
143-
docs.map(async doc => {
144-
const { runGenerators } = createGenerator(doc);
142+
const { runGenerators } = createGenerator(docs);
145143

146-
await runGenerators({
147-
generators: opts.target,
148-
input: opts.input,
149-
output: opts.output && resolve(opts.output),
150-
version: coerce(opts.version),
151-
releases,
152-
gitRef: opts.gitRef,
153-
threads: parseInt(opts.threads, 10),
154-
});
155-
})
156-
);
144+
await runGenerators({
145+
generators: opts.target,
146+
input: opts.input,
147+
output: opts.output && resolve(opts.output),
148+
version: coerce(opts.version),
149+
releases,
150+
gitRef: opts.gitRef,
151+
threads: parseInt(opts.threads, 10),
152+
});
157153
},
158154
};

‎src/generators/metadata/index.mjs

Lines changed: 5 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,11 @@
11
'use strict';
22

3-
import { u as createTree } from 'unist-builder';
4-
import { findAfter } from 'unist-util-find-after';
5-
import { remove } from 'unist-util-remove';
6-
import { selectAll } from 'unist-util-select';
7-
import { SKIP, visit } from 'unist-util-visit';
8-
9-
import { createNodeSlugger } from '../../utils/slugger/index.mjs';
10-
import { getRemark } from '../../utils/remark.mjs';
11-
import createQueries from '../../utils/queries/index.mjs';
12-
import createMetadata from '../../metadata.mjs';
3+
import { parseApiDoc } from './utils/parse.mjs';
134

145
/**
156
* This generator generates a flattened list of metadata entries from a API doc
167
*
17-
* @typedef {ParserOutput<import('mdast').Root>} Input
8+
* @typedef {ParserOutput<import('mdast').Root>[]} Input
189
*
1910
* @type {GeneratorMetadata<Input, ApiDocMetadataEntry[]>}
2011
*/
@@ -28,140 +19,10 @@ export default {
2819
dependsOn: 'ast',
2920

3021
/**
31-
* @param {Input} input
22+
* @param {Input} inputs
3223
* @returns {Promise<ApiDocMetadataEntry[]>}
3324
*/
34-
async generate({ file, tree }) {
35-
const {
36-
setHeadingMetadata,
37-
addYAMLMetadata,
38-
updateMarkdownLink,
39-
updateTypeReference,
40-
updateLinkReference,
41-
addStabilityMetadata,
42-
} = createQueries();
43-
44-
// Creates an instance of the Remark processor with GFM support
45-
// which is used for stringifying the AST tree back to Markdown
46-
const remarkProcessor = getRemark();
47-
48-
/**
49-
* This holds references to all the Metadata entries for a given file
50-
* this is used so we can traverse the AST tree and keep mutating things
51-
* and then stringify the whole api doc file at once without creating sub traversals
52-
*
53-
* Then once we have the whole file parsed, we can split the resulting string into sections
54-
* and seal the Metadata Entries (`.create()`) and return the result to the caller of parae.
55-
*
56-
* @type {Array<ApiDocMetadataEntry>}
57-
*/
58-
const metadataCollection = [];
59-
60-
// Creates a new Slugger instance for the current API doc file
61-
const nodeSlugger = createNodeSlugger();
62-
63-
// Get all Markdown Footnote definitions from the tree
64-
const markdownDefinitions = selectAll('definition', tree);
65-
66-
// Get all Markdown Heading entries from the tree
67-
const headingNodes = selectAll('heading', tree);
68-
69-
// Handles Markdown link references and updates them to be plain links
70-
visit(tree, createQueries.UNIST.isLinkReference, node =>
71-
updateLinkReference(node, markdownDefinitions)
72-
);
73-
74-
// Removes all the original definitions from the tree as they are not needed
75-
// anymore, since all link references got updated to be plain links
76-
remove(tree, markdownDefinitions);
77-
78-
// Handles the normalisation URLs that reference to API doc files with .md extension
79-
// to replace the .md into .html, since the API doc files get eventually compiled as HTML
80-
visit(tree, createQueries.UNIST.isMarkdownUrl, node =>
81-
updateMarkdownLink(node)
82-
);
83-
84-
// If the document has no headings but it has content, we add a fake heading to the top
85-
// so that our parsing logic can work correctly, and generate content for the whole file
86-
if (headingNodes.length === 0 && tree.children.length > 0) {
87-
tree.children.unshift(createTree('heading', { depth: 1 }, []));
88-
}
89-
90-
// Handles iterating the tree and creating subtrees for each API doc entry
91-
// where an API doc entry is defined by a Heading Node
92-
// (so all elements after a Heading until the next Heading)
93-
// and then it creates and updates a Metadata entry for each API doc entry
94-
// and then generates the final content for each API doc entry and pushes it to the collection
95-
visit(tree, createQueries.UNIST.isHeading, (headingNode, index) => {
96-
// Creates a new Metadata entry for the current API doc file
97-
const apiEntryMetadata = createMetadata(nodeSlugger);
98-
99-
// Adds the Metadata of the current Heading Node to the Metadata entry
100-
setHeadingMetadata(headingNode, apiEntryMetadata);
101-
102-
// We retrieve the immediate next Heading if it exists
103-
// This is used for ensuring that we don't include items that would
104-
// belong only to the next heading to the current Heading metadata
105-
// Note that if there is no next heading, we use the current node as the next one
106-
const nextHeadingNode =
107-
findAfter(tree, index, createQueries.UNIST.isHeading) ?? headingNode;
108-
109-
// This is the cutover index of the subtree that we should get
110-
// of all the Nodes within the AST tree that belong to this section
111-
// If `next` is equals the current heading, it means there's no next heading
112-
// and we are reaching the end of the document, hence the cutover should be the end of
113-
// the document itself.
114-
const stop =
115-
headingNode === nextHeadingNode
116-
? tree.children.length
117-
: tree.children.indexOf(nextHeadingNode);
118-
119-
// Retrieves all the nodes that should belong to the current API docs section
120-
// `index + 1` is used to skip the current Heading Node
121-
const subTree = createTree('root', tree.children.slice(index, stop));
122-
123-
// Visits all Stability Index nodes from the current subtree if there's any
124-
// and then apply the Stability Index metadata to the current metadata entry
125-
visit(subTree, createQueries.UNIST.isStabilityNode, node =>
126-
addStabilityMetadata(node, apiEntryMetadata)
127-
);
128-
129-
// Visits all HTML nodes from the current subtree and if there's any that matches
130-
// our YAML metadata structure, it transforms into YAML metadata
131-
// and then apply the YAML Metadata to the current Metadata entry
132-
visit(subTree, createQueries.UNIST.isYamlNode, node => {
133-
// TODO: Is there always only one YAML node?
134-
apiEntryMetadata.setYamlPosition(node.position);
135-
addYAMLMetadata(node, apiEntryMetadata);
136-
});
137-
138-
// Visits all Text nodes from the current subtree and if there's any that matches
139-
// any API doc type reference and then updates the type reference to be a Markdown link
140-
visit(subTree, createQueries.UNIST.isTextWithType, (node, _, parent) =>
141-
updateTypeReference(node, parent)
142-
);
143-
144-
// Removes already parsed items from the subtree so that they aren't included in the final content
145-
remove(subTree, [createQueries.UNIST.isYamlNode]);
146-
147-
// Applies the AST transformations to the subtree based on the API doc entry Metadata
148-
// Note that running the transformation on the subtree isn't costly as it is a reduced tree
149-
// and the GFM transformations aren't that heavy
150-
const parsedSubTree = remarkProcessor.runSync(subTree);
151-
152-
// We seal and create the API doc entry Metadata and push them to the collection
153-
const parsedApiEntryMetadata = apiEntryMetadata.create(
154-
file,
155-
parsedSubTree
156-
);
157-
158-
// We push the parsed API doc entry Metadata to the collection
159-
metadataCollection.push(parsedApiEntryMetadata);
160-
161-
return SKIP;
162-
});
163-
164-
// Returns the Metadata entries for the given API doc file
165-
return metadataCollection;
25+
async generate(inputs) {
26+
return inputs.flatMap(input => parseApiDoc(input));
16627
},
16728
};

‎src/generators/metadata/utils/parse.mjs

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
'use strict';
2+
3+
import { u as createTree } from 'unist-builder';
4+
import { findAfter } from 'unist-util-find-after';
5+
import { remove } from 'unist-util-remove';
6+
import { selectAll } from 'unist-util-select';
7+
import { SKIP, visit } from 'unist-util-visit';
8+
9+
import createQueries from '../../../utils/queries/index.mjs';
10+
import { getRemark } from '../../../utils/remark.mjs';
11+
import { createNodeSlugger } from '../../../utils/slugger/index.mjs';
12+
import createMetadata from '../../../metadata.mjs';
13+
14+
/**
15+
* This generator generates a flattened list of metadata entries from a API doc
16+
*
17+
* @param {ParserOutput<import('mdast').Root>} input
18+
* @returns {Promise<ApiDocMetadataEntry[]>}
19+
*/
20+
export const parseApiDoc = ({ file, tree }) => {
21+
/**
22+
* This holds references to all the Metadata entries for a given file
23+
* this is used so we can traverse the AST tree and keep mutating things
24+
* and then stringify the whole api doc file at once without creating sub traversals
25+
*
26+
* Then once we have the whole file parsed, we can split the resulting string into sections
27+
* and seal the Metadata Entries (`.create()`) and return the result to the caller of parae.
28+
*
29+
* @type {Array<ApiDocMetadataEntry>}
30+
*/
31+
const metadataCollection = [];
32+
33+
const {
34+
setHeadingMetadata,
35+
addYAMLMetadata,
36+
updateMarkdownLink,
37+
updateTypeReference,
38+
updateLinkReference,
39+
addStabilityMetadata,
40+
} = createQueries();
41+
42+
// Creates an instance of the Remark processor with GFM support
43+
// which is used for stringifying the AST tree back to Markdown
44+
const remarkProcessor = getRemark();
45+
46+
// Creates a new Slugger instance for the current API doc file
47+
const nodeSlugger = createNodeSlugger();
48+
49+
// Get all Markdown Footnote definitions from the tree
50+
const markdownDefinitions = selectAll('definition', tree);
51+
52+
// Get all Markdown Heading entries from the tree
53+
const headingNodes = selectAll('heading', tree);
54+
55+
// Handles Markdown link references and updates them to be plain links
56+
visit(tree, createQueries.UNIST.isLinkReference, node =>
57+
updateLinkReference(node, markdownDefinitions)
58+
);
59+
60+
// Removes all the original definitions from the tree as they are not needed
61+
// anymore, since all link references got updated to be plain links
62+
remove(tree, markdownDefinitions);
63+
64+
// Handles the normalisation URLs that reference to API doc files with .md extension
65+
// to replace the .md into .html, since the API doc files get eventually compiled as HTML
66+
visit(tree, createQueries.UNIST.isMarkdownUrl, node =>
67+
updateMarkdownLink(node)
68+
);
69+
70+
// If the document has no headings but it has content, we add a fake heading to the top
71+
// so that our parsing logic can work correctly, and generate content for the whole file
72+
if (headingNodes.length === 0 && tree.children.length > 0) {
73+
tree.children.unshift(createTree('heading', { depth: 1 }, []));
74+
}
75+
76+
// Handles iterating the tree and creating subtrees for each API doc entry
77+
// where an API doc entry is defined by a Heading Node
78+
// (so all elements after a Heading until the next Heading)
79+
// and then it creates and updates a Metadata entry for each API doc entry
80+
// and then generates the final content for each API doc entry and pushes it to the collection
81+
visit(tree, createQueries.UNIST.isHeading, (headingNode, index) => {
82+
// Creates a new Metadata entry for the current API doc file
83+
const apiEntryMetadata = createMetadata(nodeSlugger);
84+
85+
// Adds the Metadata of the current Heading Node to the Metadata entry
86+
setHeadingMetadata(headingNode, apiEntryMetadata);
87+
88+
// We retrieve the immediate next Heading if it exists
89+
// This is used for ensuring that we don't include items that would
90+
// belong only to the next heading to the current Heading metadata
91+
// Note that if there is no next heading, we use the current node as the next one
92+
const nextHeadingNode =
93+
findAfter(tree, index, createQueries.UNIST.isHeading) ?? headingNode;
94+
95+
// This is the cutover index of the subtree that we should get
96+
// of all the Nodes within the AST tree that belong to this section
97+
// If `next` is equals the current heading, it means there's no next heading
98+
// and we are reaching the end of the document, hence the cutover should be the end of
99+
// the document itself.
100+
const stop =
101+
headingNode === nextHeadingNode
102+
? tree.children.length
103+
: tree.children.indexOf(nextHeadingNode);
104+
105+
// Retrieves all the nodes that should belong to the current API docs section
106+
// `index + 1` is used to skip the current Heading Node
107+
const subTree = createTree('root', tree.children.slice(index, stop));
108+
109+
// Visits all Stability Index nodes from the current subtree if there's any
110+
// and then apply the Stability Index metadata to the current metadata entry
111+
visit(subTree, createQueries.UNIST.isStabilityNode, node =>
112+
addStabilityMetadata(node, apiEntryMetadata)
113+
);
114+
115+
// Visits all HTML nodes from the current subtree and if there's any that matches
116+
// our YAML metadata structure, it transforms into YAML metadata
117+
// and then apply the YAML Metadata to the current Metadata entry
118+
visit(subTree, createQueries.UNIST.isYamlNode, node => {
119+
// TODO: Is there always only one YAML node?
120+
apiEntryMetadata.setYamlPosition(node.position);
121+
addYAMLMetadata(node, apiEntryMetadata);
122+
});
123+
124+
// Visits all Text nodes from the current subtree and if there's any that matches
125+
// any API doc type reference and then updates the type reference to be a Markdown link
126+
visit(subTree, createQueries.UNIST.isTextWithType, (node, _, parent) =>
127+
updateTypeReference(node, parent)
128+
);
129+
130+
// Removes already parsed items from the subtree so that they aren't included in the final content
131+
remove(subTree, [createQueries.UNIST.isYamlNode]);
132+
133+
// Applies the AST transformations to the subtree based on the API doc entry Metadata
134+
// Note that running the transformation on the subtree isn't costly as it is a reduced tree
135+
// and the GFM transformations aren't that heavy
136+
const parsedSubTree = remarkProcessor.runSync(subTree);
137+
138+
// We seal and create the API doc entry Metadata and push them to the collection
139+
const parsedApiEntryMetadata = apiEntryMetadata.create(file, parsedSubTree);
140+
141+
// We push the parsed API doc entry Metadata to the collection
142+
metadataCollection.push(parsedApiEntryMetadata);
143+
144+
return SKIP;
145+
});
146+
147+
return metadataCollection;
148+
};

0 commit comments

Comments
 (0)