diff --git a/.evergreen/run-granular-benchmarks.sh b/.evergreen/run-granular-benchmarks.sh index ae8399bc..427fa211 100644 --- a/.evergreen/run-granular-benchmarks.sh +++ b/.evergreen/run-granular-benchmarks.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh -set -o xtrace -WARMUP=$WARMUP -ITERATIONS=$ITERATIONS -WARMUP=$WARMUP ITERATIONS=$ITERATIONS LIBRARY=$(pwd) npm run check:granular-bench +set -euxo pipefail + +export WARMUP=$WARMUP +export ITERATIONS=$ITERATIONS + +npm run check:granular-bench diff --git a/.evergreen/run-spec-benchmarks.sh b/.evergreen/run-spec-benchmarks.sh index dd85250b..fa0e04d4 100644 --- a/.evergreen/run-spec-benchmarks.sh +++ b/.evergreen/run-spec-benchmarks.sh @@ -2,4 +2,6 @@ source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh +set -euxo pipefail + npm run check:spec-bench diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e98d3feb..2c0889bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,15 +83,16 @@ WARMUP= ITERATIONS= npm run check:granul This will build the granular tests and run them with `test/bench/etc/run_granular_benchmarks.js`. The `WARMUP` and `ITERATIONS` environment variables can be optionally provided to configure how these granular benchmarks are run. `WARMUP` changes the number of iterations run before results are collected to give v8's optimizing compiler time to reach steady state. `ITERATIONS` changes the number of iterations that -are measured and used to calculate summary statistics. Note also that the test can be configured to -make use of the local copy of bson when testing performance changes locally by setting the `LIBRARY` -variable to the root directory of the bson library to be tested. +are measured and used to calculate summary statistics. ```bash -WARMUP=100 ITERATIONS=1000 LIBRARY=$(pwd) npm run check:granular-bench +WARMUP=100 ITERATIONS=1000 npm run check:granular-bench ``` -When the `LIBRARY` environment variable is unset, the benchmark clones and runs against the main -branch of this repository. + +When the `LIBRARY` environment variable is unset, the benchmark clones and runs against the current working directory. + +> [!NOTE] +> The test can be configured to make use of a different copy of bson when testing performance changes locally by setting the `LIBRARY` variable to the root directory of a copy of the bson library. When the script is complete, results will be output to `test/bench/etc/resultsCollectedMeans.json`. These results will be in a format compatible with evergreen's perf.send command. To convert these results to CSV, run @@ -109,14 +110,15 @@ npm run check:spec-bench This will run the spec benchmarks in `test/bench/spec/bsonBench.ts` which also makes use of the `bson-bench` library. Results will be written to `bsonBench`. The warmup and iterations are not -configurable as these are determined by the common driver benchmarking specification, but similar -to the granular benchmarks, the spec benchmarks can be run against the local copy of bson by setting -the `LIBRARY` environment variable appropriately. +configurable as these are determined by the common driver benchmarking specification. ```bash -LIBRARY=$(pwd) npm run check:spec-bench +npm run check:spec-bench ``` +> [!NOTE] +> The test can be configured to make use of a different copy of bson when testing performance changes locally by setting the `LIBRARY` variable to the root directory of a copy of the bson library. + ### Commit messages Please follow the [Conventional Commits specification][conventional-commit-style]. diff --git a/test/bench/etc/run_granular_benchmarks.js b/test/bench/etc/run_granular_benchmarks.js index f458c04c..6016248c 100644 --- a/test/bench/etc/run_granular_benchmarks.js +++ b/test/bench/etc/run_granular_benchmarks.js @@ -66,7 +66,7 @@ const DOCUMENT_ROOT = path.resolve(`${__dirname}/../documents`); } } - console.log('No duplcate testName:Option pairs found. Now merging files...'); + console.log('No duplicate testName:Option pairs found. Now merging files...'); const meansFile = `${__dirname}/resultsCollectedMeans.json`; // Iterate over all result files and merge into one file diff --git a/test/bench/granular/common.ts b/test/bench/granular/common.ts index 7d99afca..c7782e5b 100644 --- a/test/bench/granular/common.ts +++ b/test/bench/granular/common.ts @@ -61,6 +61,7 @@ export function getMixedTestTags(documentPath: string) { export async function runSuiteAndWriteResults(suite: Suite) { const targetDirectory = path.resolve(`${__dirname}/../../etc`); await suite.run(); + if (suite.errors.length) throw suite.errors[0].error; await suite.writeResults(`${targetDirectory}/${suite.name.toLowerCase()}Results.json`); } @@ -71,7 +72,7 @@ export function readEnvVars(): { warmup: number; iterations: number; library: st const rv = { warmup: Number.isSafeInteger(envWarmup) && envWarmup > 0 ? envWarmup : 100_000, iterations: Number.isSafeInteger(envIterations) && envIterations > 0 ? envIterations : 10_000, - library: libraryPath ? `bson:${libraryPath}` : 'bson#main' + library: libraryPath ? `bson:${libraryPath}` : `bson:${process.cwd()}` }; console.log( diff --git a/test/bench/spec/bsonBench.ts b/test/bench/spec/bsonBench.ts index e5477db9..aa730c56 100644 --- a/test/bench/spec/bsonBench.ts +++ b/test/bench/spec/bsonBench.ts @@ -78,6 +78,8 @@ suite.task({ }); suite.run().then(async () => { + if (suite.errors.length) throw suite.errors[0].error; + const cpuBaseline = await readFile( join(__dirname, '..', '..', 'etc', 'cpuBaseline.json'), 'utf8'