diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 605e84044..d2d473511 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -40,6 +40,15 @@ functions: args: - .evergreen/install-dependencies.sh + perf send: + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - .evergreen/perf-send.sh + run tests: - command: subprocess.exec type: test @@ -257,9 +266,9 @@ tasks: vars: WARMUP: 1000 ITERATIONS: 1000 - - command: perf.send - params: - file: src/test/bench/etc/resultsCollectedMeans.json + - func: perf send + vars: + TARGET_FILE: ./test/bench/etc/resultsCollectedMeans.json - name: run-custom-benchmarks commands: - func: fetch source @@ -271,9 +280,9 @@ tasks: vars: NPM_VERSION: 9 - func: run custom benchmarks - - command: perf.send - params: - file: src/customBenchmarkResults.json + - func: perf send + vars: + TARGET_FILE: ./customBenchmarkResults.json - name: run-spec-benchmarks commands: - func: fetch source @@ -285,9 +294,9 @@ tasks: vars: NPM_VERSION: 9 - func: run spec benchmarks - - command: perf.send - params: - file: src/bsonBench.json + - func: perf send + vars: + TARGET_FILE: ./bsonBench.json - name: check-eslint-plugin commands: - func: fetch source diff --git a/.evergreen/perf-send.sh b/.evergreen/perf-send.sh new file mode 100644 index 000000000..a53ff6e80 --- /dev/null +++ b/.evergreen/perf-send.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh +set -o xtrace +TARGET_FILE=$TARGET_FILE + +node ./.evergreen/perf_send.mjs $TARGET_FILE diff --git a/.evergreen/perf_send.mjs b/.evergreen/perf_send.mjs new file mode 100644 index 000000000..9d71c4aeb --- /dev/null +++ b/.evergreen/perf_send.mjs @@ -0,0 +1,73 @@ +import fs from 'fs/promises'; +import util from 'util'; + +const API_PATH = 'https://performance-monitoring-api.corp.mongodb.com/raw_perf_results'; + +const resultFile = process.argv[2]; +if (resultFile == null) { + throw new Error('Must specify result file'); +} + +// Get expansions +const { + execution, + requester, + project, + task_id, + task_name, + revision_order_id, + build_variant: variant, + version_id: version +} = process.env; + +const orderSplit = revision_order_id?.split('_'); +const order = Number(orderSplit ? orderSplit[orderSplit.length - 1] : undefined); + +if (!Number.isInteger(order)) throw new Error(`Failed to parse integer from order, revision_order_id=${revision_order_id}`); + +const results = JSON.parse(await fs.readFile(resultFile, 'utf8')); + +// FIXME(NODE-6838): We are using dummy dates here just to be able to successfully post our results +for (const r of results) { + r.created_at = new Date().toISOString(); + r.completed_at = new Date().toISOString(); +} + +const body = { + id: { + project, + version, + variant, + order, + task_name, + task_id, + execution, + mainline: requester === 'commit' + }, + results +}; + +console.log('POST', util.inspect(body, { depth: Infinity })); + +const resp = await fetch(API_PATH, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + accept: 'application/json' + }, + body: JSON.stringify(body) +}); + +const responseText = await resp.text(); +let jsonResponse = null; +try { + jsonResponse = JSON.parse(responseText) +} catch (cause) { + console.log('Failed to parse json response', cause); +} + +console.log(resp.statusText, util.inspect(jsonResponse ?? responseText, { depth: Infinity })); + +if (jsonResponse.message == null) throw new Error("Didn't get success message"); + +console.log(jsonResponse.message); diff --git a/package-lock.json b/package-lock.json index c37e1732b..76fc203d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "benchmark": "^2.1.4", "chai": "^4.4.1", "chalk": "^5.3.0", - "dbx-js-tools": "github:mongodb-js/dbx-js-tools", + "dbx-js-tools": "github:mongodb-js/dbx-js-tools#main", "eslint": "^9.8.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-no-bigint-usage": "file:etc/eslint/no-bigint-usage", @@ -2560,7 +2560,7 @@ } }, "node_modules/dbx-js-tools": { - "resolved": "git+ssh://git@github.com/mongodb-js/dbx-js-tools.git#b2d505fcfa6fc13566c0cb5ab0ff9741fc7580a1", + "resolved": "git+ssh://git@github.com/mongodb-js/dbx-js-tools.git#fe3ea15fcd75f9a81986d6c3f29a4df94db4b750", "dev": true, "workspaces": [ "packages/bson-bench", diff --git a/package.json b/package.json index ebbf6da44..2399e2c73 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "benchmark": "^2.1.4", "chai": "^4.4.1", "chalk": "^5.3.0", - "dbx-js-tools": "github:mongodb-js/dbx-js-tools", + "dbx-js-tools": "github:mongodb-js/dbx-js-tools#main", "eslint": "^9.8.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-no-bigint-usage": "file:etc/eslint/no-bigint-usage", @@ -102,9 +102,10 @@ "check:tsd": "npm run build:dts && tsd", "check:web": "WEB=true mocha test/node", "check:web-no-bigint": "WEB=true NO_BIGINT=true mocha test/node", - "check:granular-bench": "npm run build:bench && node ./test/bench/etc/run_granular_benchmarks.js", - "check:spec-bench": "npm run build:bench && node ./test/bench/lib/spec/bsonBench.js", - "check:custom-bench": "npm run build && node ./test/bench/custom/main.mjs", + "check:granular-bench": "npm run build:bench && npm run check:baseline-bench && node ./test/bench/etc/run_granular_benchmarks.js", + "check:spec-bench": "npm run build:bench && npm run check:baseline-bench && node ./test/bench/lib/spec/bsonBench.js", + "check:custom-bench": "npm run build && npm run check:baseline-bench && node ./test/bench/custom/main.mjs", + "check:baseline-bench": "node ./test/bench/etc/cpuBaseline.js", "build:bench": "cd test/bench && npx tsc", "build:ts": "node ./node_modules/typescript/bin/tsc", "build:dts": "npm run build:ts && api-extractor run --typescript-compiler-folder node_modules/typescript --local && node etc/clean_definition_files.cjs", diff --git a/test/bench/custom/benchmarks.mjs b/test/bench/custom/benchmarks.mjs index 74170fb90..8a17e3edd 100644 --- a/test/bench/custom/benchmarks.mjs +++ b/test/bench/custom/benchmarks.mjs @@ -1,22 +1,26 @@ /* eslint-disable strict */ import { BSON } from '../../../lib/bson.mjs'; -const ObjectId_isValid = [ - function objectid_isvalid_wrong_string_length() { - BSON.ObjectId.isValid('a'); - }, - /** wrong character at the start, could be the most short circuited code path */ - function objectid_isvalid_invalid_hex_at_start() { - BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf'); - }, - /** wrong character at the end, could be the least short circuited code path */ - function objectid_isvalid_invalid_hex_at_end() { - BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbg'); - }, - function objectid_isvalid_valid_hex_string() { - BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbf'); - } -]; +const ObjectId_isValid = { + name: 'ObjectId_isValid', + tags: ['alerting-benchmark', 'objectid'], + benchmarks: [ + function objectid_isvalid_wrong_string_length() { + BSON.ObjectId.isValid('a'); + }, + /** wrong character at the start, could be the most short circuited code path */ + function objectid_isvalid_invalid_hex_at_start() { + BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf'); + }, + /** wrong character at the end, could be the least short circuited code path */ + function objectid_isvalid_invalid_hex_at_end() { + BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbg'); + }, + function objectid_isvalid_valid_hex_string() { + BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbf'); + } + ] +}; // Add benchmarks here: -export const benchmarks = [...ObjectId_isValid]; +export const suites = [ObjectId_isValid]; diff --git a/test/bench/custom/main.mjs b/test/bench/custom/main.mjs index b375418f1..2f7687270 100644 --- a/test/bench/custom/main.mjs +++ b/test/bench/custom/main.mjs @@ -4,7 +4,7 @@ import util from 'node:util'; import fs from 'node:fs'; import os from 'node:os'; import benchmark from 'benchmark'; -import { benchmarks } from './benchmarks.mjs'; +import { suites } from './benchmarks.mjs'; const hw = os.cpus(); const ram = os.totalmem() / 1024 ** 3; @@ -20,20 +20,71 @@ const systemInfo = () => ].join('\n'); console.log(systemInfo()); -const suite = new benchmark.Suite(); - -for (const bench of benchmarks) suite.add(bench.name, bench); - -suite - .on('cycle', function logBenchmark(event) { - console.log(String(event.target)); - }) - .on('complete', function outputPerfSend() { - const data = Array.from(this).map(bench => ({ - info: { test_name: bench.name }, - metrics: [{ name: 'ops_per_sec', value: bench.hz }] - })); - console.log(util.inspect(data, { depth: Infinity, colors: true })); - fs.writeFileSync('customBenchmarkResults.json', JSON.stringify(data), 'utf8'); - }) - .run(); +function logBenchmark(event) { + console.log(String(event.target)); +} + +function processBenchmarkResult(bench, metadata) { + return { + info: { test_name: bench.name }, + metrics: [{ name: 'ops_per_sec', value: bench.hz, metadata }] + }; +} + +let completedSuites = 0; +async function completeSuite() { + const metadata = { improvement_direction: 'up' }; + if (++completedSuites >= collectedSuites.length) { + let cpuBaselineResults; + try { + cpuBaselineResults = await import('../etc/cpuBaseline.json', { assert: { type: 'json' } }); + } catch (cause) { + throw new Error("Couldn't find baseline results", { cause }); + } + + cpuBaselineResults = cpuBaselineResults.default; + const cpuBaselineResult = cpuBaselineResults.hz; + if (typeof cpuBaselineResult !== 'number') { + throw new Error("Couldn't find baseline result"); + } + + const data = []; + for (const { suite, suiteConfig } of collectedSuites) { + const { tags } = suiteConfig; + for (const bench of Array.from(suite)) { + const result = processBenchmarkResult(bench, { ...metadata, tags }); + result.metrics.push({ + name: 'normalized_throughput', + value: bench.hz / cpuBaselineResult, + metadata: { ...metadata, tags } + }); + data.push(result); + } + + data.push({ + info: { test_name: 'cpuBaseline' }, + metrics: [{ name: 'ops_per_sec', value: cpuBaselineResult, metadata }] + }); + + console.log(util.inspect(data, { depth: Infinity, colors: true })); + fs.writeFileSync('customBenchmarkResults.json', JSON.stringify(data), 'utf8'); + } + } +} + +function processSuite(suiteModule, cycleHandler, completeHandler) { + let suite = new benchmark.Suite(suiteModule.name); + for (const b of suiteModule.benchmarks) { + suite.add(b.name, b); + } + + suite = suite.on('cycle', cycleHandler).on('complete', completeHandler).run({ async: true }); + + return { suite, suiteConfig: suiteModule }; +} + +const collectedSuites = []; +for (const suite of suites) { + const newSuite = processSuite(suite, logBenchmark, completeSuite); + collectedSuites.push(newSuite); +} diff --git a/test/bench/custom/readme.md b/test/bench/custom/readme.md index ed69ae218..294d2479f 100644 --- a/test/bench/custom/readme.md +++ b/test/bench/custom/readme.md @@ -4,21 +4,25 @@ In this directory are tests for code paths not covered by our spec or granular ( ## How to write your own -In `benchmarks.mjs` add a new test to an existing array or make a new array for a new subject area. -Try to fit the name of the function into the format of: "subject area", "method or function" "test case that is being covered" (Ex. `objectid_isvalid_bestcase_false`). +In `benchmarks.mjs` add a new test to an existing benchmark object or make a new object for a new subject area. +Try to fit the name of the variables and the benchmark functions into the format of: "subject area", "method or function" "test case that is being covered" (Ex. `objectid_isvalid_bestcase_false`). Make sure your test is added to the `benchmarks` export. ### Example ```js -const ObjectId_isValid = [ - function objectid_isvalid_strlen() { - BSON.ObjectId.isValid('a'); - }, - // ... -]; - -export const benchmarks = [...ObjectId_isValid]; +const ObjectId_isValid = { + name: 'ObjectId_isValid', + tags: ['objectid'], + benchmarks : [ + function objectid_isvalid_strlen() { + BSON.ObjectId.isValid('a'); + }, + // ... + ] +}; + +export const benchmarks = [ObjectId_isValid]; ``` ## Output @@ -28,9 +32,9 @@ The JSON emitted at the end of the benchmarks must follow our performance tracki The JSON must be an array of "`Test`"s: ```ts -type Metric = { name: string, value: number } +type Metric = { name: string, value: number, metadata: { improvement_direction: 'up' | 'down' } } type Test = { - info: { test_name: string }, + info: { test_name: string, tags?: string[]}, metrics: Metric[] } ``` diff --git a/test/bench/etc/.gitignore b/test/bench/etc/.gitignore index df8b875d8..fb04d547d 100644 --- a/test/bench/etc/.gitignore +++ b/test/bench/etc/.gitignore @@ -1,2 +1,3 @@ *Results.json -resultsCollected.json +cpuBaseline.json +resultsCollected*.json diff --git a/test/bench/etc/cpuBaseline.js b/test/bench/etc/cpuBaseline.js new file mode 100644 index 000000000..a6f37bc4d --- /dev/null +++ b/test/bench/etc/cpuBaseline.js @@ -0,0 +1,63 @@ +'use strict'; +const fs = require('fs'); +const { join } = require('path'); +const benchmark = require('benchmark'); + +const stableRegionMean = 42.82; +const taskSize = 3.1401000000000003 / stableRegionMean; // ~3MB worth of work scaled down by the mean of the current stable region in CI to bring this value to roughly 1 + +function sieveOfEratosthenes(n) { + // Create a boolean array "prime[0..n]" and initialize + // all entries as true. A value in prime[i] will + // become false if i is Not a prime + const prime = Array.from({ length: n + 1 }, () => true); + + // We know 0 and 1 are not prime + prime[0] = false; + prime[1] = false; + + for (let p = 2; p * p <= n; p++) { + // If prime[p] is not changed, then it is a prime + if (prime[p] === true) { + // Update all multiples of p as false + for (let i = p * p; i <= n; i += p) { + prime[i] = false; + } + } + } + + // Collecting all prime numbers + const primes = []; + for (let i = 2; i <= n; i++) { + if (prime[i] === true) { + primes.push(i); + } + } + + return primes; +} + +new benchmark.Suite() + .add('cpuBaseline', function () { + sieveOfEratosthenes(1_000_000); + }) + .on('complete', function () { + const data = {}; + for (const b of Array.from(this)) { + if (b.name !== 'cpuBaseline') continue; + data.name = b.name; + data.stats = b.stats; + data.times = b.times; + data.hz = b.hz; + data.count = b.count; + data.cycles = b.cycles; + data.megabytes_per_second = taskSize / b.stats.mean; + } + + console.log(data); + const path = join(__dirname, 'cpuBaseline.json'); + fs.writeFileSync(path, JSON.stringify(data)); + + console.log('Wrote to ', path); + }) + .run(); diff --git a/test/bench/etc/run_granular_benchmarks.js b/test/bench/etc/run_granular_benchmarks.js index 150ecab3d..f458c04c7 100644 --- a/test/bench/etc/run_granular_benchmarks.js +++ b/test/bench/etc/run_granular_benchmarks.js @@ -28,6 +28,9 @@ const DOCUMENT_ROOT = path.resolve(`${__dirname}/../documents`); .run() .catch(() => null); + // Check for benchmark results + const cpuBaselineData = require(`./cpuBaseline.json`); + // Run all benchmark files const lib = await fs.readdir(BENCHMARK_PATH); for await (const dirent of lib) { @@ -74,13 +77,31 @@ const DOCUMENT_ROOT = path.resolve(`${__dirname}/../documents`); collectedResults.push(...results); } } + const metadata = { + improvement_direction: 'up' + }; const means = collectedResults.map(result => { const rv = { ...result }; rv.metrics = rv.metrics.filter(metric => metric.type === 'MEAN'); + rv.metrics = rv.metrics.map(m => { + return { ...m, metadata: { ...m.metadata, improvement_direction: 'up' } }; + }); + rv.metrics.push({ + name: 'normalized_throughput', + value: rv.metrics[0].value / cpuBaselineData.megabytes_per_second, + metadata + }); return rv; }); + means.push({ + info: { test_name: 'cpuBaseline' }, + metrics: [ + { name: 'megabytes_per_second', value: cpuBaselineData.megabytes_per_second, metadata } + ] + }); + await fs.writeFile(meansFile, JSON.stringify(means)); console.log(`Means in ${meansFile}`); diff --git a/test/bench/granular/binary.bench.ts b/test/bench/granular/binary.bench.ts index e68d1a2d7..8a040b0b0 100644 --- a/test/bench/granular/binary.bench.ts +++ b/test/bench/granular/binary.bench.ts @@ -5,7 +5,8 @@ import { BOOL, ITERATIONS, LIBRARY_SPEC, - WARMUP + WARMUP, + getTypeTestTags } from './common'; async function main() { @@ -13,6 +14,7 @@ async function main() { const testDocs = await getTestDocs('binary'); // deserialize for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); for (const promoteBuffers of BOOL) { suite.task({ documentPath, @@ -22,7 +24,8 @@ async function main() { operation: 'deserialize', options: { promoteBuffers - } + }, + tags }); } @@ -33,7 +36,8 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'serialize', - options: { checkKeys: true, ignoreUndefined: false } + options: { checkKeys: true, ignoreUndefined: false }, + tags }); } await runSuiteAndWriteResults(suite); diff --git a/test/bench/granular/boolean.bench.ts b/test/bench/granular/boolean.bench.ts index 8b6f1414b..599b6172d 100644 --- a/test/bench/granular/boolean.bench.ts +++ b/test/bench/granular/boolean.bench.ts @@ -5,7 +5,8 @@ import { BOOL, ITERATIONS, WARMUP, - LIBRARY_SPEC + LIBRARY_SPEC, + getTypeTestTags } from './common'; const OPTIONS = { @@ -21,6 +22,7 @@ async function main() { const testDocs = await getTestDocs('boolean'); // deserialize for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); for (const promoteValues of BOOL) { suite.task({ documentPath, @@ -28,20 +30,23 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'deserialize', - options: { ...OPTIONS.deserialize, promoteValues } + options: { ...OPTIONS.deserialize, promoteValues }, + tags }); } } // serialize for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation: 'deserialize', - options: OPTIONS.serialize + options: OPTIONS.serialize, + tags }); } await runSuiteAndWriteResults(suite); diff --git a/test/bench/granular/code.bench.ts b/test/bench/granular/code.bench.ts index 32092734f..323c10c8a 100644 --- a/test/bench/granular/code.bench.ts +++ b/test/bench/granular/code.bench.ts @@ -5,7 +5,8 @@ import { OPERATIONS, ITERATIONS, LIBRARY_SPEC, - WARMUP + WARMUP, + getTypeTestTags } from './common'; const OPTIONS = { @@ -19,15 +20,17 @@ async function main() { await getTestDocs('code-with-scope') ); - for (const operation of OPERATIONS) { - for (const documentPath of testDocs) { + for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); + for (const operation of OPERATIONS) { suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation, - options: OPTIONS[operation] + options: OPTIONS[operation], + tags }); } } diff --git a/test/bench/granular/common.ts b/test/bench/granular/common.ts index 3631878b2..7d99afca2 100644 --- a/test/bench/granular/common.ts +++ b/test/bench/granular/common.ts @@ -25,6 +25,39 @@ export async function getTestDocs(type: string) { return docs; } +const ALERTING_DOCS = new Set([ + 'objectid_array_1000.json', + 'double_array_1000.json', + 'int32_array_1000.json', + 'long_array_1000.json', + 'string_array_1000.json', + 'binary_array_1000.json', + 'bestbuy_medium.json' +]); + +export const ALERT_TAG = 'alerting-benchmark'; + +export function getTypeTestTags(documentPath: string) { + const basename = path.basename(documentPath); + const type = basename.split('_')[0]; + + if (ALERTING_DOCS.has(basename)) { + return [type, ALERT_TAG]; + } else { + return [type]; + } +} + +export function getMixedTestTags(documentPath: string) { + const basename = path.basename(documentPath).split('.')[0]; + + if (ALERTING_DOCS.has(basename)) { + return ['mixed', ALERT_TAG]; + } + + return ['mixed']; +} + export async function runSuiteAndWriteResults(suite: Suite) { const targetDirectory = path.resolve(`${__dirname}/../../etc`); await suite.run(); diff --git a/test/bench/granular/date.bench.ts b/test/bench/granular/date.bench.ts index 6472d66ff..5abec5b2e 100644 --- a/test/bench/granular/date.bench.ts +++ b/test/bench/granular/date.bench.ts @@ -5,7 +5,8 @@ import { OPERATIONS, ITERATIONS, LIBRARY_SPEC, - WARMUP + WARMUP, + getTypeTestTags } from './common'; const OPTIONS = { @@ -17,15 +18,17 @@ async function main() { const suite = new Suite('Date'); const testDocs = await getTestDocs('date'); - for (const operation of OPERATIONS) { - for (const documentPath of testDocs) { + for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); + for (const operation of OPERATIONS) { suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation, - options: OPTIONS[operation] + options: OPTIONS[operation], + tags }); } } diff --git a/test/bench/granular/decimal128.bench.ts b/test/bench/granular/decimal128.bench.ts index 62c65d114..cbf779bf6 100644 --- a/test/bench/granular/decimal128.bench.ts +++ b/test/bench/granular/decimal128.bench.ts @@ -5,7 +5,8 @@ import { OPERATIONS, ITERATIONS, LIBRARY_SPEC, - WARMUP + WARMUP, + getTypeTestTags } from './common'; const OPTIONS = { @@ -21,15 +22,17 @@ const OPTIONS = { async function main() { const suite = new Suite('Decimal128'); const testDocs = await getTestDocs('decimal128'); - for (const operation of OPERATIONS) { - for (const documentPath of testDocs) { + for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); + for (const operation of OPERATIONS) { suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation, - options: OPTIONS[operation] + options: OPTIONS[operation], + tags }); } } diff --git a/test/bench/granular/double.bench.ts b/test/bench/granular/double.bench.ts index d7b8625de..03edf4113 100644 --- a/test/bench/granular/double.bench.ts +++ b/test/bench/granular/double.bench.ts @@ -5,7 +5,8 @@ import { LIBRARY_SPEC, BOOL, ITERATIONS, - WARMUP + WARMUP, + getTypeTestTags } from './common'; const OPTIONS = { @@ -20,6 +21,7 @@ async function main() { const testDocs = await getTestDocs('double'); for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); // deserialize for (const promoteValues of BOOL) { suite.task({ @@ -28,7 +30,8 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'deserialize', - options: { ...OPTIONS.deserialize, promoteValues } + options: { ...OPTIONS.deserialize, promoteValues }, + tags }); } @@ -39,7 +42,8 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'serialize', - options: OPTIONS.serialize + options: OPTIONS.serialize, + tags }); } await runSuiteAndWriteResults(suite); diff --git a/test/bench/granular/int32.bench.ts b/test/bench/granular/int32.bench.ts index 80d80af41..826fe8b79 100644 --- a/test/bench/granular/int32.bench.ts +++ b/test/bench/granular/int32.bench.ts @@ -5,7 +5,8 @@ import { LIBRARY_SPEC, BOOL, ITERATIONS, - WARMUP + WARMUP, + getTypeTestTags } from './common'; const OPTIONS = { @@ -23,6 +24,7 @@ async function main() { const suite = new Suite('Int32'); const testDocs = await getTestDocs('int32'); for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); // deserialize for (const promoteValues of BOOL) { suite.task({ @@ -31,7 +33,8 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'deserialize', - options: { ...OPTIONS.deserialize, promoteValues } + options: { ...OPTIONS.deserialize, promoteValues }, + tags }); } //serialize @@ -41,7 +44,8 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'serialize', - options: OPTIONS.serialize + options: OPTIONS.serialize, + tags }); } await runSuiteAndWriteResults(suite); diff --git a/test/bench/granular/long.bench.ts b/test/bench/granular/long.bench.ts index 1d2808ae7..7546a4424 100644 --- a/test/bench/granular/long.bench.ts +++ b/test/bench/granular/long.bench.ts @@ -1,5 +1,12 @@ import { Suite } from 'dbx-js-tools/packages/bson-bench'; -import { getTestDocs, runSuiteAndWriteResults, ITERATIONS, LIBRARY_SPEC, WARMUP } from './common'; +import { + getTestDocs, + runSuiteAndWriteResults, + ITERATIONS, + LIBRARY_SPEC, + WARMUP, + getTypeTestTags +} from './common'; const JSBSONDeserializationOptions = [ { @@ -29,6 +36,7 @@ async function main() { const testDocs = await getTestDocs('long'); // LONG JS-BSON Deserialization tests for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); for (const options of JSBSONDeserializationOptions) { suite.task({ documentPath, @@ -36,13 +44,15 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'deserialize', - options + options, + tags }); } } // LONG JS-BSON Serialization tests for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); for (const options of JSBSONSerializationOptions) { suite.task({ documentPath, @@ -50,7 +60,8 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'serialize', - options + options, + tags }); } } diff --git a/test/bench/granular/maxkey.bench.ts b/test/bench/granular/maxkey.bench.ts index 129466dfa..333cd78ea 100644 --- a/test/bench/granular/maxkey.bench.ts +++ b/test/bench/granular/maxkey.bench.ts @@ -5,7 +5,8 @@ import { LIBRARY_SPEC, OPERATIONS, ITERATIONS, - WARMUP + WARMUP, + getTypeTestTags } from './common'; const OPTIONS = { @@ -19,15 +20,17 @@ async function main() { const suite = new Suite('MaxKey'); const testDocs = await getTestDocs('maxkey'); - for (const operation of OPERATIONS) { - for (const documentPath of testDocs) { + for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); + for (const operation of OPERATIONS) { suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation, - options: OPTIONS[operation] + options: OPTIONS[operation], + tags }); } } diff --git a/test/bench/granular/minkey.bench.ts b/test/bench/granular/minkey.bench.ts index 50e0b81e3..5d834d9fc 100644 --- a/test/bench/granular/minkey.bench.ts +++ b/test/bench/granular/minkey.bench.ts @@ -5,7 +5,8 @@ import { LIBRARY_SPEC, OPERATIONS, ITERATIONS, - WARMUP + WARMUP, + getTypeTestTags } from './common'; const OPTIONS = { @@ -21,13 +22,15 @@ async function main() { for (const operation of OPERATIONS) { for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation, - options: OPTIONS[operation] + options: OPTIONS[operation], + tags }); } } diff --git a/test/bench/granular/mixed.bench.ts b/test/bench/granular/mixed.bench.ts index 4e114dccc..cc5707004 100644 --- a/test/bench/granular/mixed.bench.ts +++ b/test/bench/granular/mixed.bench.ts @@ -5,7 +5,8 @@ import { OPERATIONS, ITERATIONS, WARMUP, - LIBRARY_SPEC + LIBRARY_SPEC, + getMixedTestTags } from './common'; import * as path from 'path'; @@ -33,13 +34,15 @@ async function main() { for (const operation of OPERATIONS) { for (const documentPath of mixedDocuments) { + const tags = getMixedTestTags(documentPath); suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation, - options: OPTIONS[operation] + options: OPTIONS[operation], + tags: tags }); } } diff --git a/test/bench/granular/null.bench.ts b/test/bench/granular/null.bench.ts index b02dfa141..0d22c57fb 100644 --- a/test/bench/granular/null.bench.ts +++ b/test/bench/granular/null.bench.ts @@ -5,7 +5,8 @@ import { LIBRARY_SPEC, OPERATIONS, ITERATIONS, - WARMUP + WARMUP, + getTypeTestTags } from './common'; async function main() { @@ -13,15 +14,17 @@ async function main() { const testDocs = await getTestDocs('null'); - for (const operation of OPERATIONS) { - for (const documentPath of testDocs) { + for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); + for (const operation of OPERATIONS) { suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation, - options: {} + options: {}, + tags }); } } diff --git a/test/bench/granular/objectid.bench.ts b/test/bench/granular/objectid.bench.ts index 32b535ded..a9aafe260 100644 --- a/test/bench/granular/objectid.bench.ts +++ b/test/bench/granular/objectid.bench.ts @@ -5,22 +5,25 @@ import { LIBRARY_SPEC, OPERATIONS, ITERATIONS, - WARMUP + WARMUP, + getTypeTestTags } from './common'; async function main() { const suite = new Suite('ObjectId'); const testDocs = await getTestDocs('objectid'); - for (const operation of OPERATIONS) { - for (const documentPath of testDocs) { + for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); + for (const operation of OPERATIONS) { suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation, - options: {} + options: {}, + tags }); } } diff --git a/test/bench/granular/regexp.bench.ts b/test/bench/granular/regexp.bench.ts index 32e240d1c..6c96a84b6 100644 --- a/test/bench/granular/regexp.bench.ts +++ b/test/bench/granular/regexp.bench.ts @@ -5,7 +5,8 @@ import { LIBRARY_SPEC, ITERATIONS, WARMUP, - BOOL + BOOL, + getTypeTestTags } from './common'; async function main() { @@ -14,6 +15,7 @@ async function main() { const testDocs = await getTestDocs('regex'); for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); // deserialize for (const bsonRegExp of BOOL) { suite.task({ @@ -22,7 +24,8 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'deserialize', - options: { bsonRegExp } + options: { bsonRegExp }, + tags }); } @@ -33,7 +36,8 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'serialize', - options: {} + options: {}, + tags }); } await runSuiteAndWriteResults(suite); diff --git a/test/bench/granular/string.bench.ts b/test/bench/granular/string.bench.ts index 98ef00c56..05399a735 100644 --- a/test/bench/granular/string.bench.ts +++ b/test/bench/granular/string.bench.ts @@ -5,7 +5,8 @@ import { LIBRARY_SPEC, ITERATIONS, WARMUP, - BOOL + BOOL, + getTypeTestTags } from './common'; async function main() { @@ -15,6 +16,7 @@ async function main() { // deserialize for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); for (const utf8 of BOOL) { suite.task({ documentPath, @@ -22,19 +24,22 @@ async function main() { iterations: ITERATIONS, warmup: WARMUP, operation: 'deserialize', - options: { validation: { utf8 } } + options: { validation: { utf8 } }, + tags }); } } // serialize for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation: 'serialize', - options: { checkKeys: true, ignoreUndefined: false } + options: { checkKeys: true, ignoreUndefined: false }, + tags }); } await runSuiteAndWriteResults(suite); diff --git a/test/bench/granular/timestamp.bench.ts b/test/bench/granular/timestamp.bench.ts index 1a3a65b41..bb90448c3 100644 --- a/test/bench/granular/timestamp.bench.ts +++ b/test/bench/granular/timestamp.bench.ts @@ -5,22 +5,25 @@ import { LIBRARY_SPEC, OPERATIONS, ITERATIONS, - WARMUP + WARMUP, + getTypeTestTags } from './common'; async function main() { const suite = new Suite('Timestamp'); const testDocs = await getTestDocs('timestamp'); - for (const operation of OPERATIONS) { - for (const documentPath of testDocs) { + for (const documentPath of testDocs) { + const tags = getTypeTestTags(documentPath); + for (const operation of OPERATIONS) { suite.task({ documentPath, library: LIBRARY_SPEC, iterations: ITERATIONS, warmup: WARMUP, operation, - options: {} + options: {}, + tags }); } } diff --git a/test/bench/spec/bsonBench.ts b/test/bench/spec/bsonBench.ts index e10896169..e5477db9e 100644 --- a/test/bench/spec/bsonBench.ts +++ b/test/bench/spec/bsonBench.ts @@ -1,113 +1,153 @@ -import { Suite } from 'dbx-js-tools/packages/bson-bench'; +import { PerfSendResult, Suite } from 'dbx-js-tools/packages/bson-bench'; import { join, resolve } from 'path'; -import { writeFile } from 'fs/promises'; -import { LIBRARY_SPEC } from '../granular/common'; +import { writeFile, readFile } from 'fs/promises'; +import { readEnvVars, ALERT_TAG } from '../granular/common'; +type Metadata = { + improvement_direction: 'up' | 'down'; +}; const suite = new Suite('bson micro benchmarks'); const DOCUMENT_ROOT = resolve(`${__dirname}/../../documents`); +const { library } = readEnvVars(); +const warmup = 1000; +const iterations = 10_000; // Add flat bson encoding suite.task({ documentPath: join(DOCUMENT_ROOT, 'flat_bson.json'), - library: LIBRARY_SPEC, - warmup: 1000, - iterations: 10_000, + library, + warmup, + iterations, operation: 'serialize', - options: {} + options: {}, + tags: [ALERT_TAG] }); // Add flat bson decoding suite.task({ documentPath: join(DOCUMENT_ROOT, 'flat_bson.json'), - library: LIBRARY_SPEC, - warmup: 1000, - iterations: 10_000, + library, + warmup, + iterations, operation: 'deserialize', - options: {} + options: {}, + tags: [ALERT_TAG] }); // Add deep bson encoding suite.task({ documentPath: join(DOCUMENT_ROOT, 'deep_bson.json'), - library: LIBRARY_SPEC, - warmup: 1000, - iterations: 10_000, + library, + warmup, + iterations, operation: 'serialize', - options: {} + options: {}, + tags: [ALERT_TAG] }); // Add deep bson decoding suite.task({ documentPath: join(DOCUMENT_ROOT, 'deep_bson.json'), - library: LIBRARY_SPEC, - warmup: 1000, - iterations: 10_000, + library, + warmup, + iterations, operation: 'deserialize', - options: {} + options: {}, + tags: [ALERT_TAG] }); // Add full bson encoding suite.task({ documentPath: join(DOCUMENT_ROOT, 'full_bson.json'), - library: LIBRARY_SPEC, - warmup: 1000, - iterations: 10_000, + library, + warmup, + iterations, operation: 'serialize', - options: {} + options: {}, + tags: [ALERT_TAG] }); // Add full bson decoding suite.task({ documentPath: join(DOCUMENT_ROOT, 'full_bson.json'), - library: LIBRARY_SPEC, - warmup: 1000, - iterations: 10_000, + library, + warmup, + iterations, operation: 'deserialize', - options: {} + options: {}, + tags: [ALERT_TAG] }); -suite.run().then( - () => { - const results = suite.results.map(result => { - const rv = { ...result }; - rv.metrics = rv.metrics.filter(metric => metric.type === 'MEAN'); - return rv; - }); - // calculte BSONBench composite score - const bsonBenchComposite = - results.reduce((prev, result) => { - // find MEAN - let resultMean: number | undefined = undefined; - for (const metric of result.metrics) { - if (metric.type === 'MEAN') { - resultMean = metric.value; - } - } +suite.run().then(async () => { + const cpuBaseline = await readFile( + join(__dirname, '..', '..', 'etc', 'cpuBaseline.json'), + 'utf8' + ); + + const cpuBaselineResult = JSON.parse(cpuBaseline).megabytes_per_second; + + if (typeof cpuBaselineResult !== 'number') + throw new Error('Could not find correctly formatted baseline results'); - if (!resultMean) throw new Error('Failed to calculate results'); - - return prev + resultMean; - }, 0) / results.length; - - // Add to results - results.push({ - info: { - test_name: 'BSONBench', - tags: [], - args: {} - }, - metrics: [ - { - name: 'BSONBench composite score', - type: 'THROUGHPUT', - value: bsonBenchComposite + const suiteResults = suite.results as { + info: PerfSendResult['info']; + metrics: (PerfSendResult['metrics'][0] & { metadata?: Metadata })[]; + }[]; + const results = suiteResults.map(result => { + const rv = { ...result }; + rv.metrics = rv.metrics.filter(metric => metric.type === 'MEAN'); + return rv; + }); + + const metadata: Metadata = { improvement_direction: 'up' }; + // calculate BSONBench composite score + const bsonBenchComposite = + results.reduce((prev, result) => { + // find MEAN + let resultMean: number | undefined = undefined; + for (const metric of result.metrics) { + if (metric.type === 'MEAN') { + resultMean = metric.value; } - ] - }); + } + + if (!resultMean) throw new Error('Failed to calculate results'); - // Write results to file - return writeFile('bsonBench.json', JSON.stringify(results)); - }, - error => { - console.error(error); + return prev + resultMean; + }, 0) / results.length; + + for (const r of results) { + r.metrics[0].metadata = { ...r.metrics[0].metadata, ...metadata }; + r.metrics.push({ + name: 'normalized_throughput', + value: r.metrics[0].value / cpuBaselineResult, + metadata: { ...r.metrics[0].metadata } + }); } -); + + // Add to results + results.push({ + info: { + test_name: 'BSONBench', + args: {} + }, + metrics: [ + { + name: 'BSONBench composite score', + type: 'THROUGHPUT', + value: bsonBenchComposite, + metadata + } + ] + }); + + results.push({ + info: { + test_name: 'cpuBaseline', + args: {} + }, + metrics: [{ name: 'mean_megabytes_per_second', value: cpuBaselineResult, metadata }] + }); + + // Write results to file + return writeFile('bsonBench.json', JSON.stringify(results)); +});