Add benchmark script

The current build setup doesn't seem to have a concept for benchmark
scripts, so for now you'll have to run it with something like ts-node.
This commit is contained in:
Asger F
2024-11-19 11:23:46 +01:00
parent fac7961e2d
commit 2cde3b9c2f

View File

@@ -0,0 +1,73 @@
import { readFile } from "fs-extra";
import { readJsonlFile } from "../../src/common/jsonl-reader";
import { performance } from "perf_hooks";
import { join } from "path";
/** An "obviously correct" implementation to test against. */
async function readJsonlReferenceImpl<T>(
path: string,
handler: (value: T) => Promise<void>,
): Promise<void> {
const logSummary = await readFile(path, "utf-8");
// Remove newline delimiters because summary is in .jsonl format.
const jsonSummaryObjects: string[] = logSummary.split(/\r?\n\r?\n/g);
for (const obj of jsonSummaryObjects) {
const jsonObj = JSON.parse(obj) as T;
await handler(jsonObj);
}
}
type ParserFn = (
text: string,
callback: (v: unknown) => Promise<void>,
) => Promise<void>;
const parsers: Record<string, ParserFn> = {
readJsonlReferenceImpl,
readJsonlFile,
};
async function main() {
const args = process.argv.slice(2);
const file =
args.length > 0
? args[0]
: join(
__dirname,
"../unit-tests/data/evaluator-log-summaries/bad-join-order.jsonl",
);
const numTrials = args.length > 1 ? Number(args[1]) : 100;
const referenceValues: any[] = [];
await readJsonlReferenceImpl(file, async (event) => {
referenceValues.push(event);
});
const referenceValueString = JSON.stringify(referenceValues);
// Do warm-up runs and check against reference implementation
for (const [name, parser] of Object.entries(parsers)) {
const values: unknown[] = [];
await parser(file, async (event) => {
values.push(event);
});
if (JSON.stringify(values) !== referenceValueString) {
console.error(`${name}: failed to match reference implementation`);
}
}
for (const [name, parser] of Object.entries(parsers)) {
const startTime = performance.now();
for (let i = 0; i < numTrials; ++i) {
await Promise.all([
parser(file, async () => {}),
parser(file, async () => {}),
]);
}
const duration = performance.now() - startTime;
const durationPerTrial = duration / numTrials;
console.log(`${name}: ${durationPerTrial.toFixed(1)} ms`);
}
}
main().catch((err: unknown) => {
console.error(err);
});