Stack benchmarks (v2)

.github/workflows/bench-pr.yml

@@ -41,3 +41,5 @@ jobs:
           path: |
             bench/bench.json
             bench/download.json
+            bench/stack.json
+            schemas/libraries/**/download_compiled/**

bench/package.json

@@ -7,11 +7,13 @@
   "exports": {
     ".": "./src/index.ts",
     "./bench.json": "./bench.json",
-    "./download.json": "./download.json"
+    "./download.json": "./download.json",
+    "./stack.json": "./stack.json"
   },
   "scripts": {
     "bench": "node ./src/scripts/bench/orchestrate.ts",
     "download": "node ./src/scripts/download.ts",
+    "stack": "node ./src/scripts/stack.ts",
     "typecheck": "tsgo"
   },
   "dependencies": {

bench/src/results/types.ts

@@ -101,3 +101,16 @@ export const downloadResultsSchema = v.object({
   unminified: v.array(downloadResultSchema),
 });
 export type DownloadResults = v.InferOutput<typeof downloadResultsSchema>;
+
+const resultTypeSchema = v.picklist(["success", "not an error", "no stack", "not found"]);
+
+export const stackResultSchema = v.object({
+  type: resultTypeSchema,
+  libraryName: v.string(),
+  version: v.string(),
+  snippet: v.string(),
+  frame: v.optional(v.number()),
+  output: v.string(),
+  lineCount: v.number(),
+});
+export type StackResult = v.InferOutput<typeof stackResultSchema>;

bench/src/scripts/bench/orchestrate.ts

@@ -5,6 +5,7 @@ import { promisify } from "node:util";
 
 import { libraries } from "@schema-benchmarks/schemas/libraries";
 import { unsafeEntries } from "@schema-benchmarks/utils";
+import { forwardStd } from "@schema-benchmarks/utils/node";
 import * as v from "valibot";
 
 import type { BenchResults } from "../../results/types.ts";
@@ -19,20 +20,12 @@ const execFile = promisify(child_process.execFile);
 
 const allResults: Array<BenchResults> = [];
 
-function forward<T>(promise: child_process.PromiseWithChild<T>) {
-  promise.child.stdout?.pipe(process.stdout);
-  promise.child.stderr?.pipe(process.stderr);
-  return promise;
-}
-
 for (const lib of Object.keys(libraries)) {
-  const libResult = await forward(
+  const libResult = await forwardStd(
     execFile(
       process.execPath,
       [path.resolve(process.cwd(), "./src/scripts/bench/library.ts"), `--lib=${lib}`],
-      {
-        signal: sigintAc.signal,
-      },
+      { signal: sigintAc.signal },
     ),
   );
   const results = libResult.stdout.split("\n").slice(-3).filter(Boolean).pop();

bench/src/scripts/stack.ts

@@ -0,0 +1,117 @@
+import * as child_process from "node:child_process";
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import * as process from "node:process";
+import * as url from "node:url";
+import { promisify } from "node:util";
+
+import { assertNotReached, errorData } from "@schema-benchmarks/schemas";
+import { libraries } from "@schema-benchmarks/schemas/libraries";
+import { forwardStd } from "@schema-benchmarks/utils/node";
+
+import type { StackResult } from "../results/types.ts";
+
+const execFile = promisify(child_process.execFile);
+
+// this is probably quite fragile, worth keeping an eye on when we update node
+
+const libDist = import.meta
+  .resolve("@schema-benchmarks/schemas/libraries")
+  .replace("/index.js", "");
+
+const search = `    at Object.throw (${libDist}/`;
+
+const cwdRegex = new RegExp(
+  RegExp.escape(
+    url.pathToFileURL(path.resolve(process.cwd(), "..")).href.replace(/^file:\/\//, ""),
+  ),
+  "g",
+);
+
+async function getLoggedOutput(lib: string) {
+  const { stderr } = await forwardStd(
+    execFile(
+      process.execPath,
+      [path.resolve(process.cwd(), "./src/scripts/stack/log.ts"), `--lib=${lib}`],
+      { env: { ...process.env, FORCE_COLOR: "1" } },
+    ),
+  );
+  const output = stderr.replace(cwdRegex, "");
+  const lineCount = output.split("\n").length;
+  return { output, lineCount };
+}
+
+const results: Array<StackResult> = [];
+
+function getScriptLineNumber(stack?: string) {
+  if (!stack) return;
+  const lines = stack.split("\n");
+  let i = 0;
+  while (i < lines.length) {
+    if (lines[i]?.startsWith(search)) return i === 0 ? undefined : i + 1;
+    i++;
+  }
+}
+
+for (const [lib, getConfig] of Object.entries(libraries)) {
+  const {
+    library: { name: libraryName, version },
+    createContext,
+    stack,
+  } = await getConfig();
+  if (stack) {
+    const { snippet } = stack;
+    try {
+      const context = await createContext();
+      await stack.throw(context, errorData);
+      assertNotReached();
+    } catch (e) {
+      const output = await getLoggedOutput(lib);
+      if (Error.isError(e)) {
+        if (e.name === "ShouldHaveThrownError") throw e;
+        const hasFrames = e.stack?.includes("    at ");
+        if (!hasFrames) {
+          results.push({
+            type: "no stack",
+            libraryName,
+            version,
+            snippet,
+            ...output,
+          });
+          continue;
+        }
+        const frames = e.stack?.slice(e.stack.indexOf("    at "));
+        const frame = getScriptLineNumber(frames);
+        results.push({
+          type: "success",
+          libraryName,
+          version,
+          snippet,
+          frame,
+          ...output,
+        });
+      } else {
+        results.push({
+          type: "not an error",
+          libraryName,
+          version,
+          snippet,
+          ...output,
+        });
+      }
+    }
+  }
+  global.gc?.();
+}
+
+await fs.writeFile(
+  path.join(process.cwd(), "stack.json"),
+  JSON.stringify(
+    results.toSorted((a, b) => {
+      if (typeof a.frame === "number" && typeof b.frame === "number") return a.frame - b.frame;
+      if (typeof a.frame === "number") return -1;
+      if (typeof b.frame === "number") return 1;
+      return 0;
+    }),
+  ),
+);

bench/src/scripts/stack/log.ts

@@ -0,0 +1,32 @@
+import { parseArgs } from "node:util";
+
+import { assertNotReached, errorData } from "@schema-benchmarks/schemas";
+import { libraries } from "@schema-benchmarks/schemas/libraries";
+
+const {
+  values: { lib },
+} = parseArgs({
+  options: {
+    lib: {
+      type: "string",
+      short: "l",
+    },
+  },
+});
+
+if (!lib || !libraries[lib]) {
+  throw new Error(`Library not found: ${lib}`);
+}
+
+const config = await libraries[lib]();
+
+try {
+  await config.stack?.throw(await config.createContext(), errorData);
+  assertNotReached();
+} catch (e) {
+  if (Error.isError(e) && e.name === "ShouldHaveThrownError") {
+    throw e;
+  }
+  // log the coloured output
+  console.error(e);
+}

bench/stack.d.json.ts

@@ -0,0 +1,5 @@
+import type { StackResult } from "./src/results/types.ts";
+
+declare const results: Array<StackResult>;
+
+export default results;

package.json

@@ -15,7 +15,8 @@
     "website:storybook": "pnpm run --filter website storybook",
     "bench:bench": "pnpm run --filter '*/bench' bench",
     "bench:download": "pnpm run --filter '*/bench' download",
-    "bench:all": "pnpm bench:download && pnpm bench:bench",
+    "bench:stack": "pnpm run --filter '*/bench' stack",
+    "bench:all": "pnpm bench:download && pnpm bench:bench && pnpm bench:stack",
     "postinstall": "lefthook install"
   },
   "devDependencies": {

schemas/libraries/ajv/benchmarks.ts

@@ -1,5 +1,6 @@
 import { getVersion } from "@schema-benchmarks/utils/node" with { type: "macro" };
 import type Ajv from "ajv";
+import { ValidationError } from "ajv";
 import type { FormatName } from "ajv-formats";
 import addFormats from "ajv-formats";
 import ts from "dedent";
@@ -66,4 +67,15 @@ export default defineBenchmarks({
     ipv4: createStringBenchmark("ipv4"),
     ipv6: createStringBenchmark("ipv6"),
   },
+  stack: {
+    throw: ({ validate }, data) => {
+      validate(data);
+      throw new ValidationError(validate.errors || []);
+    },
+    snippet: ts`
+      // const validate = ajv.compile(schema);
+      validate(data);
+      throw new ValidationError(validate.errors || []);
+    `,
+  },
 });

schemas/libraries/arktype/benchmarks.ts

@@ -3,7 +3,7 @@ import { type } from "arktype";
 import ts from "dedent";
 
 import type { StringBenchmarkConfig } from "#src";
-import { defineBenchmarks } from "#src";
+import { assertNotReached, defineBenchmarks } from "#src";
 
 import { getArkTypeSchema } from ".";
 
@@ -61,4 +61,11 @@ export default defineBenchmarks({
     ipv4: createStringBenchmark("ip.v4"),
     ipv6: createStringBenchmark("ip.v6"),
   },
+  stack: {
+    throw: ({ schema }, data) => {
+      schema.assert(data);
+      assertNotReached();
+    },
+    snippet: ts`schema.assert(data)`,
+  },
 });

schemas/libraries/effect/benchmarks.ts

@@ -1,9 +1,9 @@
 import { getVersion } from "@schema-benchmarks/utils/node" with { type: "macro" };
 import ts from "dedent";
-import { Either } from "effect";
+import { Effect, Either } from "effect";
 import * as Schema from "effect/Schema";
 
-import { defineBenchmarks } from "#src";
+import { assertNotReached, defineBenchmarks } from "#src";
 
 import { getEffectSchema } from ".";
 
@@ -99,4 +99,17 @@ export default defineBenchmarks({
       `,
     },
   },
+  stack: {
+    throw: ({ decodeAll }, data) => {
+      Effect.runSync(decodeAll(data));
+      assertNotReached();
+    },
+    snippet: ts`
+      // const decodeAll = Schema.decodeUnknownEither(
+      //  schema, 
+      //  { errors: "all" }
+      // );
+      Effect.runSync(decodeAll(data));
+    `,
+  },
 });

schemas/libraries/effect___beta/benchmarks.ts

@@ -3,7 +3,7 @@ import ts from "dedent";
 import { isSome } from "effect___beta/Option";
 import * as Schema from "effect___beta/Schema";
 
-import { defineBenchmarks } from "#src";
+import { assertNotReached, defineBenchmarks } from "#src";
 
 import { getEffectSchema } from ".";
 
@@ -94,4 +94,13 @@ export default defineBenchmarks({
       `,
     },
   },
+  stack: {
+    throw: ({ schema }, data) => {
+      Schema.decodeUnknownSync(schema)(data, { errors: "first" });
+      assertNotReached();
+    },
+    snippet: ts`
+      Schema.decodeUnknownSync(schema)(data, { errors: "first" })
+    `,
+  },
 });

schemas/libraries/joi/benchmarks.ts

@@ -67,4 +67,10 @@ export default defineBenchmarks({
     ipv4: createStringBenchmark("ip", ts`ip({ version: "ipv4" })`, { version: "ipv4" }),
     ipv6: createStringBenchmark("ip", ts`ip({ version: "ipv6" })`, { version: "ipv6" }),
   },
+  stack: {
+    throw: ({ schema }, data) => {
+      throw schema.validate(data).error;
+    },
+    snippet: ts`throw schema.validate(data).error`,
+  },
 });