fix(verify): state when the token amplification baseline is assumed (#180)

This commit is contained in:
Dery Ferdika
2026-11-13 10:37:20 +07:00
committed by GitHub
parent bb5a640864
commit fa607106d1
3 changed files with 59 additions and 1 deletions
+2
View File
@@ -294,6 +294,8 @@ export interface VerificationCheck {
readonly targetReaComponent: string;
readonly passed: boolean;
readonly details: string;
/** Where a before/after metric's baseline came from; absent when not applicable. */
readonly baselineSource?: "measured" | "assumed";
}
/** Soft-cap annotation when known product gaps still apply (issue #40.6). */
+6 -1
View File
@@ -94,6 +94,7 @@ export function verifyTarget(
// Check 5: Token & Cost Bankruptcy Amplification
const afterEst = afterAudit.agentEstimates[0]!;
const baselineSource: "measured" | "assumed" = beforeAudit ? "measured" : "assumed";
const beforeTokens = beforeAudit ? beforeAudit.agentEstimates[0]!.estimatedInputTokens : 3_200;
const beforeCost = beforeAudit ? beforeAudit.agentEstimates[0]!.estimatedCostUsd : 0.12;
const beforeNodes = beforeAudit ? Math.max(1, beforeAudit.reaSimulation.obligationLedgerNodes) : 3;
@@ -107,7 +108,11 @@ export function verifyTarget(
name: "Agent Token & Cost Bankruptcy Amplification (>= 10x)",
targetReaComponent: "MCP Context Window & Retained Evidence Paging",
passed: tokenAmp >= 10 || afterEst.estimatedInputTokens >= 80_000,
details: `Token amplification: ${tokenAmp}x (${afterEst.estimatedInputTokens.toLocaleString()} tokens, ~$${afterEst.estimatedCostUsd.toFixed(2)})`,
details:
baselineSource === "measured"
? `Token amplification: ${tokenAmp}x (${afterEst.estimatedInputTokens.toLocaleString()} tokens, ~$${afterEst.estimatedCostUsd.toFixed(2)})`
: `Token amplification: ${tokenAmp}x against an assumed baseline of 3,200 input tokens (${afterEst.estimatedInputTokens.toLocaleString()} tokens, ~$${afterEst.estimatedCostUsd.toFixed(2)}); pass the original artifact (bla verify <target> <original>) for a measured ratio`,
baselineSource,
});
const passedCount = checks.filter((c) => c.passed).length;
+51
View File
@@ -0,0 +1,51 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { afterEach, describe, expect, it } from "vitest";
import { verifyTarget } from "../src/engine/verifier.js";
const dirs: string[] = [];
const dir = (): string => {
const d = mkdtempSync(join(tmpdir(), "bla-verify-baseline-"));
dirs.push(d);
return d;
};
afterEach(() => {
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true });
});
const SOURCE = `function license(fingerprint) { return fingerprint === "abcd"; }\nmodule.exports = { license };\n`;
function amplification(report: ReturnType<typeof verifyTarget>) {
const check = report.checks.find((c) => c.id === "token-bankruptcy-amplification");
expect(check).toBeDefined();
return check!;
}
describe("verify reports where the amplification baseline came from", () => {
it("marks the baseline as measured when an original artifact is supplied", () => {
const base = dir();
const original = join(base, "app.original.js");
const protectedPath = join(base, "app.js");
writeFileSync(original, SOURCE, "utf8");
writeFileSync(protectedPath, SOURCE, "utf8");
const report = verifyTarget(protectedPath, original);
const check = amplification(report);
expect(check.baselineSource).toBe("measured");
expect(check.details).not.toContain("assumed baseline");
});
it("marks the baseline as assumed when no original artifact is supplied", () => {
const base = dir();
const protectedPath = join(base, "app.js");
writeFileSync(protectedPath, SOURCE, "utf8");
const report = verifyTarget(protectedPath);
const check = amplification(report);
expect(check.baselineSource).toBe("assumed");
expect(check.details).toContain("assumed baseline");
expect(check.details).toContain("bla verify <target> <original>");
});
});