mirror of
https://github.com/qybaihe/bla.git
synced 2026-11-20 08:20:59 +08:00
fix(verify): state when the token amplification baseline is assumed (#180)
This commit is contained in:
@@ -294,6 +294,8 @@ export interface VerificationCheck {
|
||||
readonly targetReaComponent: string;
|
||||
readonly passed: boolean;
|
||||
readonly details: string;
|
||||
/** Where a before/after metric's baseline came from; absent when not applicable. */
|
||||
readonly baselineSource?: "measured" | "assumed";
|
||||
}
|
||||
|
||||
/** Soft-cap annotation when known product gaps still apply (issue #40.6). */
|
||||
|
||||
@@ -94,6 +94,7 @@ export function verifyTarget(
|
||||
|
||||
// Check 5: Token & Cost Bankruptcy Amplification
|
||||
const afterEst = afterAudit.agentEstimates[0]!;
|
||||
const baselineSource: "measured" | "assumed" = beforeAudit ? "measured" : "assumed";
|
||||
const beforeTokens = beforeAudit ? beforeAudit.agentEstimates[0]!.estimatedInputTokens : 3_200;
|
||||
const beforeCost = beforeAudit ? beforeAudit.agentEstimates[0]!.estimatedCostUsd : 0.12;
|
||||
const beforeNodes = beforeAudit ? Math.max(1, beforeAudit.reaSimulation.obligationLedgerNodes) : 3;
|
||||
@@ -107,7 +108,11 @@ export function verifyTarget(
|
||||
name: "Agent Token & Cost Bankruptcy Amplification (>= 10x)",
|
||||
targetReaComponent: "MCP Context Window & Retained Evidence Paging",
|
||||
passed: tokenAmp >= 10 || afterEst.estimatedInputTokens >= 80_000,
|
||||
details: `Token amplification: ${tokenAmp}x (${afterEst.estimatedInputTokens.toLocaleString()} tokens, ~$${afterEst.estimatedCostUsd.toFixed(2)})`,
|
||||
details:
|
||||
baselineSource === "measured"
|
||||
? `Token amplification: ${tokenAmp}x (${afterEst.estimatedInputTokens.toLocaleString()} tokens, ~$${afterEst.estimatedCostUsd.toFixed(2)})`
|
||||
: `Token amplification: ${tokenAmp}x against an assumed baseline of 3,200 input tokens (${afterEst.estimatedInputTokens.toLocaleString()} tokens, ~$${afterEst.estimatedCostUsd.toFixed(2)}); pass the original artifact (bla verify <target> <original>) for a measured ratio`,
|
||||
baselineSource,
|
||||
});
|
||||
|
||||
const passedCount = checks.filter((c) => c.passed).length;
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { verifyTarget } from "../src/engine/verifier.js";
|
||||
|
||||
const dirs: string[] = [];
|
||||
const dir = (): string => {
|
||||
const d = mkdtempSync(join(tmpdir(), "bla-verify-baseline-"));
|
||||
dirs.push(d);
|
||||
return d;
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const SOURCE = `function license(fingerprint) { return fingerprint === "abcd"; }\nmodule.exports = { license };\n`;
|
||||
|
||||
function amplification(report: ReturnType<typeof verifyTarget>) {
|
||||
const check = report.checks.find((c) => c.id === "token-bankruptcy-amplification");
|
||||
expect(check).toBeDefined();
|
||||
return check!;
|
||||
}
|
||||
|
||||
describe("verify reports where the amplification baseline came from", () => {
|
||||
it("marks the baseline as measured when an original artifact is supplied", () => {
|
||||
const base = dir();
|
||||
const original = join(base, "app.original.js");
|
||||
const protectedPath = join(base, "app.js");
|
||||
writeFileSync(original, SOURCE, "utf8");
|
||||
writeFileSync(protectedPath, SOURCE, "utf8");
|
||||
|
||||
const report = verifyTarget(protectedPath, original);
|
||||
const check = amplification(report);
|
||||
expect(check.baselineSource).toBe("measured");
|
||||
expect(check.details).not.toContain("assumed baseline");
|
||||
});
|
||||
|
||||
it("marks the baseline as assumed when no original artifact is supplied", () => {
|
||||
const base = dir();
|
||||
const protectedPath = join(base, "app.js");
|
||||
writeFileSync(protectedPath, SOURCE, "utf8");
|
||||
|
||||
const report = verifyTarget(protectedPath);
|
||||
const check = amplification(report);
|
||||
expect(check.baselineSource).toBe("assumed");
|
||||
expect(check.details).toContain("assumed baseline");
|
||||
expect(check.details).toContain("bla verify <target> <original>");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user