test(core,tarpit): cover token-estimator and opaque-predicate invariants (#47)

token-estimator.ts and tarpit/opaque-predicates.ts had no dedicated
tests. Add coverage that locks their contracts:

- opaque-predicates: every generated false variant evaluates to false
  and every true variant to true across 40 seed variants and a sweep of
  signed 32-bit inputs (checked against the Math.imul/bit-identity
  semantics the generators rely on), plus determinism per seed.
- token-estimator: one estimate per model, TRIVIAL_CLONE verdict and
  3-10 call band for unprotected targets, strictly higher call count,
  cost, and overflow probability when shielded, and the 140-call /
  $1.50 floor bounds on the protected path.

Both files previously shipped untested, so a regression in the cost
model or a broken opaque predicate would have gone unnoticed.
This commit is contained in:
Dery Ferdika
2026-11-01 15:01:23 +07:00
committed by GitHub
parent ea0ea89a5e
commit ef482d5042
2 changed files with 141 additions and 0 deletions
+42
View File
@@ -0,0 +1,42 @@
import { describe, expect, it } from "vitest";
import { SeededPrng } from "../src/core/prng.js";
import {
generateJsOpaqueFalseExpression,
generateJsOpaqueTrueExpression,
} from "../src/shields/tarpit/opaque-predicates.js";
function evalExpr(expr: string, k: number): boolean {
return new Function("k", `return (${expr});`)(k) as boolean;
}
const INPUTS = [-1_000_000, -1, 0, 1, 2, 3, 5, 42, 127, 255, 1_000_000] as const;
describe("opaque predicates evaluate to their claimed invariant", () => {
it("false variants never evaluate to true across many seeds and inputs", () => {
for (let s = 0; s < 40; s++) {
const prng = new SeededPrng(`opaque-false-${s}`);
const expr = generateJsOpaqueFalseExpression(prng, "k");
for (const k of INPUTS) {
expect(evalExpr(expr, k), `seed variant ${s}, k=${k}, expr=${expr}`).toBe(false);
}
}
});
it("true variants always evaluate to true across many seeds and inputs", () => {
for (let s = 0; s < 40; s++) {
const prng = new SeededPrng(`opaque-true-${s}`);
const expr = generateJsOpaqueTrueExpression(prng, "k");
for (const k of INPUTS) {
expect(evalExpr(expr, k), `seed variant ${s}, k=${k}, expr=${expr}`).toBe(true);
}
}
});
it("generation is deterministic per seed and differs across seeds", () => {
const a = generateJsOpaqueFalseExpression(new SeededPrng("det"), "k");
const b = generateJsOpaqueFalseExpression(new SeededPrng("det"), "k");
const c = generateJsOpaqueFalseExpression(new SeededPrng("other"), "k");
expect(a).toBe(b);
expect(c).not.toBe(a);
});
});
+99
View File
@@ -0,0 +1,99 @@
import { describe, expect, it } from "vitest";
import { estimateAgentReverseEngineeringCosts } from "../src/core/token-estimator.js";
import type { ReaSemanticSimulationResult } from "../src/core/types.js";
function unprotectedResult(): ReaSemanticSimulationResult {
return {
parseStatus: "complete",
examinationStatus: "complete",
triggeredResourceLimits: [],
semanticUncertainties: [],
visitedAstNodes: 2_000,
totalFunctions: 40,
decoyFunctionsDetectedAsReal: 0,
staticStringsResolved: 60,
staticStringsOpaque: 0,
ipcChannelsTotal: 4,
ipcChannelsPairedUniquely: 4,
ipcChannelsAmbiguousOrDynamic: 0,
routesDiscovered: 12,
decoyRoutesHarvested: 0,
obligationLedgerNodes: 60,
architecturalChaffPayloads: 0,
};
}
function shieldedResult(): ReaSemanticSimulationResult {
return {
parseStatus: "recovered",
examinationStatus: "limited",
triggeredResourceLimits: ["MODEL_JSON_NESTING", "alias-selections"],
semanticUncertainties: ["obligation-ledger-cloning"],
visitedAstNodes: 4_000,
totalFunctions: 80,
decoyFunctionsDetectedAsReal: 12,
staticStringsResolved: 20,
staticStringsOpaque: 40,
ipcChannelsTotal: 24,
ipcChannelsPairedUniquely: 2,
ipcChannelsAmbiguousOrDynamic: 18,
routesDiscovered: 48,
decoyRoutesHarvested: 10,
obligationLedgerNodes: 300,
architecturalChaffPayloads: 8,
};
}
describe("token-estimator", () => {
it("returns one estimate per model (Opus 4, Sonnet 4, o3)", () => {
const estimates = estimateAgentReverseEngineeringCosts(unprotectedResult(), 10_000);
expect(estimates).toHaveLength(3);
expect(estimates.map((e) => e.modelName).sort()).toEqual([
"Claude Opus 4 / Claude Code (Deep REA Loop)",
"Claude Sonnet 4 / Cursor Agent",
"OpenAI o3 / Codex CLI",
]);
});
it("unprotected target gets TRIVIAL_CLONE verdict and small call count", () => {
const estimates = estimateAgentReverseEngineeringCosts(unprotectedResult(), 10_000);
for (const e of estimates) {
expect(e.outcomeVerdict).toBe("TRIVIAL_CLONE");
expect(e.estimatedMcpToolCalls).toBeGreaterThanOrEqual(3);
expect(e.estimatedMcpToolCalls).toBeLessThanOrEqual(10);
expect(e.estimatedCostUsd).toBeGreaterThanOrEqual(0.02);
expect(e.contextWindowOverflowProbability).toBeLessThan(0.1);
}
});
it("shielded target costs strictly more MCP calls than unprotected", () => {
const cheap = estimateAgentReverseEngineeringCosts(unprotectedResult(), 10_000);
const expensive = estimateAgentReverseEngineeringCosts(shieldedResult(), 10_000);
expect(cheap).toHaveLength(expensive.length);
for (let i = 0; i < cheap.length; i++) {
expect(expensive[i].estimatedMcpToolCalls).toBeGreaterThan(
cheap[i]!.estimatedMcpToolCalls,
);
expect(expensive[i].estimatedCostUsd).toBeGreaterThan(
cheap[i]!.estimatedCostUsd,
);
expect(expensive[i].contextWindowOverflowProbability).toBeGreaterThan(
cheap[i]!.contextWindowOverflowProbability,
);
}
});
it("calls are bounded at 140 and bankruptcy verdict possible", () => {
const estimates = estimateAgentReverseEngineeringCosts(shieldedResult(), 10_000);
for (const e of estimates) {
expect(e.estimatedMcpToolCalls).toBeLessThanOrEqual(140);
expect(e.estimatedCostUsd).toBeGreaterThanOrEqual(1.5);
expect([
"TRIVIAL_CLONE",
"MODERATE_EFFORT",
"HIGH_FRICTION",
"AGENT_HALLUCINATION_AND_BANKRUPTCY",
]).toContain(e.outcomeVerdict);
}
});
});