diff --git a/docs/process-capture.md b/docs/process-capture.md index f7a117b4a0..0c5841470f 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -94,6 +94,14 @@ deleted, modified, or unchanged. These are bounded snapshots, not a syscall trace; short-lived changes between snapshots may be missed. With no selected paths, filesystem effects remain unknown. +A missing entry proves creation or deletion only when the corresponding +observation root's path enumeration was exhausted. Otherwise its effect has +`status: "unknown"`, the available before/after state, and a reason; a null +state in this variant means unobserved. Paths below symlinks that REA did not +follow also remain unknown. Partial enumeration of one root does not erase +known effects in another, and unavailable content hashes do not by themselves +make path absence unknown. + Terminal frame `data` is the comparison text after the selected normalization. New captures also preserve `raw_data` when normalization changes a PTY chunk; otherwise `data` is already the original text. Older captures may lack the diff --git a/src/domain/process/processCapture.ts b/src/domain/process/processCapture.ts index 45803af000..24ef7f8624 100644 --- a/src/domain/process/processCapture.ts +++ b/src/domain/process/processCapture.ts @@ -85,6 +85,12 @@ type FileEffect = FileEffectIdentity & readonly before: FileState; readonly after: FileState; } + | { + readonly status: "unknown"; + readonly before: FileState | null; + readonly after: FileState | null; + readonly reason: string; + } ); /** A sampled owned-process observation; sampling cannot prove syscall completeness. */ @@ -348,6 +354,13 @@ const fileEffectSchema = z.discriminatedUnion("status", [ before: fileStateSchema, after: fileStateSchema, }), + z.object({ + path: z.string(), + status: z.literal("unknown"), + before: fileStateSchema.nullable(), + after: fileStateSchema.nullable(), + reason: z.string().min(1), + }), ]); /** Exact serialized shape of a process capture. */ const processSettlementSchema = z.discriminatedUnion("state", [ diff --git a/src/process/capture/FilesystemCoverage.test.ts b/src/process/capture/FilesystemCoverage.test.ts new file mode 100644 index 0000000000..b169242434 --- /dev/null +++ b/src/process/capture/FilesystemCoverage.test.ts @@ -0,0 +1,137 @@ +import { mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { parseProcessScenario } from "../../domain/process/processCapture.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { snapshotRoots } from "./FilesystemSnapshot.js"; +import { classifyFilesystemEffects } from "./ProcessFilesystemEffects.js"; + +it.each(["after", "before"] as const)( + "keeps absence unknown when the %s snapshot omits an existing file", + async (partialSide) => { + const root = await createTestTempDirectory("rea-fs-partial-absence-"); + await writeFile(join(root, "z.txt"), "unchanged"); + const scenario = parseProcessScenario({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [root], + limits: { files: 2 }, + }); + if (partialSide === "before") await writeFile(join(root, "a.txt"), "other"); + const before = await snapshotRoots(scenario); + if (partialSide === "after") await writeFile(join(root, "a.txt"), "other"); + else await rm(join(root, "a.txt")); + const after = await snapshotRoots(scenario); + expect(await readFile(join(root, "z.txt"), "utf8")).toBe("unchanged"); + const effect = classifyFilesystemEffects(before, after).find( + ({ path }) => path === "root_0:z.txt", + ); + expect(effect?.status).toBe("unknown"); + expect(effect).toMatchObject( + partialSide === "after" + ? { before: { path: "root_0:z.txt" }, after: null } + : { before: null, after: { path: "root_0:z.txt" } }, + ); + }, +); + +it("preserves proven deletion in an exhausted root while another root is partial", async () => { + const root = await createTestTempDirectory("rea-fs-multiple-roots-"); + const complete = join(root, "complete"); + const partial = join(root, "partial"); + await mkdir(complete); + await mkdir(partial); + await writeFile(join(complete, "removed.txt"), "removed"); + await writeFile(join(partial, "first.txt"), "first"); + await writeFile(join(partial, "second.txt"), "second"); + const scenario = parseProcessScenario({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [complete, partial], + limits: { files: 3 }, + }); + const before = await snapshotRoots(scenario); + await rm(join(complete, "removed.txt")); + const after = await snapshotRoots(scenario); + expect(before.truncated && after.truncated).toBe(true); + expect( + classifyFilesystemEffects(before, after).find( + ({ path }) => path === "root_0:removed.txt", + )?.status, + ).toBe("deleted"); +}); + +it("distinguishes unavailable content hashes from completed path enumeration", async () => { + const root = await createTestTempDirectory("rea-fs-hash-coverage-"); + await writeFile(join(root, "removed.txt"), "larger than the byte budget"); + const scenario = parseProcessScenario({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [root], + limits: { file_bytes: 1 }, + }); + const before = await snapshotRoots(scenario); + await rm(join(root, "removed.txt")); + const after = await snapshotRoots(scenario); + expect(before.truncated).toBe(true); + expect( + classifyFilesystemEffects(before, after).find( + ({ path }) => path === "root_0:removed.txt", + ), + ).toMatchObject({ status: "deleted", before: { sha256: null }, after: null }); +}); + +it("preserves creation when the selected root was observed missing before capture", async () => { + const root = await createTestTempDirectory("rea-fs-missing-root-"); + const file = join(root, "created.txt"); + const scenario = parseProcessScenario({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [file], + }); + const before = await snapshotRoots(scenario); + await writeFile(file, "created"); + const after = await snapshotRoots(scenario); + expect(before.files).toEqual([]); + expect(before.truncated).toBe(false); + expect(classifyFilesystemEffects(before, after)).toMatchObject([ + { + path: "root_0:.", + status: "created", + before: null, + after: { type: "file" }, + }, + ]); +}); + +it.skipIf(process.platform === "win32")( + "keeps paths below an unfollowed replacement symlink unknown while preserving sibling deletion", + async () => { + const root = await createTestTempDirectory("rea-fs-symlink-coverage-"); + const target = await createTestTempDirectory("rea-fs-symlink-target-"); + await mkdir(join(root, "subtree")); + await writeFile(join(root, "subtree", "z.txt"), "unchanged"); + await writeFile(join(target, "z.txt"), "unchanged"); + await writeFile(join(root, "removed.txt"), "removed"); + const scenario = parseProcessScenario({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [root], + }); + const before = await snapshotRoots(scenario); + await rm(join(root, "subtree"), { recursive: true }); + await symlink(target, join(root, "subtree")); + await rm(join(root, "removed.txt")); + const after = await snapshotRoots(scenario); + expect(await readFile(join(root, "subtree", "z.txt"), "utf8")).toBe( + "unchanged", + ); + const effects = classifyFilesystemEffects(before, after); + expect( + effects.find(({ path }) => path === "root_0:subtree/z.txt")?.status, + ).toBe("unknown"); + expect( + effects.find(({ path }) => path === "root_0:removed.txt")?.status, + ).toBe("deleted"); + }, +); diff --git a/src/process/capture/FilesystemSnapshot.ts b/src/process/capture/FilesystemSnapshot.ts index aaf15fda82..b8880a6306 100644 --- a/src/process/capture/FilesystemSnapshot.ts +++ b/src/process/capture/FilesystemSnapshot.ts @@ -11,6 +11,8 @@ import type { Stats } from "node:fs"; export interface SnapshotResult { readonly files: readonly FileState[]; readonly truncated: boolean; + /** Root aliases whose path enumeration was exhausted, regardless of hash coverage. */ + readonly completeRoots: readonly string[]; } const hasSameIdentity = ( @@ -92,6 +94,7 @@ export const snapshotRoots = async ( signal?: AbortSignal, ): Promise => { const entries: FileState[] = []; + const completeRoots: string[] = []; let remainingBytes = scenario.limits.file_bytes; let truncated = false; const visit = async ( @@ -99,24 +102,24 @@ export const snapshotRoots = async ( rootAlias: string, path: string, depth: number, - ): Promise => { + ): Promise => { signal?.throwIfAborted(); if ( entries.length >= scenario.limits.files || depth > scenario.limits.filesystem_depth ) { truncated = true; - return; + return false; } const stats = await lstatIfPresent(path); - if (stats === undefined) return; + if (stats === undefined) return true; const relativePath = relative(root, path) || "."; if (stats.isSymbolicLink()) { const target = resolve(dirname(path), await readlink(path)); const afterRead = await lstat(path); if (!hasSameIdentity(stats, afterRead, "symlink")) { truncated = true; - return; + return false; } entries.push({ path: `${rootAlias}:${relativePath}`, @@ -126,7 +129,7 @@ export const snapshotRoots = async ( sha256: null, symlink_target: target, }); - return; + return true; } if (stats.isFile()) { const sha256 = @@ -143,7 +146,7 @@ export const snapshotRoots = async ( sha256, symlink_target: null, }); - return; + return true; } const type = stats.isDirectory() ? "directory" : "other"; entries.push({ @@ -154,22 +157,27 @@ export const snapshotRoots = async ( sha256: null, symlink_target: null, }); - if (type !== "directory") return; + if (type !== "directory") return true; const children = await readdir(path); const afterRead = await lstat(path); if (!hasSameIdentity(stats, afterRead, "directory")) { truncated = true; - return; + return false; } - for (const child of children.sort()) - await visit(root, rootAlias, join(path, child), depth + 1); + let complete = true; + for (const child of children.sort()) { + if (!(await visit(root, rootAlias, join(path, child), depth + 1))) + complete = false; + } + return complete; }; for (const [index, root] of scenario.filesystem_observation_paths.entries()) { + const alias = `root_${String(index)}`; if ((await lstatIfPresent(root)) === undefined) { - truncated = true; + completeRoots.push(alias); continue; } - await visit(root, `root_${String(index)}`, root, 0); + if (await visit(root, alias, root, 0)) completeRoots.push(alias); } - return { files: entries, truncated }; + return { files: entries, truncated, completeRoots }; }; diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts index d417e9a2f8..7d8325384f 100644 --- a/src/process/capture/ProcessCaptureLifecycle.ts +++ b/src/process/capture/ProcessCaptureLifecycle.ts @@ -243,6 +243,15 @@ export const buildCaptureResult = ( "No filesystem observation paths were selected; filesystem effects remain unknown."; const hasFilesystemObservations = options.scenario.filesystem_observation_paths.length > 0; + const filesystemEffects = classifyFilesystemEffects( + options.before, + options.after, + ); + const hasUnknownFilesystemEffects = filesystemEffects.some( + ({ status }) => status === "unknown", + ); + const incompleteFilesystemUnknown = + "Path absence in incomplete enumeration or below an unfollowed symbolic link remains unknown."; return { manifest: options.manifest, normalization: options.scenario.normalization, @@ -267,10 +276,7 @@ export const buildCaptureResult = ( event_journal: options.eventJournal, files_before: options.before.files, files_after: options.after.files, - filesystem_effects: classifyFilesystemEffects( - options.before.files, - options.after.files, - ), + filesystem_effects: filesystemEffects, truncated: options.truncated, limitations: [ "The executable digest is a prelaunch file sample; matching path metadata immediately after spawn does not prove an atomic operating-system image binding.", @@ -281,6 +287,7 @@ export const buildCaptureResult = ( "Filesystem observations are before/after snapshots, not syscall traces.", "Inherited host environment variables are not recorded and may affect results.", ...(!hasFilesystemObservations ? [filesystemObservationUnknown] : []), + ...(hasUnknownFilesystemEffects ? [incompleteFilesystemUnknown] : []), ...(hasSensitiveScriptedInput ? [sensitiveInputUnknown] : []), ], residual_unknowns: [ @@ -310,6 +317,14 @@ export const buildCaptureResult = ( }, ] : []), + ...(hasUnknownFilesystemEffects + ? [ + { + scope: "filesystem" as const, + reason: incompleteFilesystemUnknown, + }, + ] + : []), ...(hasSensitiveScriptedInput ? [{ scope: "interaction" as const, reason: sensitiveInputUnknown }] : []), diff --git a/src/process/capture/ProcessCaptureObservation.test.ts b/src/process/capture/ProcessCaptureObservation.test.ts index 9f476bf262..f73efb2bfe 100644 --- a/src/process/capture/ProcessCaptureObservation.test.ts +++ b/src/process/capture/ProcessCaptureObservation.test.ts @@ -74,8 +74,8 @@ it("preserves rendered observation order instead of timestamp sorting", () => { frames: [], exit: { exitCode: 0, reason: "exited" }, samples: [], - before: { files: [], truncated: false }, - after: { files: [], truncated: false }, + before: { files: [], truncated: false, completeRoots: [] }, + after: { files: [], truncated: false, completeRoots: [] }, truncated: false, scenario: parseProcessScenario(base), rootPid: 1, @@ -100,8 +100,8 @@ it("marks redacted scripted input as an interaction unknown", () => { frames: [], exit: { exitCode: 0, reason: "exited" }, samples: [], - before: { files: [], truncated: false }, - after: { files: [], truncated: false }, + before: { files: [], truncated: false, completeRoots: [] }, + after: { files: [], truncated: false, completeRoots: [] }, truncated: false, scenario: parseProcessScenario({ ...base, diff --git a/src/process/capture/ProcessFilesystemEffects.ts b/src/process/capture/ProcessFilesystemEffects.ts index 59befa226a..ec47c90eca 100644 --- a/src/process/capture/ProcessFilesystemEffects.ts +++ b/src/process/capture/ProcessFilesystemEffects.ts @@ -2,19 +2,71 @@ import type { FileState, FilesystemCheckpoint, } from "../../domain/process/processCapture.js"; +import { sep } from "node:path"; +import type { SnapshotResult } from "./FilesystemSnapshot.js"; -/** Classify path-stable filesystem effects between two bounded states. */ +const unobservedAbsenceReason = ( + path: string, + phase: "before" | "after", + completeRoots: ReadonlySet, + filesByPath: ReadonlyMap, +): string | undefined => { + const alias = path.slice(0, path.indexOf(":")); + if (!completeRoots.has(alias)) + return `The ${phase} snapshot did not exhaust path enumeration under ${alias}; absence remains unknown.`; + const rootPath = `${alias}:.`; + if (path !== rootPath && filesByPath.get(rootPath)?.type === "symlink") + return `The ${phase} snapshot did not follow the symbolic link at ${rootPath}; descendant absence remains unknown.`; + for ( + let index = path.lastIndexOf(sep); + index > 0; + index = path.lastIndexOf(sep, index - 1) + ) { + const parent = path.slice(0, index); + if (filesByPath.get(parent)?.type === "symlink") + return `The ${phase} snapshot did not follow the symbolic link at ${parent}; descendant absence remains unknown.`; + } + return undefined; +}; + +/** Classify effects only when a snapshot establishes the missing path's absence. */ export const classifyFilesystemEffects = ( - before: readonly FileState[], - after: readonly FileState[], + before: SnapshotResult, + after: SnapshotResult, ): FilesystemCheckpoint["effects"] => { - const beforeByPath = new Map(before.map((file) => [file.path, file])); - const afterByPath = new Map(after.map((file) => [file.path, file])); + const beforeByPath = new Map(before.files.map((file) => [file.path, file])); + const afterByPath = new Map(after.files.map((file) => [file.path, file])); + const beforeCompleteRoots = new Set(before.completeRoots); + const afterCompleteRoots = new Set(after.completeRoots); return [...new Set([...beforeByPath.keys(), ...afterByPath.keys()])] .sort() .map((path) => { const beforeFile = beforeByPath.get(path) ?? null; const afterFile = afterByPath.get(path) ?? null; + const reason = + beforeFile === null + ? unobservedAbsenceReason( + path, + "before", + beforeCompleteRoots, + beforeByPath, + ) + : afterFile === null + ? unobservedAbsenceReason( + path, + "after", + afterCompleteRoots, + afterByPath, + ) + : undefined; + if (reason !== undefined) + return { + path, + status: "unknown", + before: beforeFile, + after: afterFile, + reason, + } as const; if (beforeFile === null) { if (afterFile === null) throw new TypeError(`Filesystem effect ${path} has no file state`); diff --git a/src/process/capture/ProcessHarness.ts b/src/process/capture/ProcessHarness.ts index 592202fd3d..d137124998 100644 --- a/src/process/capture/ProcessHarness.ts +++ b/src/process/capture/ProcessHarness.ts @@ -433,7 +433,7 @@ const completeCapture = async (options: { name: "after_settlement", at_ms: Math.max(0, Date.now() - runtime.started), files: after.files, - effects: classifyFilesystemEffects(options.before.files, after.files), + effects: classifyFilesystemEffects(options.before, after), truncated: after.truncated, }, ]; diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index 520caa14b8..dc9e84d28b 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -226,7 +226,8 @@ itWithCaptureCapability( let snapshotCalls = 0; const captureSnapshot: typeof snapshotRoots = async () => { snapshotCalls += 1; - if (snapshotCalls === 1) return { files: [], truncated: false }; + if (snapshotCalls === 1) + return { files: [], truncated: false, completeRoots: [] }; throw new Error("fixture final snapshot failure"); }; const cleanupHost: ProcessCaptureCleanupHost = { diff --git a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts new file mode 100644 index 0000000000..66ca326ec3 --- /dev/null +++ b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts @@ -0,0 +1,139 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { expect, it, onTestFinished } from "vitest"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { + compareProcessCaptures, + parseProcessCapture, +} from "../../../src/domain/process/processCapture.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + CAPTURE_SKIP_REASON, + itWithCaptureCapability, +} from "./processCaptureCapability.js"; +import { z } from "zod"; + +const exec = promisify(execFile); + +const captureViaMcp = async (scenario: Record) => { + const session = createTestBinarySession(() => { + throw new Error("Process capture must not launch a binary provider"); + }); + const server = createServer(session, session); + const client = new Client({ name: "filesystem-coverage", version: "1" }); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const result = await client.callTool({ + name: "capture_process_scenario", + arguments: scenario, + }); + expect(result.isError, JSON.stringify(result)).not.toBe(true); + const evidence = parseEvidence( + z.object({ evidence: z.unknown() }).parse(result.structuredContent) + .evidence, + ); + await client.ping(); + return evidence; +}; + +itWithCaptureCapability.each(["cli", "mcp"] as const)( + "reports unknown absence instead of deletion in truncated %s capture Evidence", + async (adapter) => { + const root = await createTestTempDirectory("rea-fs-coverage-adapter-"); + await writeFile(join(root, "z.txt"), "unchanged"); + const scenario = { + executable: process.execPath, + arguments: ["-e", "require('node:fs').writeFileSync('a.txt','added')"], + working_directory: root, + filesystem_observation_paths: [root], + limits: { files: 2 }, + }; + let evidence; + if (adapter === "cli") { + const scenarioPath = join( + await createTestTempDirectory("rea-fs-scenario-"), + "scenario.json", + ); + await writeFile(scenarioPath, JSON.stringify(scenario)); + const { stdout } = await exec(process.execPath, [ + "scripts/rea.mjs", + "capture-process", + scenarioPath, + "--format", + "json", + ]); + evidence = parseEvidence(JSON.parse(stdout)); + } else evidence = await captureViaMcp(scenario); + const capture = parseProcessCapture(evidence.normalized_result); + expect(capture.truncated).toBe(true); + expect(await readFile(join(root, "z.txt"), "utf8")).toBe("unchanged"); + expect( + capture.filesystem_effects.find(({ path }) => path === "root_0:z.txt") + ?.status, + ).toBe("unknown"); + expect( + capture.filesystem_checkpoints + .find(({ name }) => name === "after_settlement") + ?.effects.find(({ path }) => path === "root_0:z.txt")?.status, + ).toBe("unknown"); + }, + 20_000, +); + +it.skipIf(CAPTURE_SKIP_REASON || process.platform === "win32")( + "keeps unfollowed descendant absence unknown through MCP Evidence and comparison", + async () => { + const root = await createTestTempDirectory("rea-fs-symlink-adapter-"); + const target = await createTestTempDirectory( + "rea-fs-symlink-adapter-target-", + ); + await mkdir(join(root, "subtree")); + await writeFile(join(root, "subtree", "z.txt"), "unchanged"); + await writeFile(join(target, "z.txt"), "unchanged"); + await writeFile(join(root, "removed.txt"), "removed"); + const evidence = await captureViaMcp({ + executable: process.execPath, + arguments: [ + "-e", + "const fs=require('node:fs');fs.rmSync('subtree',{recursive:true});fs.symlinkSync(process.argv[1],'subtree');fs.unlinkSync('removed.txt');", + target, + ], + working_directory: root, + filesystem_observation_paths: [root], + }); + const capture = parseProcessCapture(evidence.normalized_result); + expect(await readFile(join(root, "subtree", "z.txt"), "utf8")).toBe( + "unchanged", + ); + expect( + capture.filesystem_effects.find( + ({ path }) => path === "root_0:subtree/z.txt", + )?.status, + ).toBe("unknown"); + expect( + capture.filesystem_effects.find( + ({ path }) => path === "root_0:removed.txt", + )?.status, + ).toBe("deleted"); + expect(capture.residual_unknowns).toContainEqual({ + scope: "filesystem", + reason: expect.any(String), + }); + expect(["unknown", "truncated"]).toContain( + compareProcessCaptures(capture, capture).filesystem, + ); + }, + 20_000, +); diff --git a/tests/boundary/process/processPreparation.test.ts b/tests/boundary/process/processPreparation.test.ts index 262a0cb260..2b0d1bee20 100644 --- a/tests/boundary/process/processPreparation.test.ts +++ b/tests/boundary/process/processPreparation.test.ts @@ -53,7 +53,7 @@ it("cleans the temporary root when cancellation arrives during identity capture" working_directory: "/tmp", }), controller.signal, - async () => ({ files: [], truncated: false }), + async () => ({ files: [], truncated: false, completeRoots: [] }), host, ), ).rejects.toThrow(/cancelled/u);