From f3404be3aa3ca211ebf8440b17d4e323ffabfab6 Mon Sep 17 00:00:00 2001 From: qybaihe Date: Fri, 13 Nov 2026 02:28:03 +0800 Subject: [PATCH] =?UTF-8?q?feat(sidecar):=20GGUF=20metadata-note=20strip?= =?UTF-8?q?=E2=87=92fail=20+=20protectGgufBuffer=20alignment=20fix=20+=20b?= =?UTF-8?q?la-agents@3.2.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend opt-in sidecarEntangle to GGUF via a bla.sidecar.v1 STRING metadata KV (generateSidecarGgufMetadataNote / bindGguf / entangleGguf / verifyGguf), with alignment-safe rebuildGguf so tensor bytes stay bit-exact. Host-verify only (loaders ignore unknown STRING keys); do not record gguf-sidecar-entangle until a load-time hook lands; defenseScore gap gguf-sidecar-host-verify-only. Also fix protectGgufBuffer which previously corrupted tensor data by splicing KV bytes without re-padding the aligned data section. Default OFF. --- .agents/skills/blind-llm-agents/SKILL.md | 4 +- .claude-plugin/marketplace.json | 4 +- .codex-plugin/plugin.json | 4 +- CHANGELOG.md | 11 + README.md | 2 +- STATE.md | 2 +- challenge/v6/SEAL.json | 2 +- challenge/v7/SEAL.json | 2 +- challenge/v8/SEAL.json | 2 +- challenge/v9/SEAL.json | 2 +- docs/design/sidecar-entry-entanglement.md | 5 +- docs/readme/README_zh.md | 2 +- package-lock.json | 4 +- package.json | 2 +- src/core/types.ts | 1 + src/engine/defense-score-honesty.ts | 14 +- src/engine/harden.ts | 43 +++ src/engine/protector.ts | 50 ++- src/identity.ts | 2 +- src/index.ts | 29 ++ src/shields/formats/gguf-layout.ts | 258 ++++++++++++++++ src/shields/formats/model.ts | 20 +- src/shields/runtime/sidecar-entangle.ts | 354 +++++++++++++++++++++- tests/defense-score-honesty.test.ts | 17 ++ tests/gguf-layout.test.ts | 144 +++++++++ tests/sidecar-gguf-inject.test.ts | 276 +++++++++++++++++ 26 files changed, 1232 insertions(+), 24 deletions(-) create mode 100644 src/shields/formats/gguf-layout.ts create mode 100644 tests/gguf-layout.test.ts create mode 100644 tests/sidecar-gguf-inject.test.ts diff --git a/.agents/skills/blind-llm-agents/SKILL.md b/.agents/skills/blind-llm-agents/SKILL.md index 62c88f1..85e7514 100644 --- a/.agents/skills/blind-llm-agents/SKILL.md +++ b/.agents/skills/blind-llm-agents/SKILL.md @@ -9,7 +9,7 @@ description: > MCP tools / CLI — do not invent new MCP tools. Default verb: bla_harden. Refuse crack/RE of third-party software. metadata: - version: "3.2.5" + version: "3.2.6" --- # BLA: Blind LLM Agents (3.0 Agent-Driven Hardening Playbook) @@ -121,7 +121,7 @@ Also: do **not** ship a stable public WASM ABI without explicit `wasm_preserve_e - Integrity checkers must be interleaved and mutually covering (coverage matrix `bla-integrity-coverage/v1`); single-point self-seal is refuse-worthy. Lint with `bla_lint_integrity_coverage`. - Optional bind-scope (domain/date) is a **public** license contract (`bla_bind_scope` → `bind-scope.json` + `BLA_BIND_SCOPE_CHECK`). Default OFF. Never a covert backdoor — `public:true` mandatory; lint with `bla_lint_bind_scope`. - Coverage claims must be real, not paper: an entry may only count as covering itself when its runtime check hard-fails on its own bytes (canonical self-digest, finalized at write). Check `integrityCoverageSelfVerified` on the harden report; when false the matrix drops self edges and refuses. Keep host-trust values (e.g. certificate digests) inside a covered artifact — unhardened data sidecars like `seal.json` are not covered. -- A countered strip⇒fail sidecar vector (`asar-sidecar-entangle`, or `pyc-sidecar-entangle` once it exists at import time) may be recorded only when the entry hook hard-fails at process start or import. A host-only verifier (for example `verifyPycSidecarMarshalNote` on a `.pyc` TYPE_ASCII trailer that `importlib` ignores) is a binding note, not coverage: do not list the vector, and do not treat enabling it as closing the `sidecar-entangle-not-enabled` defenseScore gap (annotate `pyc-sidecar-host-verify-only` instead). +- A countered strip⇒fail sidecar vector (`asar-sidecar-entangle`, or `pyc-sidecar-entangle` / `gguf-sidecar-entangle` once they exist at import/load time) may be recorded only when the entry hook hard-fails at process start, import, or load. A host-only verifier (for example `verifyPycSidecarMarshalNote` on a `.pyc` TYPE_ASCII trailer that `importlib` ignores, or `verifyGgufSidecarMetadataNote` on a GGUF `bla.sidecar.v1` STRING KV that loaders ignore) is a binding note, not coverage: do not list the vector, and do not treat enabling it as closing the `sidecar-entangle-not-enabled` defenseScore gap (annotate `pyc-sidecar-host-verify-only` / `gguf-sidecar-host-verify-only` instead). - **Pinned sibling digests must fail-closed on absence:** if an entry pins `EXPECTED_*_SHA256` (or equivalent) for a data sidecar, `ENOENT` / unreadable MUST throw the same integrity-class error as digest mismatch. Never soft-fold missing pinned files into business `DENIED` — catch blocks that only rethrow `/integrity/` and map everything else to DENIED are a footgun (CrackMe v9 red 2026-11-10: `seal.json` delete looked like a wrong key). Held across R8–R12 (NO PRIZE). Keep in gate design; do **not** invent packaging refuse lints for these: diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 64a2c24..2397c74 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,6 +1,6 @@ { "name": "bla", - "description": "The BLA agent skill — Counter-REA hardening via bla MCP/CLI", + "description": "The BLA agent skill \u2014 Counter-REA hardening via bla MCP/CLI", "owner": { "name": "BLA Contributors", "url": "https://bla.tools" @@ -12,7 +12,7 @@ "description": "Teaches coding agents the BLA Agent-Driven Hardening playbook: harden, audit, weave decoys, and verify Counter-REA defenses via the bla MCP tools and CLI. Use when protecting your own software against AI reverse-engineering agents.", "source": "./", "strict": false, - "version": "3.2.5", + "version": "3.2.6", "author": { "name": "BLA Contributors", "url": "https://bla.tools" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 68669fd..9b111de 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "bla", - "version": "3.2.5", + "version": "3.2.6", "description": "Teaches coding agents the BLA Agent-Driven Hardening playbook: when to harden, audit, weave decoys, and verify Counter-REA defenses via the bla MCP tools and CLI.", "author": { "name": "BLA Contributors", @@ -23,7 +23,7 @@ "interface": { "displayName": "BLA", "shortDescription": "Harden software against AI reverse-engineering agents with BLA", - "longDescription": "BLA (Blind LLM Agents) is a Counter-REA Blue Defense toolkit. This plugin teaches coding agents the Agent-Driven Hardening playbook in `.agents/skills/blind-llm-agents/SKILL.md`: when to call `bla_harden` / `bla_defend`, how to run packaging-gate lint (`bla_lint`), weave domain-plausible shadow business logic from `bla_weave_plan`, and verify with `bla_verify_weave` / `bla_verify`.\n\nUse it when hardening your own software against AI reverse-engineering agents (REA / WorkBuddy-class), auditing release exposure, exhausting attacker token budgets, or one-click defending a project. It points agents at the existing BLA MCP tools and CLI — not at inventing new MCP surfaces. Defensive use only.", + "longDescription": "BLA (Blind LLM Agents) is a Counter-REA Blue Defense toolkit. This plugin teaches coding agents the Agent-Driven Hardening playbook in `.agents/skills/blind-llm-agents/SKILL.md`: when to call `bla_harden` / `bla_defend`, how to run packaging-gate lint (`bla_lint`), weave domain-plausible shadow business logic from `bla_weave_plan`, and verify with `bla_verify_weave` / `bla_verify`.\n\nUse it when hardening your own software against AI reverse-engineering agents (REA / WorkBuddy-class), auditing release exposure, exhausting attacker token budgets, or one-click defending a project. It points agents at the existing BLA MCP tools and CLI \u2014 not at inventing new MCP surfaces. Defensive use only.", "developerName": "BLA Contributors", "category": "Developer Tools", "capabilities": [ diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c91e4c..40be630 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.2.6] — 2026-11-13 + +Patch extending opt-in `sidecarEntangle` to **GGUF STRING metadata notes** (alignment-safe), fixing tensor-corrupting KV splices in `protectGgufBuffer`, and shipping the already-landed ELF64 mapped `.enclave` PT_LOAD honesty work. + ### Added +- **Sidecar GGUF metadata-note (`generateSidecarGgufMetadataNote` / `bindGgufSidecarEntanglement` / `entangleGgufEntryWithSidecars` / `verifyGgufSidecarMetadataNote` / `extractGgufSidecarMetadataNote` / `stripGgufSidecarMetadataNote`):** When `sidecarEntangle` is on, `protectTarget` and the `hardenProject` GGUF post-pass plant a `bla.sidecar.v1` STRING metadata KV. GGUF loaders ignore unknown STRING keys, so tensor load stays runtime-equivalent. Host-side verify re-hashes sibling sidecars (sidecars-mode digest). Vector `gguf-sidecar-entangle` reserved but **not recorded** until a load-time hook lands. Default still OFF. +- **Alignment-safe GGUF layout rewriter (`src/shields/formats/gguf-layout.ts`):** `parseGgufLayout` / `rebuildGguf` keep tensor infos + tensor-data byte-exact and recompute only the `align(endOfTensorInfos, general.alignment)` pad so relative tensor offsets stay valid. +- **defenseScore honesty:** `model-gguf` joins `SIDECAR_ENTANGLE_FORMATS`; enabling the host-only note annotates `gguf-sidecar-host-verify-only` (soft-cap 93) instead of clearing the process-start strip⇒fail gap. - **ELF64 `.so` Mapped `PT_LOAD` `.enclave` Section Hardening (`src/shields/formats/native.ts`, closes #176):** Upgrades ELF64 shared-library (`ET_DYN` on `x86_64` / `aarch64`) post-link hardening from an unmapped EOF trailer to injecting a mapped `PT_LOAD` (`PF_R | PF_X`, strict `W^X`) segment and `.enclave` (`SHT_PROGBITS`, `SHF_ALLOC | SHF_EXECINSTR`) section containing ABI-valid 16-byte aligned System V AMD64 / AArch64 function islands, Itanium C++ ABI RTTI symbols (`_ZTS...`), and POSIX OpenSSL/PKCS#11/SGX dynamic dispatch strings while relocating `e_phoff`/`PT_PHDR` into the mapped segment so `head -c ` truncation invalidates the ELF layout. ### Fixed +- **`protectGgufBuffer` tensor corruption:** previously spliced decoy KV bytes in front of tensor infos without re-padding the data section, so every relative tensor offset resolved to the wrong bytes while the file still "parsed". Now routes tensor-bearing inputs through `rebuildGguf` (verified bit-exact against the reference `gguf` Python reader). - **Post-link Native `defenseScore` Honesty & Trailer Marker Hygiene (`src/engine/defense-score-honesty.ts`, `src/engine/verifier.ts`, `src/shields/formats/go-flutter-exhaustion.ts`, `src/shields/formats/native.ts`, #176):** Soft-caps post-link native `defenseScore` at `78` (`native-postlink-cfg-untouched`, pointing agents to compile-time `bla_emit_headers`) and `68` when trailer-only (`native-trailer-unmapped`), replaces the 1 MiB `'A'` Go buildinfo filler with realistic `dep github.com/...` module lines, and strips self-identifying `[BLA-*]` / `[COGNITIVE_ARMOR_INTERLOCK:*]` banners from native outputs. - **Honesty: `.pyc` marshal-note is host-verify only.** `protectTarget` no longer records `ReaAttackVectorId` `pyc-sidecar-entangle` for planting a TYPE_ASCII trailer. `importlib` ignores post-root trailers, so strip or sibling delete still execute; host `verifyPycSidecarMarshalNote` remains the offline check. An import-time hook is still TBD (same class as the WASM JS-loader). Enabling the note now annotates defenseScore gap `pyc-sidecar-host-verify-only` (soft-cap 93) instead of clearing the process-start strip⇒fail gap. - **CrackMe v9 `seal.json` DELETE fail-closed (red 2026-11-10):** `loadSeal()` now throws `integrity check failed: seal.json missing` on absence instead of letting ENOENT soft-fold to `[license] DENIED` (indistinguishable from a wrong key). Skill B principle: pinned sibling digests must fail-closed on absence. Fixture/cleartext tests cover the invariant; published `challenge/v9/hardened` still needs a release-secret rebuild to absorb the host fix. +### Notes +- REA tip `a8527d43b` (+31 since `0909d89c9`): UnityFS/PE/ELF declared-bounds enforcement, process-capture ownership, setup/MCP catalog + Cline client, release/CI — **no new 1:1 trip-able densify constants**. + ## [3.2.5] — 2026-11-11 Patch extending opt-in `sidecarEntangle` to **CPython `.pyc` marshal trailer notes**, plus Counter-REA limit catalog hygiene for NativeAOT and IL2CPP v108. diff --git a/README.md b/README.md index 0ab49f4..7490121 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ **The open-source Counter-REA Blue Defense MCP & CLI that protects software releases against AI reverse-engineering agents (`morluto/rea`, autonomous decompilers, and LLM recon pipelines) — verified across 14 showcase benchmark targets.** -[![npm version](https://img.shields.io/badge/npm-bla--agents_v3.2.5-2563eb?style=for-the-badge&logo=npm&logoColor=white)](https://www.npmjs.com/package/bla-agents) +[![npm version](https://img.shields.io/badge/npm-bla--agents_v3.2.6-2563eb?style=for-the-badge&logo=npm&logoColor=white)](https://www.npmjs.com/package/bla-agents) [![GitHub Trending](https://img.shields.io/badge/GitHub_Trending-%2315_Daily-7c3aed?style=for-the-badge&logo=github&logoColor=white)](https://trendshift.io/repositories/323569) [![MCP Server](https://img.shields.io/badge/MCP-31_Defense_Tools-0ea5e9?style=for-the-badge)](docs/cli-and-mcp.md) [![Formats](https://img.shields.io/badge/Formats-19%2B_Artifacts-0284c7?style=for-the-badge)](#format-coverage) diff --git a/STATE.md b/STATE.md index fddd722..14eaebf 100644 --- a/STATE.md +++ b/STATE.md @@ -1,6 +1,6 @@ # BLA state of the project -**Version:** 3.2.5 (`package.json` ↔ `src/identity.ts`, verified by build; see `CHANGELOG.md`) +**Version:** 3.2.6 (`package.json` ↔ `src/identity.ts`, verified by build; see `CHANGELOG.md`) **Status:** CI matrix runs typecheck, build, tests and the benchmark on ubuntu / macos / windows with Node 24. Windows checkouts keep `challenge/**` LF via `.gitattributes` so the CrackMe seals verify. **Publishing:** tagged releases (`git tag bla-agents-`) plus GitHub Releases. The release workflow can also publish the packed tarball to npm with provenance when `NPM_TOKEN`/OIDC is configured; a release cut may intentionally skip the npm step (see `CHANGELOG.md`). diff --git a/challenge/v6/SEAL.json b/challenge/v6/SEAL.json index 8591b7f..a5d5abf 100644 --- a/challenge/v6/SEAL.json +++ b/challenge/v6/SEAL.json @@ -1,7 +1,7 @@ { "schema": "bla-crackme-v6-seal/v2", "mode": "release", - "blaVersion": "3.2.5", + "blaVersion": "3.2.6", "recipe": "npm run crackme:build (hardenProject == `bla harden --intensity bankrupt --seed `)", "seed": "bla-crackme-v6-harden-native-2026-11", "intensity": "bankrupt", diff --git a/challenge/v7/SEAL.json b/challenge/v7/SEAL.json index b0ab372..efbd824 100644 --- a/challenge/v7/SEAL.json +++ b/challenge/v7/SEAL.json @@ -2,7 +2,7 @@ "schema": "bla-crackme-v7-seal/v1", "mode": "release", "product": "Meridian", - "blaVersion": "3.2.5", + "blaVersion": "3.2.6", "hardenSeedLabel": "bound-at-build-into-opaque-predicates", "intensity": "bankrupt", "llmDirectives": false, diff --git a/challenge/v8/SEAL.json b/challenge/v8/SEAL.json index 19101ac..c12c38a 100644 --- a/challenge/v8/SEAL.json +++ b/challenge/v8/SEAL.json @@ -2,7 +2,7 @@ "schema": "bla-crackme-v8-seal/v2", "mode": "release", "product": "Aegis", - "blaVersion": "3.2.5", + "blaVersion": "3.2.6", "recipe": "npm run crackme:v8:build (hardenProject == `bla harden --intensity bankrupt --seed --self-seal`); harden/schedule seed is private (derived from release secrets) and NOT published", "label": "bla-crackme-v8-aegis-2026-11", "intensity": "bankrupt", diff --git a/challenge/v9/SEAL.json b/challenge/v9/SEAL.json index edc6f24..adff634 100644 --- a/challenge/v9/SEAL.json +++ b/challenge/v9/SEAL.json @@ -2,7 +2,7 @@ "schema": "bla-crackme-v9-seal/v1", "mode": "release", "product": "Sovereign", - "blaVersion": "3.2.5", + "blaVersion": "3.2.6", "recipe": "npm run crackme:v9:build (4-Stage BLA 3.0 Playbook: bla_lint -> bla_weave_plan -> bla_verify_weave -> bla_harden --intensity bankrupt --weave-decoys --self-seal --sidecar-entangle); harden/schedule seed is private", "label": "bla-crackme-v9-sovereign-2026-11", "intensity": "bankrupt", diff --git a/docs/design/sidecar-entry-entanglement.md b/docs/design/sidecar-entry-entanglement.md index 4103d39..606c99c 100644 --- a/docs/design/sidecar-entry-entanglement.md +++ b/docs/design/sidecar-entry-entanglement.md @@ -1,6 +1,6 @@ # DESIGN: Sidecar / entry entanglement (issue #40 / STATE) -Status: **JS + Python + WASM + Bun companion + ASAR archive-note + ASAR main JS preamble strip⇒fail; .pyc marshal-note is host-verify only** — binding digest + JS/Python preambles + WASM `bla.sidecar.v1` custom-section note + Bun sibling `.bla-sidecar.v1.json` companion note (byte-exact binary) + ASAR in-archive `.bla/sidecar.v1.json` note + optional Electron main JS preamble (`generateSidecarAsarMainJsPreamble`) + CPython `.pyc` TYPE_ASCII marshal trailer (`generateSidecarPycMarshalNote` / `verifyPycSidecarMarshalNote`, importlib-ignored, no process-start fail) + protect/harden opt-in landed; defenseScore gap when `sidecarEntangle` is off landed; `.pyc` host-only notes annotate `pyc-sidecar-host-verify-only` and must not record `pyc-sidecar-entangle`; optional JS-loader `verifyWasmSidecarCustomSection` and `.pyc` import-time hook still TBD. +Status: **JS + Python + WASM + Bun companion + ASAR archive-note + ASAR main JS preamble strip⇒fail; .pyc marshal-note + GGUF STRING metadata note are host-verify only** — binding digest + JS/Python preambles + WASM `bla.sidecar.v1` custom-section note + Bun sibling `.bla-sidecar.v1.json` companion note (byte-exact binary) + ASAR in-archive `.bla/sidecar.v1.json` note + optional Electron main JS preamble (`generateSidecarAsarMainJsPreamble`) + CPython `.pyc` TYPE_ASCII marshal trailer (`generateSidecarPycMarshalNote` / `verifyPycSidecarMarshalNote`, importlib-ignored, no process-start fail) + GGUF `bla.sidecar.v1` STRING metadata KV (`generateSidecarGgufMetadataNote` / `verifyGgufSidecarMetadataNote`, alignment-safe rewrite keeps tensor bytes bit-exact; loaders ignore unknown STRING keys) + protect/harden opt-in landed; defenseScore gap when `sidecarEntangle` is off landed; `.pyc` / GGUF host-only notes annotate `pyc-sidecar-host-verify-only` / `gguf-sidecar-host-verify-only` and must not record the process-start vectors; optional JS-loader `verifyWasmSidecarCustomSection`, `.pyc` import-time hook, and GGUF load-time hook still TBD. ## Goal @@ -44,4 +44,5 @@ checks beat orphan honeypot files. - ~~Honesty: when entanglement is off, defenseScore gap annotation (like self-seal).~~ — 2026-11-08 nightly: `sidecar-entangle-not-enabled` soft-cap (93) via `sidecarEntangleEnabled` / MCP `sidecar_entangle_enabled`. - ~~Extend to ASAR archive-note companion~~ — 2026-11-09 nightly: `generateSidecarAsarArchiveNote` / `bindAsarSidecarEntanglement` / `entangleAsarEntryWithSidecars` / `verifyAsarSidecarArchiveNote`; protect + harden ASAR post-pass plant `.bla/sidecar.v1.json` binding `enclave/hardware-tee-bridge.js`. - ~~ASAR main JS preamble (Electron fs-patched re-measure)~~ — 2026-11-10 nightly: `generateSidecarAsarMainJsPreamble` / `selectAsarMainJsMember` / `entangleAsarMainJsWithSidecars`; protect + harden prepend strip⇒fail preamble to package.json#main (or heuristic entry). -- ~~CPython `.pyc` marshal trailer note (host verify)~~ — 2026-11-11 nightly: `generateSidecarPycMarshalNote` / `bindPycSidecarEntanglement` / `entanglePycEntryWithSidecars` / `verifyPycSidecarMarshalNote`; TYPE_ASCII trailer after root (importlib-safe). **2026-11-12 honesty:** do not record `pyc-sidecar-entangle` until an import-time hook lands; defenseScore gap `pyc-sidecar-host-verify-only`. Default OFF. Remaining: `.pyc` import-time hook; optional JS-loader `verifyWasmSidecarCustomSection`; next candidates: DEX string pool / SQLite comments / Safetensors·GGUF header metadata. +- ~~CPython `.pyc` marshal trailer note (host verify)~~ — 2026-11-11 nightly: `generateSidecarPycMarshalNote` / `bindPycSidecarEntanglement` / `entanglePycEntryWithSidecars` / `verifyPycSidecarMarshalNote`; TYPE_ASCII trailer after root (importlib-safe). **Honesty:** do not record `pyc-sidecar-entangle` until an import-time hook lands; defenseScore gap `pyc-sidecar-host-verify-only`. Default OFF. +- ~~GGUF STRING metadata note (host verify)~~ — 2026-11-12 nightly: `generateSidecarGgufMetadataNote` / `bindGgufSidecarEntanglement` / `entangleGgufEntryWithSidecars` / `verifyGgufSidecarMetadataNote`; `bla.sidecar.v1` STRING KV via alignment-safe `rebuildGguf` (tensor bytes bit-exact). **Honesty:** do not record `gguf-sidecar-entangle` until a load-time hook lands; defenseScore gap `gguf-sidecar-host-verify-only`. Also fixed `protectGgufBuffer` to use the same rewriter so tensor-bearing GGUFs no longer silently shift the data section. Default OFF. Remaining: `.pyc` import-time hook; GGUF load-time hook; optional JS-loader `verifyWasmSidecarCustomSection`; next candidates: DEX string pool / SQLite comments / Safetensors header metadata. diff --git a/docs/readme/README_zh.md b/docs/readme/README_zh.md index c910777..7422731 100644 --- a/docs/readme/README_zh.md +++ b/docs/readme/README_zh.md @@ -12,7 +12,7 @@ **开源的 Counter-REA 蓝队防御 MCP 服务与 CLI 工具 —— 专为保护软件发布产物免受 AI 逆向分析代理(`morluto/rea`、自动化反编译与大模型逆向流水线)还原而设计,同时保持 100% 运行时等价。** -[![npm version](https://img.shields.io/badge/npm-bla--agents_v3.2.5-2563eb?style=for-the-badge&logo=npm&logoColor=white)](https://www.npmjs.com/package/bla-agents) +[![npm version](https://img.shields.io/badge/npm-bla--agents_v3.2.6-2563eb?style=for-the-badge&logo=npm&logoColor=white)](https://www.npmjs.com/package/bla-agents) [![GitHub Trending](https://img.shields.io/badge/GitHub_Trending-%2315_Daily-7c3aed?style=for-the-badge&logo=github&logoColor=white)](https://trendshift.io/repositories/323569) [![MCP Server](https://img.shields.io/badge/MCP-25_项防御工具-0ea5e9?style=for-the-badge)](../cli-and-mcp.md) [![Formats](https://img.shields.io/badge/支持格式-19%2B_种构建产物-0284c7?style=for-the-badge)](#支持的产物格式与覆盖矩阵) diff --git a/package-lock.json b/package-lock.json index cb21064..1b0618f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "bla-agents", - "version": "3.2.5", + "version": "3.2.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "bla-agents", - "version": "3.2.5", + "version": "3.2.6", "license": "MIT", "dependencies": { "@babel/generator": "^8.1.0", diff --git a/package.json b/package.json index 1463aed..ff66d72 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bla-agents", - "version": "3.2.5", + "version": "3.2.6", "description": "BLA (Blind LLM Agents) \u2014 Adversarial defense shield that protects binaries, Electron apps, JS bundles, Python, WebAssembly, SQLite, AI Models, and Android DEX/IL2CPP against AI reverse-engineering agents like REA.", "keywords": [ "reverse-engineering-defense", diff --git a/src/core/types.ts b/src/core/types.ts index a76b4e1..001efcf 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -57,6 +57,7 @@ export type ReaAttackVectorId = | "asar-header-admission" | "asar-sidecar-entangle" | "pyc-sidecar-entangle" + | "gguf-sidecar-entangle" | "erlang-beam-inspection" | "evm-bytecode-inspection" | "pcap-capture-inspection" diff --git a/src/engine/defense-score-honesty.ts b/src/engine/defense-score-honesty.ts index eaf7f87..a6660f5 100644 --- a/src/engine/defense-score-honesty.ts +++ b/src/engine/defense-score-honesty.ts @@ -19,6 +19,7 @@ export type DefenseScoreGapId = | "self-seal-not-enabled" | "sidecar-entangle-not-enabled" | "pyc-sidecar-host-verify-only" + | "gguf-sidecar-host-verify-only" | "numeric-template-hygiene-partial" | "native-postlink-cfg-untouched" | "native-trailer-unmapped"; @@ -74,6 +75,7 @@ const SIDECAR_ENTANGLE_FORMATS: ReadonlySet = new Set([ "wasm", "python-source", "python-pyc", + "model-gguf", "native-elf", "native-macho", "native-pe", @@ -114,7 +116,7 @@ export function collectDefenseScoreGaps( gaps.push({ id: "sidecar-entangle-not-enabled", summary: - "Opt-in sidecarEntangle strip⇒fail binding was not enabled (JS/Python preamble, WASM bla.sidecar.v1, Bun companion, or a .pyc import-time hook); a host-only .pyc marshal trailer does not count. Default remains OFF (issue #40 / DESIGN).", + "Opt-in sidecarEntangle strip⇒fail binding was not enabled (JS/Python preamble, WASM bla.sidecar.v1, Bun companion, .pyc import-time hook, or GGUF load-time hook); a host-only .pyc marshal trailer or GGUF STRING KV does not count. Default remains OFF (issue #40 / DESIGN).", scoreCap: 93, }); } @@ -130,6 +132,16 @@ export function collectDefenseScoreGaps( }); } + // GGUF STRING KV is host-verify only (loaders ignore unknown metadata keys). + if (input.detectedFormat === "model-gguf" && input.sidecarEntangleEnabled === true) { + gaps.push({ + id: "gguf-sidecar-host-verify-only", + summary: + "sidecarEntangle planted a GGUF bla.sidecar.v1 STRING metadata note verified only by host verifyGgufSidecarMetadataNote; GGUF loaders still load after strip or sibling delete. Load-time hook TBD — do not treat this as process-start strip⇒fail (issue #40 / DESIGN).", + scoreCap: 93, + }); + } + if (input.numericTemplateLeakKnown === true) { gaps.push({ id: "numeric-template-hygiene-partial", diff --git a/src/engine/harden.ts b/src/engine/harden.ts index 527b0c2..1ef5e9c 100644 --- a/src/engine/harden.ts +++ b/src/engine/harden.ts @@ -44,6 +44,7 @@ import { entangleBunEntryWithSidecars, entangleJsEntryWithSidecars, entanglePycEntryWithSidecars, + entangleGgufEntryWithSidecars, entanglePythonEntryWithSidecars, entangleWasmEntryWithSidecars, planSidecarEntanglement, @@ -1185,6 +1186,48 @@ export function hardenProject( } } + // GGUF STRING-metadata entanglement only when no JS/Python/WASM/.pyc + // entry was bound — bla.sidecar.v1 KV measured against FINAL siblings; + // GGUF loaders ignore unknown STRING keys (tensor bytes stay bit-exact). + const ggufLike = summaries.filter( + (s) => + s.success && + (s.format === "model-gguf" || + /\.gguf$/i.test(s.relativePath.replaceAll("\\", "/"))), + ); + const ggufEntry = + !jsEntry && !pyEntry && !wasmEntry && !pycEntry + ? ggufLike[0] + : undefined; + if (ggufEntry) { + const entryAbs = join(outputRoot, ggufEntry.relativePath); + if (existsSync(entryAbs)) { + const entryDir = dirname(entryAbs); + const sidecars: { pathLabel: string; bytes: Buffer }[] = []; + for (const s of summaries) { + if (!s.success || s.relativePath === ggufEntry.relativePath) continue; + const abs = join(outputRoot, s.relativePath); + const rel = relative(entryDir, abs).split(sep).join("/"); + if (rel.length === 0 || rel.startsWith("..") || !existsSync(abs)) continue; + sidecars.push({ pathLabel: rel, bytes: readFileSync(abs) }); + } + if (sidecars.length > 0) { + const entryBytes = readFileSync(entryAbs); + const bound = entangleGgufEntryWithSidecars({ + entryPath: basename(entryAbs), + entryBytes, + sidecars, + seedTag: String(options.seed ?? runId), + }); + if (bound) { + writeFileAtomicSync(entryAbs, bound.bytes, undefined, { + allowLinkWriteThrough: false, + }); + } + } + } + } + // Bun / EOF-preserved native companion note only when no JS/Python/WASM // entry was bound — binary stays byte-exact; `.bla-sidecar.v1.json` // binds the sibling `.bunfs-snapshot` planted by protect. diff --git a/src/engine/protector.ts b/src/engine/protector.ts index b0a0265..1c6009b 100644 --- a/src/engine/protector.ts +++ b/src/engine/protector.ts @@ -99,6 +99,7 @@ import { entangleBunEntryWithSidecars, entangleJsEntryWithSidecars, entanglePycEntryWithSidecars, + entangleGgufEntryWithSidecars, entanglePythonEntryWithSidecars, entangleWasmEntryWithSidecars, } from "../shields/runtime/sidecar-entangle.js"; @@ -306,6 +307,41 @@ function applySidecarEntangleToPyc( return { buffer: bound.bytes, applied: true }; } +function applySidecarEntangleToGguf( + ggufBytes: Buffer, + options: ProtectionOptions, + absOutput: string, + seed: string, +): { buffer: Buffer; applied: boolean } { + if (options.sidecarEntangle !== true) return { buffer: ggufBytes, applied: false }; + const paths = (options.sidecarEntanglePaths ?? []).filter( + (p) => typeof p === "string" && p.length > 0, + ); + if (paths.length === 0) return { buffer: ggufBytes, applied: false }; + const base = dirname(absOutput); + const sidecars: { pathLabel: string; bytes: Uint8Array }[] = []; + for (const rel of paths) { + const abs = isAbsolute(rel) ? rel : join(base, rel); + try { + if (!existsSync(abs)) continue; + sidecars.push({ pathLabel: rel, bytes: readFileSync(abs) }); + } catch { + /* skip unreadable */ + } + } + if (sidecars.length === 0) return { buffer: ggufBytes, applied: false }; + const entryLabel = basename(absOutput); + const bound = entangleGgufEntryWithSidecars({ + entryPath: entryLabel, + entryBytes: ggufBytes, + sidecars, + seedTag: String(seed), + softFold: options.sidecarEntangleSoftFold === true, + }); + if (!bound) return { buffer: ggufBytes, applied: false }; + return { buffer: bound.bytes, applied: true }; +} + function applyPanelDeriveRewrite( source: string, seed: string, @@ -1067,7 +1103,19 @@ function protectTargetInner(options: ProtectionOptions): ProtectionResult { } else if (format === "model-gguf") { const rawModel = readFileSync(absInput); const res = protectGgufBuffer(rawModel, { intensity, seed }); - writeProtectedOutput(absOutput, res.buffer); + let finalGguf = res.buffer; + { + const entangled = applySidecarEntangleToGguf(finalGguf, options, absOutput, seed); + finalGguf = entangled.buffer; + if (entangled.applied) { + sidecarEntangleApplied = true; + // Host-verify STRING KV only: GGUF loaders ignore unknown metadata + // keys, so strip / sibling delete still load. Do not record + // gguf-sidecar-entangle until a load-time hook lands (same honesty + // bar as .pyc marshal trailer / ASAR archive-note-only). + } + } + writeProtectedOutput(absOutput, finalGguf); decoyFunctionsInjected = 16; chaffPayloadsEmbedded = res.chaffPayloadsEmbedded; binarySectionsPatched = res.triggeredMetadataNestingBreaker ? 2 : 1; diff --git a/src/identity.ts b/src/identity.ts index d7d8672..e6c6feb 100644 --- a/src/identity.ts +++ b/src/identity.ts @@ -5,7 +5,7 @@ export const BLA_IDENTITY = Object.freeze({ packageAlias: "bla-shield", cliBinary: "bla", cliAlias: "bla-agents", - version: "3.2.5", + version: "3.2.6", slogan: "Poison -> Exhaust -> Hallucinate. Make Reverse Engineering Bankrupt Again.", counterpart: { name: "REA", diff --git a/src/index.ts b/src/index.ts index 6721a9f..c2af5dd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -173,6 +173,21 @@ export { protectGgufBuffer, type ModelInspectionSummary, } from "./shields/formats/model.js"; +export { + parseGgufLayout, + rebuildGguf, + encodeGgufString, + encodeGgufStringKv, + readGgufStringKv, + GGUF_DEFAULT_ALIGNMENT, + GGUF_TYPE_STRING, + GGUF_TYPE_ARRAY, + GGUF_TYPE_UINT32, + type GgufLayout, + type GgufKvEntry, + type GgufRebuildOptions, +} from "./shields/formats/gguf-layout.js"; + export { inspectAndroidDex, protectAndroidDexBuffer, @@ -497,6 +512,7 @@ export { generateSidecarAsarArchiveNote, generateSidecarAsarMainJsPreamble, generateSidecarPycMarshalNote, + generateSidecarGgufMetadataNote, asarMemberPathFromMain, selectAsarMainJsMember, prependAsarMainJsPreamble, @@ -506,6 +522,7 @@ export { bindBunSidecarEntanglement, bindAsarSidecarEntanglement, bindPycSidecarEntanglement, + bindGgufSidecarEntanglement, entangleJsEntryWithSidecars, entanglePythonEntryWithSidecars, entangleWasmEntryWithSidecars, @@ -513,12 +530,16 @@ export { entangleAsarEntryWithSidecars, entangleAsarMainJsWithSidecars, entanglePycEntryWithSidecars, + entangleGgufEntryWithSidecars, verifyWasmSidecarCustomSection, verifyBunSidecarCompanionNote, verifyAsarSidecarArchiveNote, verifyPycSidecarMarshalNote, extractPycSidecarMarshalNote, stripPycSidecarMarshalNote, + verifyGgufSidecarMetadataNote, + extractGgufSidecarMetadataNote, + stripGgufSidecarMetadataNote, stripWasmSidecarCustomSection, SIDECAR_ENTANGLE_SCHEMA, WASM_SIDECAR_SECTION_NAME, @@ -529,6 +550,8 @@ export { ASAR_SIDECAR_MAIN_JS_MARKER, PYC_SIDECAR_MARSHAL_MARKER, PYC_SIDECAR_MARSHAL_KIND, + GGUF_SIDECAR_METADATA_KEY, + GGUF_SIDECAR_METADATA_KIND, type SidecarEntanglePlan, type SidecarEntanglePlanOptions, type SidecarBindingPart, @@ -566,6 +589,12 @@ export { type PycSidecarMarshalNote, type PycSidecarVerifyOptions, type PycSidecarVerifyResult, + type SidecarGgufInjectOptions, + type SidecarGgufInjectResult, + type SidecarGgufBindResult, + type GgufSidecarMetadataNote, + type GgufSidecarVerifyOptions, + type GgufSidecarVerifyResult, } from "./shields/runtime/sidecar-entangle.js"; export { diff --git a/src/shields/formats/gguf-layout.ts b/src/shields/formats/gguf-layout.ts new file mode 100644 index 0000000..16199bd --- /dev/null +++ b/src/shields/formats/gguf-layout.ts @@ -0,0 +1,258 @@ +/** + * Alignment-safe GGUF v2/v3 layout parser + rewriter. + * + * GGUF tensor-info `offset` fields are RELATIVE to the tensor-data section, + * whose absolute start is `align(endOfTensorInfos, general.alignment)` + * (default 32). Inserting or removing metadata KV bytes shifts the end of the + * tensor infos, so the data section MUST be re-padded to the new aligned + * start — otherwise every tensor offset resolves to the wrong bytes and the + * model silently loads corrupted weights while still "parsing". + * + * `rebuildGguf` keeps tensor infos and the tensor-data region byte-exact and + * recomputes only the padding, so tensor bytes are preserved bit-for-bit. + * + * DEFENSIVE ONLY. Format-preserving; no tensor bytes are ever modified. + */ + +/** GGUF metadata value type ids (gguf spec). */ +export const GGUF_TYPE_UINT32 = 4; +export const GGUF_TYPE_STRING = 8; +export const GGUF_TYPE_ARRAY = 9; + +/** Spec default when `general.alignment` is absent. */ +export const GGUF_DEFAULT_ALIGNMENT = 32; + +/** Bounds that keep parsing cheap on hostile / malformed input. */ +const MAX_KV_COUNT = 100_000n; +const MAX_TENSOR_COUNT = 1_000_000n; +const MAX_TENSOR_DIMS = 8; +const MAX_VALUE_DEPTH = 64; + +export interface GgufKvEntry { + readonly key: string; + /** Absolute offset of the KV record (start of the key length prefix). */ + readonly start: number; + /** Absolute offset one past the end of the KV record. */ + readonly end: number; + readonly valueType: number; +} + +export interface GgufLayout { + readonly version: number; + readonly tensorCount: bigint; + readonly kvCount: bigint; + readonly kvs: readonly GgufKvEntry[]; + readonly kvEnd: number; + /** Absolute offset one past the last tensor-info record. */ + readonly tensorInfoEnd: number; + readonly alignment: number; + /** Absolute aligned start of the tensor-data section. */ + readonly dataStart: number; + /** True when bytes exist beyond the tensor infos (padding and/or data). */ + readonly hasDataRegion: boolean; +} + +function skipValue( + bytes: Buffer, + offset: number, + valType: number, + depth: number, +): number | null { + if (depth > MAX_VALUE_DEPTH || offset > bytes.length) return null; + switch (valType) { + case 0: + case 1: + case 7: + return offset + 1 <= bytes.length ? offset + 1 : null; + case 2: + case 3: + return offset + 2 <= bytes.length ? offset + 2 : null; + case 4: + case 5: + case 6: + return offset + 4 <= bytes.length ? offset + 4 : null; + case 10: + case 11: + case 12: + return offset + 8 <= bytes.length ? offset + 8 : null; + case GGUF_TYPE_STRING: { + if (offset + 8 > bytes.length) return null; + const len = bytes.readBigUInt64LE(offset); + const end = BigInt(offset) + 8n + len; + return end <= BigInt(bytes.length) ? Number(end) : null; + } + case GGUF_TYPE_ARRAY: { + if (offset + 12 > bytes.length) return null; + const elemType = bytes.readUInt32LE(offset); + const count = bytes.readBigUInt64LE(offset + 4); + // Each element consumes >= 1 byte, so a count larger than the remaining + // bytes is necessarily malformed (also bounds the loop). + if (count > BigInt(bytes.length - (offset + 12))) return null; + let cur = offset + 12; + for (let i = 0n; i < count; i++) { + const next = skipValue(bytes, cur, elemType, depth + 1); + if (next === null) return null; + cur = next; + } + return cur; + } + default: + return null; + } +} + +function alignUp(value: number, alignment: number): number { + const rem = value % alignment; + return rem === 0 ? value : value + (alignment - rem); +} + +/** + * Parse the GGUF v2/v3 container layout. Returns null for non-GGUF input, + * unsupported versions (v1 uses u32 counts), or any malformed / truncated + * structure. Never throws on hostile input. + */ +export function parseGgufLayout(input: Uint8Array): GgufLayout | null { + const bytes = Buffer.isBuffer(input) ? input : Buffer.from(input); + if ( + bytes.length < 24 || + bytes[0] !== 0x47 || + bytes[1] !== 0x47 || + bytes[2] !== 0x55 || + bytes[3] !== 0x46 + ) { + return null; + } + const version = bytes.readUInt32LE(4); + if (version !== 2 && version !== 3) return null; + const tensorCount = bytes.readBigUInt64LE(8); + const kvCount = bytes.readBigUInt64LE(16); + if (kvCount > MAX_KV_COUNT || tensorCount > MAX_TENSOR_COUNT) return null; + + let offset = 24; + const kvs: GgufKvEntry[] = []; + let alignment = GGUF_DEFAULT_ALIGNMENT; + for (let i = 0n; i < kvCount; i++) { + const start = offset; + if (offset + 8 > bytes.length) return null; + const keyLen = bytes.readBigUInt64LE(offset); + const keyEnd = BigInt(offset) + 8n + keyLen; + if (keyEnd + 4n > BigInt(bytes.length)) return null; + const keyEndN = Number(keyEnd); + const key = bytes.subarray(offset + 8, keyEndN).toString("utf8"); + const valueType = bytes.readUInt32LE(keyEndN); + const valueStart = keyEndN + 4; + const next = skipValue(bytes, valueStart, valueType, 0); + if (next === null) return null; + if (key === "general.alignment" && valueType === GGUF_TYPE_UINT32) { + const a = bytes.readUInt32LE(valueStart); + // Spec: a non-zero multiple of 8. Ignore nonsensical values. + if (a > 0 && a % 8 === 0) alignment = a; + } + kvs.push({ key, start, end: next, valueType }); + offset = next; + } + const kvEnd = offset; + + for (let t = 0n; t < tensorCount; t++) { + if (offset + 8 > bytes.length) return null; + const nameLen = bytes.readBigUInt64LE(offset); + const nameEnd = BigInt(offset) + 8n + nameLen; + if (nameEnd + 4n > BigInt(bytes.length)) return null; + offset = Number(nameEnd); + const nDims = bytes.readUInt32LE(offset); + if (nDims > MAX_TENSOR_DIMS) return null; + offset += 4; + const tail = nDims * 8 + 4 + 8; // dims (u64 each) + ggml_type (u32) + offset (u64) + if (offset + tail > bytes.length) return null; + offset += tail; + } + const tensorInfoEnd = offset; + const hasDataRegion = bytes.length > tensorInfoEnd; + const dataStart = alignUp(tensorInfoEnd, alignment); + + return { + version, + tensorCount, + kvCount, + kvs, + kvEnd, + tensorInfoEnd, + alignment, + dataStart, + hasDataRegion, + }; +} + +/** Encode a GGUF string (u64 LE length + UTF-8 bytes). */ +export function encodeGgufString(str: string): Buffer { + const utf8 = Buffer.from(str, "utf8"); + const len = Buffer.alloc(8); + len.writeBigUInt64LE(BigInt(utf8.length), 0); + return Buffer.concat([len, utf8]); +} + +/** Encode a full STRING-typed GGUF metadata KV record. */ +export function encodeGgufStringKv(key: string, value: string): Buffer { + const type = Buffer.alloc(4); + type.writeUInt32LE(GGUF_TYPE_STRING, 0); + return Buffer.concat([encodeGgufString(key), type, encodeGgufString(value)]); +} + +export interface GgufRebuildOptions { + /** Existing KV keys to drop (e.g. a stale sidecar note for idempotent re-bind). */ + readonly dropKeys?: ReadonlySet; + /** Fully encoded KV records to append after the retained KVs. */ + readonly appendKvs?: readonly Buffer[]; +} + +/** + * Rebuild a GGUF container with modified metadata while keeping tensor infos + * and the tensor-data region byte-exact. The data section is re-padded to the + * new `align(tensorInfoEnd, alignment)` so every relative tensor offset still + * resolves to the original bytes. + */ +export function rebuildGguf( + input: Uint8Array, + layout: GgufLayout, + options: GgufRebuildOptions = {}, +): Buffer { + const bytes = Buffer.isBuffer(input) ? input : Buffer.from(input); + const drop = options.dropKeys ?? new Set(); + const append = options.appendKvs ?? []; + const kept = layout.kvs.filter((kv) => !drop.has(kv.key)); + + const header = Buffer.alloc(24); + header.write("GGUF", 0, "ascii"); + header.writeUInt32LE(layout.version, 4); + header.writeBigUInt64LE(layout.tensorCount, 8); + header.writeBigUInt64LE(BigInt(kept.length + append.length), 16); + + const tensorInfos = bytes.subarray(layout.kvEnd, layout.tensorInfoEnd); + const metaAndInfos = Buffer.concat([ + header, + ...kept.map((kv) => bytes.subarray(kv.start, kv.end)), + ...append, + tensorInfos, + ]); + if (!layout.hasDataRegion) return metaAndInfos; + + const newDataStart = alignUp(metaAndInfos.length, layout.alignment); + const pad = Buffer.alloc(newDataStart - metaAndInfos.length, 0); + const data = bytes.subarray(Math.min(layout.dataStart, bytes.length)); + return Buffer.concat([metaAndInfos, pad, data]); +} + +/** Read a STRING-typed KV value by key, or null when absent / not a string. */ +export function readGgufStringKv( + input: Uint8Array, + layout: GgufLayout, + key: string, +): string | null { + const bytes = Buffer.isBuffer(input) ? input : Buffer.from(input); + const kv = layout.kvs.find((k) => k.key === key); + if (!kv || kv.valueType !== GGUF_TYPE_STRING) return null; + const keyLen = Number(bytes.readBigUInt64LE(kv.start)); + const valueStart = kv.start + 8 + keyLen + 4; + const len = Number(bytes.readBigUInt64LE(valueStart)); + return bytes.subarray(valueStart + 8, valueStart + 8 + len).toString("utf8"); +} diff --git a/src/shields/formats/model.ts b/src/shields/formats/model.ts index 94c8c1f..6de0199 100644 --- a/src/shields/formats/model.ts +++ b/src/shields/formats/model.ts @@ -1,3 +1,4 @@ +import { parseGgufLayout, rebuildGguf } from "./gguf-layout.js"; import { REA_INTERNAL_LIMITS } from "../../identity.js"; import { SeededPrng } from "../../core/prng.js"; import type { ProtectionIntensity } from "../../core/types.js"; @@ -149,7 +150,8 @@ export function inspectModelBuffer(bytes: Buffer): ModelInspectionSummary { const textLatin1 = bytes.toString("latin1"); const blaProtected = textLatin1.includes("enclave.delegation") || - textLatin1.includes("BLA-MODEL-SHIELD"); + textLatin1.includes("BLA-MODEL-SHIELD") || + textLatin1.includes("bla.sidecar.v1"); return { validModel: true, format: "gguf", @@ -491,6 +493,22 @@ export function protectGgufBuffer( kvEntries.push(Buffer.concat(frames)); } + // Real models carry tensors whose `offset` fields are RELATIVE to an + // aligned data section. Splicing KV bytes in front of the tensor infos + // without re-padding shifts that section and silently corrupts every + // weight. Route tensor-bearing inputs through the alignment-safe rewriter, + // which keeps tensor infos + data byte-exact and recomputes only padding. + const layout = parsed && parsed.tensorCount > 0n ? parseGgufLayout(bytes) : null; + if (layout) { + return { + buffer: rebuildGguf(bytes, layout, { appendKvs: kvEntries }), + chaffPayloadsEmbedded: chaffPayloads.length, + triggeredMetadataNestingBreaker: true, + canaryTokensEmbedded: deception.canary.syntheticCanaryTokens.length, + diagnosticsEmbedded: deception.diagnostics.diagnosticsInjected.length, + }; + } + const header = Buffer.alloc(24); header.write("GGUF", 0, "ascii"); header.writeUInt32LE(parsed ? parsed.version : 3, 4); diff --git a/src/shields/runtime/sidecar-entangle.ts b/src/shields/runtime/sidecar-entangle.ts index 5d25b50..c788590 100644 --- a/src/shields/runtime/sidecar-entangle.ts +++ b/src/shields/runtime/sidecar-entangle.ts @@ -14,7 +14,9 @@ * re-measures members via Electron's fs-patched `readFileSync`; CPython `.pyc` * appends a TYPE_ASCII marshal trailer note (`__blaPycSidecarDigest=…`) verified * by `verifyPycSidecarMarshalNote` (importlib ignores post-root trailers — see - * `docs/design/sidecar-entry-entanglement.md`). + * `docs/design/sidecar-entry-entanglement.md`); GGUF plants a STRING metadata + * KV note (`bla.sidecar.v1`) verified by `verifyGgufSidecarMetadataNote`, with + * alignment-safe rewrite so tensor bytes stay bit-exact. * * DEFENSIVE ONLY. Hash-only — no guessing. */ @@ -39,6 +41,12 @@ import { stripWasmCustomSectionByName, } from "../formats/wasm.js"; import { unpackAsarBuffer } from "../formats/asar.js"; +import { + encodeGgufStringKv, + parseGgufLayout, + readGgufStringKv, + rebuildGguf, +} from "../formats/gguf-layout.js"; export const SIDECAR_ENTANGLE_SCHEMA = "bla-sidecar-entangle/v1" as const; @@ -86,7 +94,7 @@ export interface SidecarEntanglePlan { readonly softFold: boolean; /** Offline binding id (hash of paths + seedTag) — not the runtime content digest. */ readonly planId: string; - readonly entryHookKind: "js-preamble" | "python-preamble" | "wasm-custom-section" | "bun-companion-note" | "asar-archive-note" | "pyc-marshal-note" | "unspecified"; + readonly entryHookKind: "js-preamble" | "python-preamble" | "wasm-custom-section" | "bun-companion-note" | "asar-archive-note" | "pyc-marshal-note" | "gguf-metadata-note" | "unspecified"; /** `ready` when coverage matrix satisfies refuse predicates; else `incomplete`. */ readonly status: "ready" | "incomplete" | "bound"; readonly note: string; @@ -101,6 +109,7 @@ function guessHookKind(entryPath: string): SidecarEntanglePlan["entryHookKind"] if (/\.pyc$/.test(lower)) return "pyc-marshal-note"; if (/\.wasm$/.test(lower)) return "wasm-custom-section"; if (/\.asar$/.test(lower)) return "asar-archive-note"; + if (/\.gguf$/.test(lower)) return "gguf-metadata-note"; return "unspecified"; } @@ -2264,6 +2273,347 @@ export function verifyPycSidecarMarshalNote( } + +// --------------------------------------------------------------------------- +// GGUF STRING metadata note (alignment-safe) +// --------------------------------------------------------------------------- + +/** Metadata KV key planted inside the GGUF header. */ +export const GGUF_SIDECAR_METADATA_KEY = "bla.sidecar.v1" as const; +export const GGUF_SIDECAR_METADATA_KIND = "gguf-sidecar-metadata" as const; + +export interface GgufSidecarMetadataNote { + readonly schema: typeof SIDECAR_ENTANGLE_SCHEMA; + readonly kind: typeof GGUF_SIDECAR_METADATA_KIND; + readonly key: typeof GGUF_SIDECAR_METADATA_KEY; + readonly digest: string; + readonly seedTag: string; + readonly entryPath: string; + readonly sidecarPaths: readonly string[]; + readonly softFold: boolean; + readonly mode: "sidecars"; +} + +export interface SidecarGgufInjectOptions { + readonly entryPath: string; + readonly sidecarPaths: readonly string[]; + readonly seedTag?: string; + /** Offline expected digest (from computeSidecarBindingDigest). */ + readonly expectedHex: string; + readonly softFold?: boolean; +} + +export interface SidecarGgufInjectResult { + readonly schema: typeof SIDECAR_ENTANGLE_SCHEMA; + readonly kind: typeof GGUF_SIDECAR_METADATA_KIND; + readonly key: typeof GGUF_SIDECAR_METADATA_KEY; + readonly noteJson: string; + readonly note: GgufSidecarMetadataNote; + readonly expectedHex: string; + readonly expectedU32: number; + readonly softFold: boolean; + readonly sidecarCount: number; + /** Fully encoded STRING-typed GGUF KV record ready to append. */ + readonly kvRecord: Buffer; +} + +/** + * Build a GGUF STRING metadata KV (`bla.sidecar.v1`) that carries the sidecar + * binding note. Tensor bytes, tensor infos, and relative tensor offsets stay + * untouched when the entry is rewritten via `rebuildGguf` (alignment-safe). + * + * DEFENSIVE ONLY. Opt-in. No crack / RE advice. + */ +export function generateSidecarGgufMetadataNote( + options: SidecarGgufInjectOptions, +): SidecarGgufInjectResult { + const sidecars = (options.sidecarPaths ?? []).filter((p) => p.length > 0); + const seedTag = options.seedTag ?? "bla-sidecar-v1"; + const softFold = options.softFold === true; + const expectedHex = options.expectedHex.toLowerCase(); + if (!/^[0-9a-f]{64}$/.test(expectedHex)) { + throw new Error("expectedHex must be a 64-char lowercase hex SHA-256 digest"); + } + const expectedU32 = parseInt(expectedHex.slice(0, 8), 16) >>> 0; + const note: GgufSidecarMetadataNote = { + schema: SIDECAR_ENTANGLE_SCHEMA, + kind: GGUF_SIDECAR_METADATA_KIND, + key: GGUF_SIDECAR_METADATA_KEY, + digest: expectedHex, + seedTag, + entryPath: options.entryPath, + sidecarPaths: sidecars, + softFold, + mode: "sidecars", + }; + const noteJson = JSON.stringify(note); + return { + schema: SIDECAR_ENTANGLE_SCHEMA, + kind: GGUF_SIDECAR_METADATA_KIND, + key: GGUF_SIDECAR_METADATA_KEY, + noteJson, + note, + expectedHex, + expectedU32, + softFold, + sidecarCount: sidecars.length, + kvRecord: encodeGgufStringKv(GGUF_SIDECAR_METADATA_KEY, noteJson), + }; +} + +export interface SidecarGgufBindResult { + readonly plan: SidecarEntanglePlan; + readonly digest: SidecarBindingDigest; + readonly inject: SidecarGgufInjectResult; + /** GGUF bytes with the bla.sidecar.v1 STRING KV planted (tensors bit-exact). */ + readonly bytes: Buffer; +} + +/** + * Plan + bind for a GGUF entry: compute the offline digest from in-memory + * sidecar bytes and return a `bound` plan plus the rewritten GGUF with a + * `bla.sidecar.v1` STRING metadata note. Uses sidecars-mode digest so planting + * the note does not invalidate H. Tensor bytes stay bit-exact via + * alignment-safe rebuild. + */ +export function bindGgufSidecarEntanglement( + options: SidecarEntanglePlanOptions & { + readonly entryBytes: Uint8Array; + readonly sidecarBytes: readonly Uint8Array[]; + }, +): SidecarGgufBindResult { + const plan = planSidecarEntanglement(options); + if (options.sidecarBytes.length !== plan.sidecarPaths.length) { + throw new Error("sidecarBytes length must match sidecarPaths length"); + } + const entryBuf = Buffer.from(options.entryBytes); + const layout = parseGgufLayout(entryBuf); + if (!layout) { + throw new Error("entryBytes is not a valid GGUF v2/v3 container"); + } + const digest = computeSidecarBindingDigest( + { pathLabel: options.entryPath, bytes: options.entryBytes }, + plan.sidecarPaths.map((pathLabel, i) => ({ + pathLabel, + bytes: options.sidecarBytes[i]!, + })), + plan.seedTag, + "sidecars", + ); + const inject = generateSidecarGgufMetadataNote({ + entryPath: options.entryPath, + sidecarPaths: plan.sidecarPaths, + seedTag: plan.seedTag, + expectedHex: digest.hex, + softFold: plan.softFold, + }); + // Drop any prior note so re-bind is idempotent, then append the fresh one. + const bytes = rebuildGguf(entryBuf, layout, { + dropKeys: new Set([GGUF_SIDECAR_METADATA_KEY]), + appendKvs: [inject.kvRecord], + }); + const boundPlan: SidecarEntanglePlan = { + ...plan, + entryHookKind: "gguf-metadata-note", + status: "bound", + note: "Bound — GGUF bla.sidecar.v1 STRING metadata note planted; ship sidecars alongside and verify via verifyGgufSidecarMetadataNote. Host verify fails closed on strip/alter; GGUF loaders ignore unknown STRING KVs (no process-start fail until a load-time hook lands). Tensor bytes stay bit-exact.", + }; + return { + plan: boundPlan, + digest, + inject, + bytes, + }; +} + +/** + * Convenience: given final on-disk sidecar bytes + GGUF entry bytes, produce + * the strip⇒fail-bound entry. Returns null when there are no sidecars to bind. + */ +export function entangleGgufEntryWithSidecars(options: { + readonly entryPath: string; + readonly entryBytes: Uint8Array; + readonly sidecars: readonly SidecarBindingPart[]; + readonly seedTag?: string; + readonly softFold?: boolean; +}): SidecarGgufBindResult | null { + if (options.sidecars.length === 0) return null; + return bindGgufSidecarEntanglement({ + entryPath: options.entryPath, + sidecarPaths: options.sidecars.map((s) => s.pathLabel), + seedTag: options.seedTag, + softFold: options.softFold, + entryBytes: options.entryBytes, + sidecarBytes: options.sidecars.map((s) => s.bytes), + fileBytes: Object.fromEntries( + options.sidecars.map((s) => [s.pathLabel, Buffer.from(s.bytes)]), + ), + }); +} + +export interface GgufSidecarVerifyOptions { + /** Full GGUF bytes (header + KVs + tensor infos + data). */ + readonly entryBytes: Uint8Array; + /** + * Absolute directory used to resolve relative sidecar paths from the note. + * Required when note.sidecarPaths are relative and not supplied in-memory. + */ + readonly baseDir?: string; + /** Optional in-memory sidecar bytes keyed by pathLabel (overrides disk read). */ + readonly sidecarBytes?: Readonly>; +} + +export interface GgufSidecarVerifyResult { + readonly ok: boolean; + readonly reason?: string; + readonly note?: GgufSidecarMetadataNote; + readonly expectedHex?: string; + readonly measuredHex?: string; + readonly measuredU32?: number; + readonly softFold?: boolean; +} + +/** + * Extract the `bla.sidecar.v1` STRING metadata note from a GGUF buffer. + * Returns null when the key is absent or the value is not a well-formed note. + */ +export function extractGgufSidecarMetadataNote( + entryBytes: Uint8Array, +): GgufSidecarMetadataNote | null { + const layout = parseGgufLayout(entryBytes); + if (!layout) return null; + const jsonText = readGgufStringKv(entryBytes, layout, GGUF_SIDECAR_METADATA_KEY); + if (jsonText === null) return null; + try { + const parsed = JSON.parse(jsonText) as GgufSidecarMetadataNote; + if ( + parsed.schema !== SIDECAR_ENTANGLE_SCHEMA || + parsed.kind !== GGUF_SIDECAR_METADATA_KIND || + typeof parsed.digest !== "string" || + !/^[0-9a-f]{64}$/i.test(parsed.digest) + ) { + return null; + } + return { + schema: SIDECAR_ENTANGLE_SCHEMA, + kind: GGUF_SIDECAR_METADATA_KIND, + key: GGUF_SIDECAR_METADATA_KEY, + digest: parsed.digest.toLowerCase(), + seedTag: typeof parsed.seedTag === "string" ? parsed.seedTag : "bla-sidecar-v1", + entryPath: typeof parsed.entryPath === "string" ? parsed.entryPath : "", + sidecarPaths: Array.isArray(parsed.sidecarPaths) + ? parsed.sidecarPaths.filter((p): p is string => typeof p === "string" && p.length > 0) + : [], + softFold: parsed.softFold === true, + mode: "sidecars", + }; + } catch { + return null; + } +} + +/** + * Remove a previously planted GGUF sidecar metadata note (idempotent). + * Tensor bytes stay bit-exact via alignment-safe rebuild. + */ +export function stripGgufSidecarMetadataNote(entryBytes: Uint8Array): Buffer { + const layout = parseGgufLayout(entryBytes); + if (!layout) return Buffer.from(entryBytes); + if (!layout.kvs.some((kv) => kv.key === GGUF_SIDECAR_METADATA_KEY)) { + return Buffer.from(entryBytes); + } + return rebuildGguf(entryBytes, layout, { + dropKeys: new Set([GGUF_SIDECAR_METADATA_KEY]), + }); +} + +/** + * Host-side strip⇒fail check for GGUF metadata notes: extract the STRING KV + * note, re-hash sibling sidecar bytes with the same layout as + * `computeSidecarBindingDigest` (sidecars mode), and compare to the embedded + * digest. Missing note / missing sibling / altered sibling ⇒ ok:false. + * + * DEFENSIVE ONLY — for tests, loaders, and harden post-pass verification. + * GGUF loaders ignore unknown STRING KVs, so strip does not break load until + * a load-time hook lands (same honesty bar as .pyc marshal trailer). + */ +export function verifyGgufSidecarMetadataNote( + options: GgufSidecarVerifyOptions, +): GgufSidecarVerifyResult { + const note = extractGgufSidecarMetadataNote(options.entryBytes); + if (!note) { + return { ok: false, reason: "gguf sidecar metadata note missing" }; + } + + const sidecarParts: SidecarBindingPart[] = []; + for (const label of note.sidecarPaths) { + const override = options.sidecarBytes?.[label]; + if (override) { + sidecarParts.push({ pathLabel: label, bytes: override }); + continue; + } + if (!options.baseDir) { + return { + ok: false, + reason: `sidecar ${label} not supplied and no baseDir to read`, + note, + expectedHex: note.digest, + softFold: note.softFold, + }; + } + const abs = isAbsolute(label) ? label : join(options.baseDir, label); + try { + if (!existsSync(abs)) { + return { + ok: false, + reason: `sidecar ${label} missing on disk`, + note, + expectedHex: note.digest, + softFold: note.softFold, + }; + } + sidecarParts.push({ pathLabel: label, bytes: readFileSync(abs) }); + } catch { + return { + ok: false, + reason: `sidecar ${label} unreadable`, + note, + expectedHex: note.digest, + softFold: note.softFold, + }; + } + } + + const measured = computeSidecarBindingDigest( + { + pathLabel: note.entryPath, + bytes: options.entryBytes, + }, + sidecarParts, + note.seedTag, + "sidecars", + ); + if (measured.hex !== note.digest) { + return { + ok: false, + reason: "gguf sidecar digest mismatch", + note, + expectedHex: note.digest, + measuredHex: measured.hex, + measuredU32: measured.u32, + softFold: note.softFold, + }; + } + return { + ok: true, + note, + expectedHex: note.digest, + measuredHex: measured.hex, + measuredU32: measured.u32, + softFold: note.softFold, + }; +} + /** Test/helper: strip the sidecar note section from a WASM binary. */ export function stripWasmSidecarCustomSection(bytes: Uint8Array): Buffer { return stripWasmCustomSectionByName(Buffer.from(bytes), WASM_SIDECAR_SECTION_NAME); diff --git a/tests/defense-score-honesty.test.ts b/tests/defense-score-honesty.test.ts index 8295f91..37da323 100644 --- a/tests/defense-score-honesty.test.ts +++ b/tests/defense-score-honesty.test.ts @@ -81,6 +81,23 @@ describe("Issue #40.6: defenseScore honesty", () => { expect(hostOnly.cappedScore).toBe(93); }); + it("keeps a GGUF host-only STRING metadata note from closing the strip⇒fail gap", () => { + const off = applyDefenseScoreHonesty(99, { + detectedFormat: "model-gguf", + sidecarEntangleEnabled: false, + }); + expect(off.gaps.map((g) => g.id)).toContain("sidecar-entangle-not-enabled"); + expect(off.cappedScore).toBe(93); + + const hostOnly = applyDefenseScoreHonesty(99, { + detectedFormat: "model-gguf", + sidecarEntangleEnabled: true, + }); + expect(hostOnly.gaps.map((g) => g.id)).toContain("gguf-sidecar-host-verify-only"); + expect(hostOnly.gaps.map((g) => g.id)).not.toContain("sidecar-entangle-not-enabled"); + expect(hostOnly.cappedScore).toBe(93); + }); + it("does not raise a low raw score when gaps apply", () => { const honesty = applyDefenseScoreHonesty(40, { detectedFormat: "wasm", diff --git a/tests/gguf-layout.test.ts b/tests/gguf-layout.test.ts new file mode 100644 index 0000000..44205df --- /dev/null +++ b/tests/gguf-layout.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from "vitest"; +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + encodeGgufStringKv, + parseGgufLayout, + rebuildGguf, + GGUF_DEFAULT_ALIGNMENT, +} from "../src/shields/formats/gguf-layout.js"; +import { protectGgufBuffer } from "../src/shields/formats/model.js"; + +const VENV_PY = "/tmp/ggufvenv/bin/python"; +const HAS_GGUF_READER = (() => { + const r = spawnSync(VENV_PY, ["-c", "import gguf"], { encoding: "utf8" }); + return r.status === 0; +})(); + +function makeRealGguf(): Buffer { + const dir = mkdtempSync(join(tmpdir(), "bla-gguf-layout-")); + try { + const out = join(dir, "tiny.gguf"); + const script = ` +import gguf, numpy as np, sys +w = gguf.GGUFWriter(sys.argv[1], "llama") +w.add_name("tiny-test") +w.add_uint32("tiny.custom", 7) +w.add_tensor("tok_emb", np.arange(15, dtype=np.float32).reshape(3, 5)) +w.add_tensor("bias", np.array([1.5, -2.25, 3.0], dtype=np.float32)) +w.add_tensor("w2", (np.arange(7, dtype=np.float32) * 0.5)) +w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() +`; + const py = join(dir, "make.py"); + writeFileSync(py, script); + const r = spawnSync(VENV_PY, [py, out], { encoding: "utf8" }); + if (r.status !== 0) throw new Error(`gguf write failed: ${r.stderr || r.stdout}`); + return readFileSync(out); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function readTensorDigests(bytes: Buffer): Record { + const dir = mkdtempSync(join(tmpdir(), "bla-gguf-read-")); + try { + const path = join(dir, "m.gguf"); + writeFileSync(path, bytes); + const script = ` +import gguf, hashlib, json, sys +r = gguf.GGUFReader(sys.argv[1]) +print(json.dumps({ + "align": int(r.alignment), + "data_offset": int(r.data_offset), + "tensors": {t.name: hashlib.sha256(bytes(t.data.tobytes())).hexdigest() for t in r.tensors}, + "keys": [k for k in r.fields.keys() if not k.startswith("GGUF.")], +})) +`; + const py = join(dir, "read.py"); + writeFileSync(py, script); + const r = spawnSync(VENV_PY, [py, path], { encoding: "utf8" }); + if (r.status !== 0) throw new Error(`gguf read failed: ${r.stderr || r.stdout}`); + return JSON.parse(r.stdout.trim()); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +describe("gguf-layout alignment-safe rewrite", () => { + it("parses a real GGUF and reports the default alignment", () => { + if (!HAS_GGUF_READER) return; + const bytes = makeRealGguf(); + const layout = parseGgufLayout(bytes); + expect(layout).not.toBeNull(); + expect(layout!.alignment).toBe(GGUF_DEFAULT_ALIGNMENT); + expect(layout!.tensorCount).toBe(3n); + expect(layout!.hasDataRegion).toBe(true); + expect(layout!.dataStart % layout!.alignment).toBe(0); + }); + + it("rebuildGguf keeps every tensor byte-exact after KV append (reference reader)", () => { + if (!HAS_GGUF_READER) return; + const original = makeRealGguf(); + const before = readTensorDigests(original); + const layout = parseGgufLayout(original)!; + const rebuilt = rebuildGguf(original, layout, { + appendKvs: [encodeGgufStringKv("bla.test.note", '{"ok":true}')], + }); + const after = readTensorDigests(rebuilt); + expect(after.tensors).toEqual(before.tensors); + expect(after.keys).toContain("bla.test.note"); + expect(after.align).toBe(before.align); + }); + + it("protectGgufBuffer keeps every tensor byte-exact (reference reader)", () => { + if (!HAS_GGUF_READER) return; + const original = makeRealGguf(); + const before = readTensorDigests(original); + const protected_ = protectGgufBuffer(original, { + intensity: "balanced", + seed: "gguf-layout-test", + }); + const after = readTensorDigests(protected_.buffer); + expect(after.tensors).toEqual(before.tensors); + expect(after.keys).toContain("enclave.delegation.manifest"); + expect(after.keys).toContain("enclave.nesting.breaker"); + }); + + it("rejects non-GGUF / truncated input without throwing", () => { + expect(parseGgufLayout(Buffer.from("not-gguf"))).toBeNull(); + expect(parseGgufLayout(Buffer.from("GGUF"))).toBeNull(); + const magic = Buffer.alloc(24, 0); + magic.write("GGUF", 0, "ascii"); + magic.writeUInt32LE(3, 4); + magic.writeBigUInt64LE(1n, 8); // tensorCount + magic.writeBigUInt64LE(1n, 16); // kvCount — truncated body + expect(parseGgufLayout(magic)).toBeNull(); + }); + + it("idempotent drop+append does not change tensor digests", () => { + if (!HAS_GGUF_READER) return; + const original = makeRealGguf(); + const layout = parseGgufLayout(original)!; + const once = rebuildGguf(original, layout, { + appendKvs: [encodeGgufStringKv("bla.sidecar.v1", '{"d":"a"}')], + }); + const layout2 = parseGgufLayout(once)!; + const twice = rebuildGguf(once, layout2, { + dropKeys: new Set(["bla.sidecar.v1"]), + appendKvs: [encodeGgufStringKv("bla.sidecar.v1", '{"d":"b"}')], + }); + expect(readTensorDigests(once).tensors).toEqual( + readTensorDigests(original).tensors, + ); + expect(readTensorDigests(twice).tensors).toEqual( + readTensorDigests(original).tensors, + ); + // Content hash of the note itself must differ after re-bind. + expect(createHash("sha256").update(once).digest("hex")).not.toBe( + createHash("sha256").update(twice).digest("hex"), + ); + }); +}); diff --git a/tests/sidecar-gguf-inject.test.ts b/tests/sidecar-gguf-inject.test.ts new file mode 100644 index 0000000..030adb7 --- /dev/null +++ b/tests/sidecar-gguf-inject.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from "vitest"; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, + existsSync, +} from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + GGUF_SIDECAR_METADATA_KEY, + GGUF_SIDECAR_METADATA_KIND, + SIDECAR_ENTANGLE_SCHEMA, + bindGgufSidecarEntanglement, + computeSidecarBindingDigest, + entangleGgufEntryWithSidecars, + extractGgufSidecarMetadataNote, + generateSidecarGgufMetadataNote, + planSidecarEntanglement, + stripGgufSidecarMetadataNote, + verifyGgufSidecarMetadataNote, +} from "../src/shields/runtime/sidecar-entangle.js"; +import { + encodeGgufString, + encodeGgufStringKv, + parseGgufLayout, + rebuildGguf, +} from "../src/shields/formats/gguf-layout.js"; +import { protectTarget } from "../src/engine/protector.js"; +import { protectGgufBuffer } from "../src/shields/formats/model.js"; + +const VENV_PY = "/tmp/ggufvenv/bin/python"; +const HAS_GGUF_READER = (() => { + const r = spawnSync(VENV_PY, ["-c", "import gguf"], { encoding: "utf8" }); + return r.status === 0; +})(); + +/** + * Minimal GGUF v3 with one STRING KV and one UINT32 tensor-info + 4 data bytes. + * Built in-process so the suite does not require the gguf Python package. + * Alignment pad is computed so dataStart % 32 == 0. + */ +function buildMinimalGguf(): Buffer { + const kv = encodeGgufStringKv("general.architecture", "llama"); + // Tensor info: name "w" (u64len+bytes) + n_dims=1 + dim0=1 + ggml_type=0 + offset=0 + const name = encodeGgufString("w"); + const rest = Buffer.alloc(4 + 8 + 4 + 8); + rest.writeUInt32LE(1, 0); // n_dims + rest.writeBigUInt64LE(1n, 4); // dim0 + rest.writeUInt32LE(0, 12); // ggml_type F32-ish placeholder + rest.writeBigUInt64LE(0n, 16); // relative offset + const header = Buffer.alloc(24); + header.write("GGUF", 0, "ascii"); + header.writeUInt32LE(3, 4); + header.writeBigUInt64LE(1n, 8); // tensorCount + header.writeBigUInt64LE(1n, 16); // kvCount + const metaAndInfos = Buffer.concat([header, kv, name, rest]); + const align = 32; + const rem = metaAndInfos.length % align; + const pad = rem === 0 ? Buffer.alloc(0) : Buffer.alloc(align - rem, 0); + const data = Buffer.from([0xde, 0xad, 0xbe, 0xef]); + return Buffer.concat([metaAndInfos, pad, data]); +} + +function tensorDataDigest(bytes: Buffer): string { + const layout = parseGgufLayout(bytes)!; + return createHash("sha256") + .update(bytes.subarray(layout.dataStart)) + .digest("hex"); +} + +describe("sidecar GGUF metadata note", () => { + it("planSidecarEntanglement guesses gguf-metadata-note for .gguf", () => { + const plan = planSidecarEntanglement({ + entryPath: "model.gguf", + sidecarPaths: ["weights.bin"], + }); + expect(plan.entryHookKind).toBe("gguf-metadata-note"); + }); + + it("generateSidecarGgufMetadataNote emits a STRING KV with the binding note", () => { + const digest = computeSidecarBindingDigest( + { pathLabel: "model.gguf", bytes: Buffer.from("entry") }, + [{ pathLabel: "sib.bin", bytes: Buffer.from("sib") }], + "seed-a", + "sidecars", + ); + const inject = generateSidecarGgufMetadataNote({ + entryPath: "model.gguf", + sidecarPaths: ["sib.bin"], + seedTag: "seed-a", + expectedHex: digest.hex, + }); + expect(inject.schema).toBe(SIDECAR_ENTANGLE_SCHEMA); + expect(inject.kind).toBe(GGUF_SIDECAR_METADATA_KIND); + expect(inject.key).toBe(GGUF_SIDECAR_METADATA_KEY); + expect(inject.note.digest).toBe(digest.hex); + expect(inject.kvRecord.length).toBeGreaterThan(32); + // KV must start with the key length of "bla.sidecar.v1" + expect(inject.kvRecord.readBigUInt64LE(0)).toBe( + BigInt(Buffer.from(GGUF_SIDECAR_METADATA_KEY, "utf8").length), + ); + }); + + it("bind + verify round-trips; strip fails closed; tensors stay bit-exact", () => { + const entry = buildMinimalGguf(); + const layout0 = parseGgufLayout(entry); + expect(layout0).not.toBeNull(); + const dataDigest0 = tensorDataDigest(entry); + + const sib = Buffer.from("sidecar-payload-v1"); + const bound = bindGgufSidecarEntanglement({ + entryPath: "model.gguf", + sidecarPaths: ["sib.bin"], + seedTag: "seed-gguf", + entryBytes: entry, + sidecarBytes: [sib], + }); + expect(bound.plan.status).toBe("bound"); + expect(bound.plan.entryHookKind).toBe("gguf-metadata-note"); + expect(tensorDataDigest(bound.bytes)).toBe(dataDigest0); + + const note = extractGgufSidecarMetadataNote(bound.bytes); + expect(note).not.toBeNull(); + expect(note!.digest).toBe(bound.digest.hex); + expect(note!.sidecarPaths).toEqual(["sib.bin"]); + + const ok = verifyGgufSidecarMetadataNote({ + entryBytes: bound.bytes, + sidecarBytes: { "sib.bin": sib }, + }); + expect(ok.ok).toBe(true); + expect(ok.measuredHex).toBe(bound.digest.hex); + + // Altered sibling ⇒ fail closed + const bad = verifyGgufSidecarMetadataNote({ + entryBytes: bound.bytes, + sidecarBytes: { "sib.bin": Buffer.from("tampered") }, + }); + expect(bad.ok).toBe(false); + expect(bad.reason).toMatch(/mismatch/); + + // Missing note ⇒ fail closed + const stripped = stripGgufSidecarMetadataNote(bound.bytes); + expect(extractGgufSidecarMetadataNote(stripped)).toBeNull(); + expect(tensorDataDigest(stripped)).toBe(dataDigest0); + const missing = verifyGgufSidecarMetadataNote({ + entryBytes: stripped, + sidecarBytes: { "sib.bin": sib }, + }); + expect(missing.ok).toBe(false); + + // Re-bind is idempotent (single note) + const rebound = bindGgufSidecarEntanglement({ + entryPath: "model.gguf", + sidecarPaths: ["sib.bin"], + seedTag: "seed-gguf", + entryBytes: bound.bytes, + sidecarBytes: [sib], + }); + const layout1 = parseGgufLayout(rebound.bytes)!; + const noteKeys = layout1.kvs.filter((k) => k.key === GGUF_SIDECAR_METADATA_KEY); + expect(noteKeys.length).toBe(1); + expect(tensorDataDigest(rebound.bytes)).toBe(dataDigest0); + }); + + it("entangleGgufEntryWithSidecars returns null when no sidecars", () => { + expect( + entangleGgufEntryWithSidecars({ + entryPath: "model.gguf", + entryBytes: buildMinimalGguf(), + sidecars: [], + }), + ).toBeNull(); + }); + + it("protectTarget with sidecarEntangle plants the note (host-verify only)", () => { + const dir = mkdtempSync(join(tmpdir(), "bla-gguf-prot-")); + try { + const entry = buildMinimalGguf(); + // Pre-protect so we have decoys, then entangle via protectTarget. + const protected_ = protectGgufBuffer(entry, { + intensity: "balanced", + seed: "gguf-sidecar-protect", + }); + const inPath = join(dir, "model.gguf"); + const sibPath = join(dir, "sib.bin"); + const outPath = join(dir, "out.gguf"); + writeFileSync(inPath, protected_.buffer); + writeFileSync(sibPath, Buffer.from("sibling-bytes")); + const report = protectTarget({ + inputPath: inPath, + outputPath: outPath, + intensity: "balanced", + seed: "gguf-sidecar-protect", + sidecarEntangle: true, + sidecarEntanglePaths: ["sib.bin"], + }); + expect(report.sidecarEntangle).toBe(true); + expect(existsSync(outPath)).toBe(true); + const out = readFileSync(outPath); + const note = extractGgufSidecarMetadataNote(out); + expect(note).not.toBeNull(); + expect(note!.key).toBe(GGUF_SIDECAR_METADATA_KEY); + const verified = verifyGgufSidecarMetadataNote({ + entryBytes: out, + sidecarBytes: { "sib.bin": readFileSync(sibPath) }, + }); + expect(verified.ok).toBe(true); + // Honesty: do NOT claim the process-start strip⇒fail vector yet + // (host-verify only; load-time hook TBD). + expect(report.counteredReaVectors).not.toContain("gguf-sidecar-entangle"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("real gguf writer round-trip keeps tensors after bind (when reader available)", () => { + if (!HAS_GGUF_READER) return; + const dir = mkdtempSync(join(tmpdir(), "bla-gguf-real-")); + try { + const out = join(dir, "real.gguf"); + const script = ` +import gguf, numpy as np, sys +w = gguf.GGUFWriter(sys.argv[1], "llama") +w.add_name("tiny") +w.add_tensor("a", np.arange(11, dtype=np.float32)) +w.add_tensor("b", np.ones((2, 3), dtype=np.float32)) +w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() +`; + writeFileSync(join(dir, "make.py"), script); + const r = spawnSync(VENV_PY, [join(dir, "make.py"), out], { encoding: "utf8" }); + expect(r.status).toBe(0); + const entry = readFileSync(out); + const sib = Buffer.from("real-sib"); + const bound = bindGgufSidecarEntanglement({ + entryPath: "real.gguf", + sidecarPaths: ["sib.bin"], + seedTag: "seed-real", + entryBytes: entry, + sidecarBytes: [sib], + }); + writeFileSync(join(dir, "bound.gguf"), bound.bytes); + const readScript = ` +import gguf, hashlib, json, sys +r = gguf.GGUFReader(sys.argv[1]) +print(json.dumps({t.name: hashlib.sha256(bytes(t.data.tobytes())).hexdigest() for t in r.tensors})) +`; + writeFileSync(join(dir, "read.py"), readScript); + const before = spawnSync( + VENV_PY, + [join(dir, "read.py"), out], + { encoding: "utf8" }, + ); + const after = spawnSync( + VENV_PY, + [join(dir, "read.py"), join(dir, "bound.gguf")], + { encoding: "utf8" }, + ); + expect(before.status).toBe(0); + expect(after.status).toBe(0); + expect(JSON.parse(after.stdout)).toEqual(JSON.parse(before.stdout)); + expect( + verifyGgufSidecarMetadataNote({ + entryBytes: bound.bytes, + sidecarBytes: { "sib.bin": sib }, + }).ok, + ).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +});