From 02c7e41ef86becc84f2eca2e8e90726eb684eefe Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 7 May 2026 17:58:50 -0400 Subject: [PATCH] loss_report: land ticket #000022 (adapter LossReport sidecar) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Typed loss ledger for adapter / canonicalizer drops, transforms, and normalizations. Sidecar — never enters cache_key, document_root, run_dag_root, or audit_events. Loss policy lives in its own loss_report_policy_hash so toggling reporting does NOT invalidate prior QA cache entries (corrected pre-land per GPT-5.5 review). - adapter_loss_reports table: PK (chunk_id, stage, canonicalization_version, loss_kind); columns include loss_mode {pure_drop|transform|quarantine| normalize}, bytes_dropped, occurrence_count, input/output_length_bytes, sample_excerpt, sample_hash, adapter_name/version, loss_report_policy_hash - arborist/sources/loss_report.py: LossEvent, LossCollector with add()/record_delta()/set_lengths()/events(), record_losses() batched idempotent insert, compute_loss_report_policy_hash() pure function - wikitext.to_base() emits ref_tag, self_closing_ref_tag, file_link, image_link, category_link, strip_code_transform, whitespace_run. loss_collector=None default keeps verifier/runner/query path unchanged - html_page parse_html / _normalize_text emit script_block, style_block, html_chrome, whitespace_run; HtmlPageSource gains loss_report_* __init__ flags. Document-scope events anchor to first chunk_id at ingest via Document.extra['loss_events'] - ingest.ingest_source: per-chunk to_base() with collector for wikipedia_* sources; persisted via record_losses inside the same transaction as chunk inserts. Default loss_report_enabled=True - arborist losses CLI subcommand: --document-root / --chunk-id / --kind / --stage / --summary / --json. arborist ingest gains --no-loss-report / --no-loss-excerpts / --loss-excerpt-bytes - tests/test_loss_report.py: 15 tests covering bit-identical regression, loss-kind taxonomy, byte-conservation property test (loss-mode-aware), idempotent persistence, document_root invariant under toggle, policy hash purity 1091 tests pass, 0 audit-chain breaks across all 7 shards. --- arborist/cli.py | 158 +++++++ arborist/ingest.py | 92 +++- arborist/sources/html_page.py | 109 ++++- arborist/sources/loss_report.py | 329 ++++++++++++++ arborist/store.py | 107 +++++ arborist/wikitext.py | 84 +++- docs/TICKETS.md | 2 +- .../ticket-000022-adapter-loss-report.md | 211 +++++++-- tests/test_loss_report.py | 417 ++++++++++++++++++ 9 files changed, 1460 insertions(+), 49 deletions(-) create mode 100644 arborist/sources/loss_report.py create mode 100644 tests/test_loss_report.py diff --git a/arborist/cli.py b/arborist/cli.py index fef6a78..164bb26 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -164,6 +164,9 @@ def _cmd_ingest(args: argparse.Namespace) -> int: batch_size=args.batch_size, resume=args.resume, progress=progress, + loss_report_enabled=not args.no_loss_report, + loss_report_excerpts=not args.no_loss_excerpts, + loss_report_max_excerpt_bytes=args.loss_excerpt_bytes, ) finally: conn.close() @@ -1150,6 +1153,98 @@ def _short(s: str, n: int) -> str: return s if len(s) <= n else s[: n - 3] + "..." +def _cmd_losses(args: argparse.Namespace) -> int: + """Sidecar diagnostic: list adapter_loss_reports rows. + + Read-only. Filters: --document-root, --chunk-id, --kind, --stage. + With --summary, aggregates per (stage, loss_kind) for a single + document. See ticket #000022 §3.6. + """ + conn = ( + connect_query(args.db, shards_dir=args.global_shards_dir) + if args.global_shards_dir + else connect(args.db) + ) + try: + where: list[str] = [] + params: list = [] + if args.document_root: + where.append("document_root = ?") + params.append(args.document_root) + if args.chunk_id is not None: + where.append("chunk_id = ?") + params.append(args.chunk_id) + if args.kind: + where.append("loss_kind = ?") + params.append(args.kind) + if args.stage: + where.append("stage = ?") + params.append(args.stage) + clause = (" WHERE " + " AND ".join(where)) if where else "" + + if args.summary: + sql = ( + "SELECT stage, loss_kind, loss_mode, " + " SUM(bytes_dropped) AS total_bytes, " + " SUM(occurrence_count) AS total_occ, " + " COUNT(*) AS row_count " + "FROM adapter_loss_reports" + + clause + + " GROUP BY stage, loss_kind, loss_mode " + "ORDER BY stage, total_bytes DESC" + ) + rows = conn.execute(sql, params).fetchall() + if args.json: + out = [dict(r) for r in rows] + print(json.dumps(out, indent=2, ensure_ascii=False)) + else: + if not rows: + print("(no loss rows match)") + return 0 + print(f"{'stage':<18} {'kind':<28} {'mode':<11} {'occ':>7} {'bytes':>10} {'rows':>6}") + for r in rows: + print( + f"{r['stage']:<18} {r['loss_kind']:<28} " + f"{r['loss_mode']:<11} {r['total_occ']:>7,} " + f"{r['total_bytes']:>10,} {r['row_count']:>6,}" + ) + return 0 + + sql = ( + "SELECT chunk_id, document_root, stage, canonicalization_version, " + " loss_kind, loss_mode, bytes_dropped, occurrence_count, " + " input_length_bytes, output_length_bytes, sample_excerpt, " + " sample_hash, adapter_name, adapter_version, " + " loss_report_policy_hash, created_at " + "FROM adapter_loss_reports" + + clause + + " ORDER BY chunk_id, stage, loss_kind LIMIT ?" + ) + params.append(args.limit) + rows = conn.execute(sql, params).fetchall() + if args.json: + out = [dict(r) for r in rows] + print(json.dumps(out, indent=2, ensure_ascii=False)) + return 0 + if not rows: + print("(no loss rows match)") + return 0 + for r in rows: + print( + f"chunk={r['chunk_id']} doc={r['document_root'][:12]}.. " + f"stage={r['stage']} kind={r['loss_kind']} " + f"mode={r['loss_mode']} occ={r['occurrence_count']} " + f"bytes={r['bytes_dropped']}" + ) + if r["sample_excerpt"]: + print(f" excerpt: {_short(r['sample_excerpt'], 120)!r}") + elif r["sample_hash"]: + print(f" sample_hash: {r['sample_hash'][:16]}..") + return 0 + finally: + conn.close() + + def _falsify_cache_key( cache_key_value: str, *, @@ -3677,6 +3772,32 @@ def build_parser() -> argparse.ArgumentParser: "+ ETA in progress output" ), ) + ingest.add_argument( + "--no-loss-report", + dest="no_loss_report", + action="store_true", + help=( + "disable adapter LossReport sidecar (ticket #000022). " + "Default: enabled. Sidecar — toggling does NOT invalidate " + "QA cache_keys" + ), + ) + ingest.add_argument( + "--no-loss-excerpts", + dest="no_loss_excerpts", + action="store_true", + help=( + "drop sample_excerpt content from LossReport rows " + "(PII-paranoid mode). sample_hash stays populated" + ), + ) + ingest.add_argument( + "--loss-excerpt-bytes", + dest="loss_excerpt_bytes", + type=int, + default=200, + help="max excerpt byte length (default 200)", + ) ingest.set_defaults(func=_cmd_ingest) search = sub.add_parser("search", help="keyword search (UNGROUNDED audit mode)") @@ -3995,6 +4116,43 @@ def build_parser() -> argparse.ArgumentParser: ) inspect_cmd.set_defaults(func=_cmd_inspect) + losses_cmd = sub.add_parser( + "losses", + help=( + "list adapter LossReport sidecar rows (ticket #000022). " + "Read-only; never enters proof path" + ), + ) + losses_cmd.add_argument( + "--document-root", dest="document_root", default=None, + help="filter to one document_root (hex)", + ) + losses_cmd.add_argument( + "--chunk-id", dest="chunk_id", type=int, default=None, + help="filter to one chunk_id", + ) + losses_cmd.add_argument( + "--kind", default=None, + help="filter to one loss_kind (ref_tag, file_link, html_chrome, ...)", + ) + losses_cmd.add_argument( + "--stage", default=None, + help="filter to one stage (wikitext_base, html_normalize, ingest)", + ) + losses_cmd.add_argument( + "--summary", action="store_true", + help="aggregate by (stage, loss_kind, loss_mode)", + ) + losses_cmd.add_argument( + "--limit", type=int, default=200, + help="row cap for non-summary mode (default 200)", + ) + losses_cmd.add_argument( + "--json", action="store_true", + help="emit raw rows as JSON (default: human render)", + ) + losses_cmd.set_defaults(func=_cmd_losses) + prov_cmd = sub.add_parser( "providence", help="list or falsify providence_cache records", diff --git a/arborist/ingest.py b/arborist/ingest.py index 025e719..b2e14f4 100644 --- a/arborist/ingest.py +++ b/arborist/ingest.py @@ -63,6 +63,9 @@ def ingest_source( batch_size: int = DEFAULT_BATCH_SIZE, resume: bool = False, progress: Progress | None = None, + loss_report_enabled: bool = True, + loss_report_excerpts: bool = True, + loss_report_max_excerpt_bytes: int = 200, ) -> IngestStats: """Ingest every document the source yields. Returns counts. @@ -91,7 +94,14 @@ def ingest_source( def flush() -> None: if not batch: return - inserted, skipped = _flush_batch(conn, batch, chunker.name) + inserted, skipped = _flush_batch( + conn, + batch, + chunker.name, + loss_report_enabled=loss_report_enabled, + loss_report_excerpts=loss_report_excerpts, + loss_report_max_excerpt_bytes=loss_report_max_excerpt_bytes, + ) stats.inserted += inserted stats.skipped_duplicate += skipped batch.clear() @@ -142,6 +152,10 @@ def _flush_batch( conn: sqlite3.Connection, batch: list[tuple[Document, _DocArtifacts]], chunker_name: str, + *, + loss_report_enabled: bool = True, + loss_report_excerpts: bool = True, + loss_report_max_excerpt_bytes: int = 200, ) -> tuple[int, int]: """Bulk-insert the whole batch. Returns (inserted, skipped_duplicate). @@ -197,6 +211,19 @@ def _flush_batch( edge_rows: list[tuple] = [] audit_events: list[dict] = [] edges_to_upsert: list[tuple[str, Document]] = [] + # (chunk_id, document_root, [LossEvent...]) collected during the + # main loop, persisted at end-of-batch in one executemany. See + # ticket #000022 §3.5. Sidecar — does NOT enter audit_events. + loss_persist: list[tuple[int, str, list]] = [] + + loss_policy_hash = "" + if loss_report_enabled: + from arborist.sources.loss_report import compute_loss_report_policy_hash + loss_policy_hash = compute_loss_report_policy_hash( + enabled=loss_report_enabled, + excerpts=loss_report_excerpts, + max_excerpt_bytes=loss_report_max_excerpt_bytes, + ) for doc, art in batch: if art.document_root in existing or art.document_root in inserted_this_batch: @@ -220,15 +247,62 @@ def _flush_batch( ingest_ts, ) ) + is_wikitext_source = doc.source_type.startswith("wikipedia_") + first_chunk_id_for_doc: int | None = None for i, c in enumerate(art.chunk_strs): chunk_id = next_chunk_id next_chunk_id += 1 + if first_chunk_id_for_doc is None: + first_chunk_id_for_doc = chunk_id chunk_rows.append( (chunk_id, art.document_root, i, art.leaves[i].hex(), pack_chunk(c)) ) # Contentless FTS5 indexes the plaintext but stores no copy; # rowid must equal chunks.chunk_id so search-time JOINs line up. fts_rows.append((chunk_id, c)) + + # Per-chunk wikitext_base loss reporting. Run to_base() + # eagerly with a collector to capture what the LLM-side + # path would later drop. Output is discarded — + # chunks.content stays raw wikitext per the + # document_root invariant. + if loss_report_enabled and is_wikitext_source: + try: + from arborist.sources.loss_report import LossCollector + from arborist.wikitext import ( + ADAPTER_NAME as _WT_ADAPTER, + BASE_VERSION as _WT_VERSION, + to_base as _wt_to_base, + ) + collector = LossCollector( + excerpts_enabled=loss_report_excerpts, + max_excerpt_bytes=loss_report_max_excerpt_bytes, + adapter_name=_WT_ADAPTER, + adapter_version=_WT_VERSION, + ) + _wt_to_base(c, loss_collector=collector) + events = collector.events() + if events: + loss_persist.append( + (chunk_id, art.document_root, events) + ) + except ImportError: + # mwparserfromhell not installed — skip silently. + # LossReport is additive metadata, never a gate. + pass + + # Document-scope losses (e.g. HTML normalize) anchor to the + # first chunk_id by convention. See ticket #000022 §3.4. + if ( + loss_report_enabled + and first_chunk_id_for_doc is not None + and isinstance(doc.extra, dict) + ): + doc_events = doc.extra.get("loss_events") + if doc_events: + loss_persist.append( + (first_chunk_id_for_doc, art.document_root, list(doc_events)) + ) for layer_idx in range(1, len(art.tree.layers)): for node_idx, h in enumerate(art.tree.layers[layer_idx]): merkle_rows.append( @@ -286,6 +360,22 @@ def _flush_batch( merkle_rows, ) + # 3.5) Adapter LossReport sidecar (ticket #000022). Persisted + # under the same transaction as chunks so an auditor never sees + # a chunk_id without its loss rows. Sidecar — does NOT enter + # audit_events / cache_key / document_root. + if loss_persist: + from arborist.sources.loss_report import record_losses + for ck, drt, evs in loss_persist: + record_losses( + conn, + chunk_id=ck, + document_root=drt, + events=evs, + loss_report_policy_hash=loss_policy_hash, + ts=ingest_ts, + ) + # 4) Edges: collect all wikilinks across the batch and resolve in # one SELECT. Then one executemany. _flush_edges(conn, edges_to_upsert) diff --git a/arborist/sources/html_page.py b/arborist/sources/html_page.py index 2a8dcd9..16cfa9e 100644 --- a/arborist/sources/html_page.py +++ b/arborist/sources/html_page.py @@ -12,7 +12,7 @@ import re import urllib.parse import urllib.robotparser from pathlib import Path -from typing import Iterable, Iterator +from typing import TYPE_CHECKING, Iterable, Iterator try: import httpx @@ -25,28 +25,93 @@ except ImportError as e: # pragma: no cover from arborist.document import Document, Edge from arborist.source import Source +if TYPE_CHECKING: + from arborist.sources.loss_report import LossCollector + USER_AGENT = "arborist/0.0.1 (+https://unturf.com)" NOISE_SELECTORS = ("script", "style", "noscript", "nav", "header", "footer", "aside") +NORMALIZE_VERSION = "html-normalize-v1" +ADAPTER_NAME = "HtmlPageSource" + +# Loss-kind taxonomy per ticket #000022 §2.1.1. Free-string in v0; promoted +# to enum after >=3 adapters. ``html_chrome`` is heuristic — selectolax may +# leave residual nav text on pages that don't tag with semantic elements; +# auditors should treat the kind as advisory, not authoritative. +_NOISE_LOSS_KINDS = { + "script": "script_block", + "style": "style_block", + "noscript": "html_chrome", + "nav": "html_chrome", + "header": "html_chrome", + "footer": "html_chrome", + "aside": "html_chrome", +} -def _normalize_text(text: str) -> str: +def _normalize_text( + text: str, + *, + loss_collector: "LossCollector | None" = None, +) -> str: + pre_len = ( + len(text.encode("utf-8", errors="surrogatepass")) + if loss_collector is not None + else 0 + ) text = re.sub(r"[ \t]+", " ", text) text = re.sub(r"\n{3,}", "\n\n", text) - return text.strip() + text = text.strip() + if loss_collector is not None: + post_len = len(text.encode("utf-8", errors="surrogatepass")) + loss_collector.record_delta( + stage="html_normalize", + canonicalization_version=NORMALIZE_VERSION, + loss_kind="whitespace_run", + loss_mode="normalize", + bytes_delta=pre_len - post_len, + ) + return text -def parse_html(url: str, html: str, source_type: str = "html") -> Document | None: - """Pure parse function. Separated so tests can run without network.""" +def parse_html( + url: str, + html: str, + source_type: str = "html", + *, + loss_collector: "LossCollector | None" = None, +) -> Document | None: + """Pure parse function. Separated so tests can run without network. + + When ``loss_collector`` is provided, drops from noise-selector + decomposition (``