feat: version-lineage report in the crawler ingestion pipeline
When a re-crawl detects a real content delta (a just-ingested root that supersedes a prior version — content hash changed, not redeploy/ETag noise the idempotent ingest already no-op'd), the pipeline now surfaces the page's document chain over time instead of just 'something changed'. bridge.py: version_chain(conn, uri) walks a URI's documents by ingest_ts (each content change = new content-addressed doc + supersedes edge); delta_report() adds the word-level similarity of the latest change; render_delta_report() prints it. ingest_crawled() detects superseding roots, emits the lineage report to stderr per changed page, and returns 'deltas' in its summary. Validated on the live russell.ballestrini.net re-crawl: 223 pages, full redeploy, exactly 1 content change (/about/), rendered as a 2-version chain (90% similar to prior). 2 tests; suite 2551 passed.
This commit is contained in:
parent
39f8aa1fb4
commit
aec4b544ab
2 changed files with 182 additions and 0 deletions
73
tests/test_version_lineage.py
Normal file
73
tests/test_version_lineage.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
"""Version lineage in the crawler ingestion pipeline (2026-05-21).
|
||||
|
||||
A content change to a URI writes a new content-addressed document + a
|
||||
``supersedes`` edge, so a URI accumulates a chain of versions over time.
|
||||
A redeploy that doesn't change content is a no-op (same Merkle root) and
|
||||
adds NO version. The crawler emits a lineage report when it detects a
|
||||
real delta. See arborist/sources/crawler/bridge.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterator
|
||||
|
||||
from arborist.document import Document
|
||||
from arborist.ingest import ingest_source
|
||||
from arborist.source import Source
|
||||
from arborist.sources.crawler.bridge import (
|
||||
delta_report,
|
||||
render_delta_report,
|
||||
version_chain,
|
||||
)
|
||||
from arborist.store import connect
|
||||
|
||||
|
||||
class FakeSource(Source):
|
||||
source_type = "fake"
|
||||
|
||||
def __init__(self, docs):
|
||||
self.docs = docs
|
||||
|
||||
def iter_documents(self) -> Iterator[Document]:
|
||||
yield from self.docs
|
||||
|
||||
|
||||
def _doc(uri: str, content: str) -> Document:
|
||||
return Document(uri=uri, content=content, source_type="fake",
|
||||
title="page", edges=[])
|
||||
|
||||
|
||||
def test_content_change_builds_a_version_chain(tmp_path):
|
||||
conn = connect(tmp_path / "t.db")
|
||||
uri = "test://page"
|
||||
ingest_source(conn, FakeSource([_doc(uri, "alpha bravo charlie delta echo")]))
|
||||
# changed content, same URI -> new root + supersedes edge
|
||||
ingest_source(conn, FakeSource(
|
||||
[_doc(uri, "alpha bravo charlie delta ECHO foxtrot golf")]))
|
||||
|
||||
chain = version_chain(conn, uri)
|
||||
assert len(chain) == 2 # two versions over time
|
||||
|
||||
rep = delta_report(conn, uri)
|
||||
assert rep["n_versions"] == 2
|
||||
assert "latest_delta" in rep
|
||||
assert 0.0 < rep["latest_delta"]["similarity"] < 1.0 # changed, not total
|
||||
|
||||
txt = render_delta_report(rep)
|
||||
assert "content changed" in txt
|
||||
assert "v1" in txt and "v2" in txt
|
||||
assert "similar to prior" in txt
|
||||
|
||||
|
||||
def test_identical_recrawl_adds_no_version(tmp_path):
|
||||
# A redeploy with byte-identical content must NOT create a new version
|
||||
# (content-addressed idempotence — the whole point).
|
||||
conn = connect(tmp_path / "t.db")
|
||||
uri = "test://stable"
|
||||
ingest_source(conn, FakeSource([_doc(uri, "same content stays the same")]))
|
||||
ingest_source(conn, FakeSource([_doc(uri, "same content stays the same")]))
|
||||
|
||||
chain = version_chain(conn, uri)
|
||||
assert len(chain) == 1 # no-op re-ingest -> still one version
|
||||
rep = delta_report(conn, uri)
|
||||
assert rep["n_versions"] == 1
|
||||
assert "latest_delta" not in rep
|
||||
Loading…
Add table
Add a link
Reference in a new issue