russell@unturf.com
aec4b544ab
When a re-crawl detects a real content delta (a just-ingested root that supersedes a prior version — content hash changed, not redeploy/ETag noise the idempotent ingest already no-op'd), the pipeline now surfaces the page's document chain over time instead of just 'something changed'. bridge.py: version_chain(conn, uri) walks a URI's documents by ingest_ts (each content change = new content-addressed doc + supersedes edge); delta_report() adds the word-level similarity of the latest change; render_delta_report() prints it. ingest_crawled() detects superseding roots, emits the lineage report to stderr per changed page, and returns 'deltas' in its summary. Validated on the live russell.ballestrini.net re-crawl: 223 pages, full redeploy, exactly 1 content change (/about/), rendered as a 2-version chain (90% similar to prior). 2 tests; suite 2551 passed.
73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
"""Version lineage in the crawler ingestion pipeline (2026-05-21).
|
|
|
|
A content change to a URI writes a new content-addressed document + a
|
|
``supersedes`` edge, so a URI accumulates a chain of versions over time.
|
|
A redeploy that doesn't change content is a no-op (same Merkle root) and
|
|
adds NO version. The crawler emits a lineage report when it detects a
|
|
real delta. See arborist/sources/crawler/bridge.py.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Iterator
|
|
|
|
from arborist.document import Document
|
|
from arborist.ingest import ingest_source
|
|
from arborist.source import Source
|
|
from arborist.sources.crawler.bridge import (
|
|
delta_report,
|
|
render_delta_report,
|
|
version_chain,
|
|
)
|
|
from arborist.store import connect
|
|
|
|
|
|
class FakeSource(Source):
|
|
source_type = "fake"
|
|
|
|
def __init__(self, docs):
|
|
self.docs = docs
|
|
|
|
def iter_documents(self) -> Iterator[Document]:
|
|
yield from self.docs
|
|
|
|
|
|
def _doc(uri: str, content: str) -> Document:
|
|
return Document(uri=uri, content=content, source_type="fake",
|
|
title="page", edges=[])
|
|
|
|
|
|
def test_content_change_builds_a_version_chain(tmp_path):
|
|
conn = connect(tmp_path / "t.db")
|
|
uri = "test://page"
|
|
ingest_source(conn, FakeSource([_doc(uri, "alpha bravo charlie delta echo")]))
|
|
# changed content, same URI -> new root + supersedes edge
|
|
ingest_source(conn, FakeSource(
|
|
[_doc(uri, "alpha bravo charlie delta ECHO foxtrot golf")]))
|
|
|
|
chain = version_chain(conn, uri)
|
|
assert len(chain) == 2 # two versions over time
|
|
|
|
rep = delta_report(conn, uri)
|
|
assert rep["n_versions"] == 2
|
|
assert "latest_delta" in rep
|
|
assert 0.0 < rep["latest_delta"]["similarity"] < 1.0 # changed, not total
|
|
|
|
txt = render_delta_report(rep)
|
|
assert "content changed" in txt
|
|
assert "v1" in txt and "v2" in txt
|
|
assert "similar to prior" in txt
|
|
|
|
|
|
def test_identical_recrawl_adds_no_version(tmp_path):
|
|
# A redeploy with byte-identical content must NOT create a new version
|
|
# (content-addressed idempotence — the whole point).
|
|
conn = connect(tmp_path / "t.db")
|
|
uri = "test://stable"
|
|
ingest_source(conn, FakeSource([_doc(uri, "same content stays the same")]))
|
|
ingest_source(conn, FakeSource([_doc(uri, "same content stays the same")]))
|
|
|
|
chain = version_chain(conn, uri)
|
|
assert len(chain) == 1 # no-op re-ingest -> still one version
|
|
rep = delta_report(conn, uri)
|
|
assert rep["n_versions"] == 1
|
|
assert "latest_delta" not in rep
|