arborist/tests/test_wikitext.py

"""Unit tests for arborist.wikitext — wikitext-base-v1.

Most tests are tiny synthetic inputs that pin one rule each. The
``test_real_chunk_*`` tests use a real Wikipedia chunk (FF7 characters)
pulled from a live shard so we catch realistic interactions: nested
templates, mixed namespaces, refs inside paragraphs.

The fixture file ``tests/fixtures/ff7_characters_chunk0.wikitext`` is
chunk 0 of ``Characters_of_the_Final_Fantasy_VII_series`` and was the
input that triggered the wikitext-base-v1 design (verifier flagged 6
"quotes" as UNGROUNDED because the model paraphrased the wikitext form).
"""

from __future__ import annotations

from pathlib import Path

import pytest

from arborist.wikitext import BASE_VERSION, extract_wikilinks, to_base


FIXTURE = Path(__file__).parent / "fixtures" / "ff7_characters_chunk0.wikitext"


def test_base_version_pinned():
    """If you change BASE_VERSION you stale every cached answer. The pin
    here is a tripwire: bumping requires intent."""
    assert BASE_VERSION == "wikitext-base-v1"


def test_empty_input():
    assert to_base("") == ""
    assert to_base("   \n\t  ") == ""


def test_plain_prose_round_trips():
    src = "This is plain prose with no markup."
    assert to_base(src) == src


def test_simple_wikilink():
    assert to_base("Hello [[Cloud Strife]] world.") == "Hello Cloud Strife world."


def test_piped_wikilink_uses_display_text():
    assert to_base("Hello [[Cloud Strife|Cloud]] world.") == "Hello Cloud world."


def test_sectioned_wikilink_uses_display_text():
    src = "See [[Cloud Strife#weapons|the buster sword]]."
    assert to_base(src) == "See the buster sword."


def test_file_link_dropped_entirely():
    src = "[[File:foo.jpg|thumb|250px|A caption with [[Cloud]] inside.]] body text"
    out = to_base(src)
    assert "thumb" not in out
    assert "250px" not in out
    assert "foo.jpg" not in out
    assert "body text" in out


def test_image_link_dropped_entirely():
    src = "[[Image:bar.png|left|Some image]] body text"
    out = to_base(src)
    assert "left" not in out
    assert "bar.png" not in out
    assert "body text" in out


def test_category_link_dropped():
    src = "Body. [[Category:Final Fantasy VII characters]]"
    out = to_base(src)
    assert "Category:" not in out
    assert "Body." in out


def test_ref_tag_inline_dropped():
    src = "Cloud is great.<ref>Smith 2010, p. 5.</ref> The end."
    assert to_base(src) == "Cloud is great. The end."


def test_ref_tag_named_dropped():
    src = 'Cloud is great.<ref name="x">Smith 2010</ref> Reuse.<ref name="x"/> End.'
    out = to_base(src)
    assert "Smith" not in out
    assert "Cloud is great." in out
    assert "Reuse." in out
    assert "End." in out


def test_template_cite_dropped():
    src = "Plain prose. {{cite book|title=Foo|author=Bar}} more."
    out = to_base(src)
    assert "Foo" not in out
    assert "Plain prose." in out
    assert "more." in out


def test_template_infobox_dropped():
    src = "{{Infobox character|name=Cloud|hair=blond}} Body."
    assert to_base(src) == "Body."


def test_bold_and_italic_preserve_text():
    assert to_base("This is '''bold''' text.") == "This is bold text."
    assert to_base("This is ''italic'' text.") == "This is italic text."


def test_header_keeps_text():
    out = to_base("== Section Title ==\nBody under it.")
    assert "Section Title" in out
    assert "Body under it." in out
    assert "==" not in out


def test_html_entities_decoded():
    out = to_base("Tom &amp; Jerry, 50&nbsp;words.")
    assert "Tom & Jerry" in out
    assert "&amp;" not in out


def test_html_tags_removed_text_kept():
    out = to_base("Inline <small>caveat</small> text. <br/>Newline.")
    assert "caveat" in out
    assert "<" not in out
    assert ">" not in out


def test_external_link_with_text():
    assert to_base("Visit [https://example.com our site].") == "Visit our site."


def test_external_link_bare_dropped():
    out = to_base("Visit [https://example.com] now.")
    assert "https://" not in out
    assert "Visit" in out
    assert "now." in out


def test_html_comment_dropped():
    assert to_base("Visible <!-- hidden --> rest.") == "Visible rest."


def test_idempotence():
    """to_base(to_base(x)) == to_base(x). Critical for the verifier:
    if context is already-clean prose (e.g. an HTML source), running
    to_base on it must not corrupt the prose."""
    raw = (
        "Hello [[Cloud Strife|Cloud]]<ref>cite</ref>, see [[File:foo.jpg]] "
        "and {{Infobox|x=1}} the end."
    )
    once = to_base(raw)
    twice = to_base(once)
    assert once == twice


def test_nested_templates_dropped():
    src = "{{cite |title={{lang|en|Foo}}}} body."
    assert to_base(src) == "body."


# ---------------------------------------------------------------------------
# Real-corpus fixture: chunk 0 of Characters_of_the_Final_Fantasy_VII_series
# ---------------------------------------------------------------------------


@pytest.fixture
def ff7_raw() -> str:
    return FIXTURE.read_text()


def test_real_chunk_strips_markup(ff7_raw):
    """The chunk that triggered this design. After to_base, the named
    characters should appear as bare prose, not wikitext."""
    base = to_base(ff7_raw)
    # Concrete topical content survives.
    assert "Cloud Strife" in base
    assert "Sephiroth" in base
    assert "Final Fantasy VII" in base
    # Wikitext markers are gone.
    assert "[[" not in base
    assert "]]" not in base
    assert "{{" not in base
    assert "}}" not in base
    assert "<ref" not in base
    assert "File:" not in base
    assert "Category:" not in base


def test_real_chunk_protagonist_phrase_recoverable(ff7_raw):
    """The wikitext source has:
        ...follows [[protagonist]] [[Cloud Strife]], a troubled mercenary...
    After to_base this should read as continuous prose. We verify the
    canonical FF7 pitch surfaces verbatim — the verifier can substring-test
    against this exact form."""
    base = to_base(ff7_raw)
    assert "follows protagonist Cloud Strife, a troubled mercenary" in base


def test_real_chunk_substantial_compression(ff7_raw):
    """Sanity check: the base form should be smaller than raw (markup
    removed) but not so small that we've lost the prose."""
    base = to_base(ff7_raw)
    assert len(base) < len(ff7_raw)
    # FF7 chunk 0 is 3722 chars raw; base form is roughly 75% of that.
    assert len(base) > len(ff7_raw) * 0.5


# ---------------------------------------------------------------------------
# extract_wikilinks — the link graph artifact
# ---------------------------------------------------------------------------


def test_extract_wikilinks_simple():
    out = extract_wikilinks("Hello [[Cloud Strife]] world.")
    assert out == [("Cloud Strife", None)]


def test_extract_wikilinks_with_display():
    out = extract_wikilinks("Hello [[Cloud Strife|Cloud]] world.")
    assert out == [("Cloud Strife", "Cloud")]


def test_extract_wikilinks_section_collapsed():
    """[[Foo#bar]] and [[Foo#baz]] both collapse to ('Foo', None)."""
    out = extract_wikilinks("See [[Foo#bar]] and [[Foo#baz|qux]].")
    assert out == [("Foo", None), ("Foo", "qux")]


def test_extract_wikilinks_includes_namespaces():
    """File: / Image: / Category: are dropped from PROSE but kept in the
    link graph — they are exactly the structural signal the graph wants."""
    src = "[[File:x.jpg]] [[Category:Y]] [[Image:z.png|caption]] [[Cloud]]"
    out = extract_wikilinks(src)
    targets = [t for t, _ in out]
    assert "File:x.jpg" in targets
    assert "Category:Y" in targets
    assert "Image:z.png" in targets
    assert "Cloud" in targets


def test_extract_wikilinks_empty_input():
    assert extract_wikilinks("") == []
    assert extract_wikilinks("   ") == []
    assert extract_wikilinks("plain prose, no links.") == []


def test_extract_wikilinks_real_chunk(ff7_raw):
    out = extract_wikilinks(ff7_raw)
    assert len(out) > 10
    targets = {t for t, _ in out}
    # A few links we expect from chunk 0.
    assert "Cloud Strife" in targets
    assert "Final Fantasy VII" in targets