"""TeX-source textbook ingest (Project Gutenberg eBook style).

Targets the canonical Project Gutenberg LaTeX format used for math /
science textbooks where no HTML edition exists (e.g., Hilbert
*Foundations of Geometry* eBook #17384, Boole *An Investigation of
the Laws of Thought* eBook #15114). One Document per TeX source URL;
the strip pipeline produces searchable plain text suitable for the
existing 512-token chunker + Merkle commitment + audit chain.

Why this exists
---------------
The first iteration of textbook surface-ingest (#000031) used
``arborist/sources/html_page.py`` against Wikisource / openmathbooks
mirrors. Two of the most foundational PD textbooks have no clean
HTML edition — Project Gutenberg ships them only as PDF + LaTeX
source. Pandoc fails on PG's custom preamble macros (`\\rfa`,
`\\thispagestyle`, etc.); a focused regex stripper is the right
amount of machinery for the well-known PG TeX format.

What gets stripped
------------------
1. Line comments (`%`-prefixed lines and trailing `%` to EOL).
2. Preamble (everything before the first ``\\begin{document}``).
3. Block environments that yield no useful prose:
   ``tabular``, ``figure``, ``thebibliography``, ``titlepage``,
   ``flushright``, ``flushleft``, ``center``, ``small``, ``footnotesize``,
   ``scshape``, ``itemize``, ``enumerate`` (kept), and other
   structural-only environments.
4. Common single-arg commands stripped to their argument:
   ``\\textbf{X}`` → ``X``; ``\\emph{X}`` → ``X``; ``\\rfa{X}`` →
   ``X``; ``\\section{X}`` → ``X``; ``\\chapter{X}`` → ``X``;
   ``\\paragraph{X}`` → ``X``.
5. Zero-arg / structural commands dropped:
   ``\\noindent``, ``\\bigskip``, ``\\smallskip``, ``\\medskip``,
   ``\\par``, ``\\linebreak``, ``\\newpage``, ``\\clearpage``,
   ``\\thispagestyle{...}``, ``\\setcounter{...}{...}``,
   ``\\pageref{...}``, ``\\label{...}``, ``\\index{...}``.
6. Math-mode delimiters preserved as-is. The body inside ``$...$``
   stays raw — FTS5 will index Greek-letter and operator tokens; the
   wikitext-base canonicalizer can handle math separately if needed.
7. Common macro substitutions:
   ``\\S`` → ``§``; ``\\dots``/``\\ldots`` → ``…``; ``\\&`` → ``&``;
   ``\\to`` → ``→``; ``\\neg`` → ``¬``; ``\\lor`` → ``∨``;
   ``\\land`` → ``∧``; ``~`` → ``\xa0`` (non-break space);
   ``\\\\`` → newline.

What's deliberately not stripped
--------------------------------
We don't try to render math, build cross-references, or extract
figures. The output is plain prose with embedded math source —
suitable for retrieval, not for reproduction. The PG PDF is the
canonical visual rendering.

Idempotency
-----------
Same TeX URL → same response body → same Merkle root → no-op
re-ingest. The TeX content is wrapped in a Document with
``source_type='textbook_tex'`` so re-runs against the same shard
deduplicate via the standard content-addressed insert.

Source: ticket #000031 §5 (PDF/TeX support).
"""

from __future__ import annotations

import re
from pathlib import Path
from typing import Iterator

import httpx

from arborist.document import Document, Edge
from arborist.source import Source


USER_AGENT = "arborist/0.0.1 (+https://unturf.com)"
NORMALIZE_VERSION = "tex-strip-v1"


_PREAMBLE_RE = re.compile(r"^.*?\\begin\{document\}", re.DOTALL)
_POSTMATTER_RE = re.compile(r"\\end\{document\}.*$", re.DOTALL)
_LINE_COMMENT_RE = re.compile(r"(?m)(?<!\\)%[^\n]*")

# Block environments that produce no useful prose. The body between
# \begin{X} ... \end{X} is dropped wholesale.
_DROP_ENVS = (
    "tabular", "tabularx", "tabular*",
    "figure", "figure*", "wrapfigure",
    "thebibliography",
    "titlepage", "abstract",
    "flushright", "flushleft", "center",
    "small", "footnotesize", "scriptsize", "tiny",
    "scshape", "rmfamily", "sffamily", "ttfamily",
    "verbatim", "lstlisting",
)
_DROP_ENV_RE = re.compile(
    r"\\begin\{(" + "|".join(re.escape(e) for e in _DROP_ENVS) + r")\}.*?\\end\{\1\}",
    re.DOTALL,
)

# Single-arg commands stripped to their argument (regex captures
# argument's TeX, not balanced-brace correct but works on PG's
# straightforward usage).
_KEEP_ARG_CMDS = (
    "textbf", "textit", "emph", "underline", "textsc",
    "rfa", "rfb",  # PG custom: roman fixed all-caps
    "chapter", "chapter*",
    "section", "section*", "subsection", "subsection*",
    "subsubsection", "paragraph", "subparagraph",
    "footnote",
    "title", "author",
    "mbox", "fbox",
)
_KEEP_ARG_RE = re.compile(
    r"\\(" + "|".join(re.escape(c) for c in _KEEP_ARG_CMDS) + r")\*?\{([^{}]*)\}"
)

# Zero-arg + simple-arg commands dropped wholesale.
_DROP_CMDS_ZEROARG = (
    "noindent", "bigskip", "smallskip", "medskip", "vspace", "hspace",
    "par", "linebreak", "newpage", "clearpage", "newline",
    "leavevmode", "centering", "raggedright", "raggedleft",
    "hline", "cline", "rule",
    "maketitle", "tableofcontents", "listoffigures", "listoftables",
)
_DROP_ZEROARG_RE = re.compile(
    r"\\(" + "|".join(re.escape(c) for c in _DROP_CMDS_ZEROARG) + r")\b\*?"
)
_DROP_BRACE_CMDS = (
    "thispagestyle", "pagestyle",
    "setcounter", "addtocounter", "stepcounter",
    "label", "ref", "pageref", "eqref", "cite",
    "index", "indexset", "marginpar",
    "documentclass", "usepackage", "input", "include",
    "newcommand", "renewcommand", "newenvironment", "renewenvironment",
    "definecolor", "color", "textcolor",
    "setlength", "addtolength", "newlength",
    "geometry", "fancyhead", "fancyfoot",
)
_DROP_BRACE_RE = re.compile(
    r"\\(" + "|".join(re.escape(c) for c in _DROP_BRACE_CMDS)
    + r")\*?(?:\[[^\]]*\])?\{[^{}]*\}(?:\{[^{}]*\})?"
)

# Macro substitutions — symbol-level.
_MACRO_SUBS = [
    (r"\\S\b", "§"),
    (r"\\P\b", "¶"),
    (r"\\copyright\b", "©"),
    (r"\\dots\b", "…"),
    (r"\\ldots\b", "…"),
    (r"\\cdots\b", "…"),
    (r"\\to\b", "→"),
    (r"\\rightarrow\b", "→"),
    (r"\\leftarrow\b", "←"),
    (r"\\leftrightarrow\b", "↔"),
    (r"\\Rightarrow\b", "⇒"),
    (r"\\Leftarrow\b", "⇐"),
    (r"\\Leftrightarrow\b", "⇔"),
    (r"\\neg\b", "¬"),
    (r"\\lnot\b", "¬"),
    (r"\\lor\b", "∨"),
    (r"\\vee\b", "∨"),
    (r"\\land\b", "∧"),
    (r"\\wedge\b", "∧"),
    (r"\\forall\b", "∀"),
    (r"\\exists\b", "∃"),
    (r"\\equiv\b", "≡"),
    (r"\\neq\b", "≠"),
    (r"\\leq\b", "≤"),
    (r"\\geq\b", "≥"),
    (r"\\times\b", "×"),
    (r"\\cdot\b", "·"),
    (r"\\pm\b", "±"),
    (r"\\infty\b", "∞"),
    (r"\\in\b", "∈"),
    (r"\\notin\b", "∉"),
    (r"\\subset\b", "⊂"),
    (r"\\supset\b", "⊃"),
    (r"\\cup\b", "∪"),
    (r"\\cap\b", "∩"),
    (r"\\alpha\b", "α"),
    (r"\\beta\b", "β"),
    (r"\\gamma\b", "γ"),
    (r"\\delta\b", "δ"),
    (r"\\epsilon\b", "ε"),
    (r"\\theta\b", "θ"),
    (r"\\lambda\b", "λ"),
    (r"\\mu\b", "μ"),
    (r"\\pi\b", "π"),
    (r"\\sigma\b", "σ"),
    (r"\\phi\b", "φ"),
    (r"\\omega\b", "ω"),
    (r"\\Sigma\b", "Σ"),
    (r"\\Theta\b", "Θ"),
    (r"\\Phi\b", "Φ"),
    (r"\\Omega\b", "Ω"),
    (r"\\&", "&"),
    (r"\\\$", "$"),
    (r"\\#", "#"),
    (r"\\_", "_"),
    (r"\\\\", "\n"),
    (r"~", "\xa0"),
]
_MACRO_SUB_PATTERNS = [(re.compile(p), s) for p, s in _MACRO_SUBS]

# Whitespace cleanup
_MULTI_SPACE_RE = re.compile(r"[ \t]{2,}")
_MULTI_NEWLINE_RE = re.compile(r"\n{3,}")


def strip_tex(tex: str) -> str:
    """Convert a Project Gutenberg LaTeX source body into plain prose.

    Imperfect by design — a focused regex pipeline tuned for the PG
    canonical format. The output is suitable for retrieval (FTS5
    indexing, embeddings, semantic search) but NOT for reproduction:
    layout, figures, exact mathematical typesetting, and
    cross-references are not preserved.

    Pipeline (order matters):
      1. Drop everything before ``\\begin{document}``.
      2. Drop everything after ``\\end{document}``.
      3. Strip line comments.
      4. Drop block environments that yield no prose.
      5. Strip 0-arg / brace-arg structural commands.
      6. Reduce single-arg commands to their argument.
      7. Apply symbol-level macro substitutions.
      8. Whitespace normalize.
    """
    text = tex
    text = _PREAMBLE_RE.sub("", text)
    text = _POSTMATTER_RE.sub("", text)
    text = _LINE_COMMENT_RE.sub("", text)
    # Run the drop-env pass twice to handle nested same-type drops
    # (e.g., a `tabular` inside a `figure`).
    for _ in range(2):
        text = _DROP_ENV_RE.sub("", text)
    text = _DROP_BRACE_RE.sub("", text)
    text = _DROP_ZEROARG_RE.sub("", text)
    text = _KEEP_ARG_RE.sub(r"\2", text)
    for pat, sub in _MACRO_SUB_PATTERNS:
        text = pat.sub(sub, text)
    # Drop any residual single-token commands (catch-all for unknown
    # cleanup-safe commands).
    text = re.sub(r"\\[A-Za-z]+\*?", "", text)
    # Drop residual {...} braces that lost their command.
    text = re.sub(r"[{}]+", "", text)
    # Whitespace normalize
    text = _MULTI_SPACE_RE.sub(" ", text)
    text = _MULTI_NEWLINE_RE.sub("\n\n", text)
    return text.strip()


class TextbookTexSource(Source):
    """Fetch + strip a Project-Gutenberg-style LaTeX source URL into a
    single Document.

    One URL = one Document. Use a manifest entry's ``tex_url`` field
    or pass URLs directly. Idempotent at the database layer because
    the strip output is deterministic (same TeX → same prose → same
    document_root).
    """

    source_type = "textbook_tex"

    def __init__(
        self,
        urls: list[str] | str,
        *,
        timeout: float = 60.0,
        default_author: str | None = None,
    ):
        """``default_author`` (Phase 1 follow-up of #000031) — fallback
        when the LaTeX source has no ``\\author{}`` macro and no PG-
        style ``Author:`` boilerplate (e.g., Peano's mdnahas/Peano_Book
        TeX). Wired into the title via ``"<title> by <author>"`` for
        the warrant resolver's author-surname haystack check. Skipped
        when None or empty."""
        if isinstance(urls, str):
            urls = [urls]
        self.urls = list(urls)
        self.timeout = timeout
        self.default_author = (default_author or "").strip()

    def iter_documents(self) -> Iterator[Document]:
        with httpx.Client(
            headers={"User-Agent": USER_AGENT},
            timeout=self.timeout,
            follow_redirects=True,
        ) as client:
            for url in self.urls:
                try:
                    resp = client.get(url)
                    resp.raise_for_status()
                except httpx.HTTPError:
                    continue
                tex = resp.text
                prose = strip_tex(tex)
                if not prose or len(prose) < 200:
                    # Stripped output too thin to be useful — likely
                    # an error page or a non-TeX response.
                    continue
                title = _extract_title(tex)
                if not title:
                    # No \title{} or PG Title: line — fall back to URL,
                    # but still append default_author when configured
                    # so the haystack picks up the surname.
                    title = url
                if self.default_author and not _title_has_author(
                    title, self.default_author
                ):
                    title = f"{title} by {self.default_author}"
                yield Document(
                    uri=str(resp.url),
                    content=prose,
                    source_type=self.source_type,
                    title=title,
                    edges=[],
                    extra={"tex_url": url, "normalize_version": NORMALIZE_VERSION},
                )

    @classmethod
    def from_file(cls, path: str | Path, **kwargs) -> "TextbookTexSource":
        urls = [
            line.strip()
            for line in Path(path).read_text(encoding="utf-8").splitlines()
            if line.strip() and not line.lstrip().startswith("#")
        ]
        return cls(urls, **kwargs)


_TITLE_RE = re.compile(r"\\title\{([^{}]+)\}")
_AUTHOR_RE = re.compile(r"\\author\{([^{}]+)\}")
_PG_TITLE_RE = re.compile(r"^Title:\s*(.+?)\s*$", re.MULTILINE)
_PG_AUTHOR_RE = re.compile(r"^Author:\s*(.+?)\s*$", re.MULTILINE)


def _title_has_author(title: str, author: str) -> bool:
    """True if any surname-shaped token (length >= 4) from ``author``
    appears in ``title`` (case-insensitive substring). Used to decide
    whether to append ``default_author`` — skip when the title
    already carries the surname (avoids double-stamping).
    """
    title_l = (title or "").lower()
    for tok in (author or "").split():
        tok_l = tok.lower()
        if len(tok_l) >= 4 and tok_l in title_l:
            return True
    return False


def _extract_title(tex: str) -> str:
    """Pull the title — prefer ``\\title{...}`` if present (LaTeX
    convention), fall back to ``Title:`` line (Project Gutenberg
    boilerplate format which doesn't always wire into a LaTeX
    title macro). Append author when discoverable so downstream
    citation-matching code (``arborist.qa.warrant_resolver``) can
    match on author last name without re-parsing the TeX."""
    title = ""
    m = _TITLE_RE.search(tex)
    if m:
        title = m.group(1).strip()
    if not title:
        m = _PG_TITLE_RE.search(tex)
        if m:
            title = m.group(1).strip()
    if not title:
        return ""
    author = ""
    m = _AUTHOR_RE.search(tex)
    if m:
        author = m.group(1).strip()
    if not author:
        m = _PG_AUTHOR_RE.search(tex)
        if m:
            author = m.group(1).strip()
    if author:
        return f"{title} by {author}"
    return title
