russell@unturf.com
8d6961fcc1
modified: .gitlab-ci.yml modified: bench/qa_questions.txt modified: bench/qa_sweep.py modified: bench/run.sh modified: docs/TICKETS.md modified: docs/_source/README.md modified: docs/_source/_ext/makefile_targets.py modified: docs/_source/api/cli.rst modified: docs/_source/api/distill.rst modified: docs/_source/api/mesh.rst modified: docs/_source/api/qa.rst modified: docs/_source/api/retrieval.rst modified: docs/_source/api/storage.rst modified: docs/_source/api/substrate.rst modified: docs/_source/concepts.rst modified: docs/_source/conf.py modified: docs/_source/cookbook.rst modified: docs/_source/index.rst modified: docs/_source/license.rst modified: docs/_source/quickstart.rst modified: docs/bench-maxing.md modified: docs/benchmarks.md modified: docs/cti-architecture.md modified: docs/diagrams/aborist-modules.dot modified: docs/diagrams/aborist-modules.svg modified: docs/diagrams/mesh-data-flow.dot modified: docs/diagrams/mesh-epoch-lifecycle.dot modified: docs/diagrams/mesh-epoch-lifecycle.svg modified: docs/diagrams/mesh-group-decisions.dot modified: docs/diagrams/mesh-group-decisions.svg modified: docs/diagrams/mesh-identity-stack.dot modified: docs/diagrams/mesh-secret-envelope.dot modified: docs/mesh.md modified: docs/qa-modes-bench.md modified: docs/seven-point-program.md modified: docs/tickets/ticket-000001-retrieval-keywords-audit-gap.md modified: docs/tickets/ticket-000002-reference-frame-polarity-contract.md modified: docs/tickets/ticket-000003-anchor-class-warrant.md modified: docs/tickets/ticket-000005-label-ladder-migration.md modified: docs/tickets/ticket-000006-bench-emergent-findings.md modified: docs/tickets/ticket-000007-query-layer-hyphen-fold.md modified: docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md modified: docs/tickets/ticket-000009-quantifier-preflight-dag-binding.md modified: docs/tickets/ticket-000010-metacognition-preflight-guard.md modified: docs/tickets/ticket-000011-soft-preflight-hint-sidecar.md modified: scripts/backfill_concepts.py modified: scripts/bench_emergent.py modified: tests/crawler/test_async_web_fetcher.py modified: tests/crawler/test_bridge.py modified: tests/crawler/test_web_fetch.py modified: tests/test_bench_qa_sweep.py modified: tests/test_burn.py modified: tests/test_burn_doc.py modified: tests/test_claim_lattice.py modified: tests/test_cli_render.py modified: tests/test_compress.py modified: tests/test_concepts.py modified: tests/test_dag.py modified: tests/test_directives.py modified: tests/test_distill.py modified: tests/test_distill_recursive.py modified: tests/test_evict.py modified: tests/test_frame.py modified: tests/test_grok_source.py modified: tests/test_html_source.py modified: tests/test_ingest.py modified: tests/test_inspect.py modified: tests/test_journal.py modified: tests/test_keys.py modified: tests/test_llm_context_base.py modified: tests/test_merkle.py modified: tests/test_mesh.py modified: tests/test_mesh_aead.py modified: tests/test_mesh_chain.py modified: tests/test_mesh_cli.py modified: tests/test_mesh_cli_pull.py modified: tests/test_mesh_wire.py modified: tests/test_mesh_wire_e2e.py modified: tests/test_metacognition.py modified: tests/test_migration_audit_mode.py modified: tests/test_providence_source.py modified: tests/test_qa.py modified: tests/test_qa_quality_live.py modified: tests/test_quantifier_caps.py modified: tests/test_quantifier_classifier.py modified: tests/test_quantifier_phase4.py modified: tests/test_quantifier_reminder.py modified: tests/test_query.py modified: tests/test_reclassify.py modified: tests/test_repair.py modified: tests/test_resume.py modified: tests/test_snapshot.py modified: tests/test_soft_preflight.py modified: tests/test_tfidf.py modified: tests/test_vcs_source.py modified: tests/test_verify.py modified: tests/test_verify_json.py modified: tests/test_versioned_ingest.py modified: tests/test_warrant.py modified: tests/test_wikipedia_old.py modified: tests/test_wikipedia_xml.py modified: tests/test_wikitext.py
83 lines
2.7 KiB
Python
83 lines
2.7 KiB
Python
"""Pure-parse tests for HtmlPageSource. No network."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
# Skip if optional extras are not installed.
|
|
selectolax = pytest.importorskip("selectolax")
|
|
|
|
from arborist.sources.html_page import parse_html
|
|
|
|
|
|
SAMPLE_HTML = """<!doctype html>
|
|
<html>
|
|
<head>
|
|
<title>Eight Forms of Capital</title>
|
|
</head>
|
|
<body>
|
|
<header><h1>Site nav we want stripped</h1></header>
|
|
<nav>nav links also stripped</nav>
|
|
<main>
|
|
<h1>Eight Forms of Capital</h1>
|
|
<p>Living, Material, Financial, Intellectual.</p>
|
|
<p>Experiential, Social, Cultural, Spiritual.</p>
|
|
<p>See also <a href="/eight-forms-of-capital/">this page</a> and
|
|
<a href="https://example.org/external#section">an external link</a>.</p>
|
|
<a href="javascript:void(0)">js link</a>
|
|
<a href="mailto:foo@example.com">mail link</a>
|
|
</main>
|
|
<footer>strip me too</footer>
|
|
<script>console.log('strip me');</script>
|
|
<style>body { color: red }</style>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def test_parse_extracts_title_and_body():
|
|
doc = parse_html("https://example.com/page", SAMPLE_HTML)
|
|
assert doc is not None
|
|
assert doc.title == "Eight Forms of Capital"
|
|
assert "Living" in doc.content
|
|
assert "Spiritual" in doc.content
|
|
|
|
|
|
def test_parse_strips_noise():
|
|
doc = parse_html("https://example.com/page", SAMPLE_HTML)
|
|
assert doc is not None
|
|
assert "console.log" not in doc.content
|
|
assert "color: red" not in doc.content
|
|
assert "Site nav we want stripped" not in doc.content
|
|
assert "strip me too" not in doc.content
|
|
assert "nav links also stripped" not in doc.content
|
|
|
|
|
|
def test_parse_extracts_edges_and_resolves_relative():
|
|
doc = parse_html("https://example.com/page", SAMPLE_HTML)
|
|
assert doc is not None
|
|
uris = {e.dst_uri for e in doc.edges}
|
|
# Relative href resolved against the page URL.
|
|
assert "https://example.com/eight-forms-of-capital/" in uris
|
|
# External link kept; fragment moved to anchor.
|
|
assert "https://example.org/external" in uris
|
|
# javascript: / mailto: / tel: / hash-only anchors must be skipped.
|
|
assert all("javascript" not in u for u in uris)
|
|
assert all("mailto" not in u for u in uris)
|
|
|
|
|
|
def test_parse_anchor_split():
|
|
doc = parse_html("https://example.com/page", SAMPLE_HTML)
|
|
assert doc is not None
|
|
external = next(e for e in doc.edges if e.dst_uri == "https://example.org/external")
|
|
assert external.anchor == "section"
|
|
|
|
|
|
def test_parse_empty_body_returns_none():
|
|
doc = parse_html("https://example.com/empty", "<html><body></body></html>")
|
|
assert doc is None
|
|
|
|
|
|
def test_parse_no_html_returns_none_or_empty():
|
|
# Selectolax tolerates non-HTML; we want no Document for empty content.
|
|
doc = parse_html("https://example.com/x", "")
|
|
assert doc is None
|