From 5a77ce49336d4c50825bc562388d7d843ec9f8a7 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 27 Apr 2026 12:15:47 -0400 Subject: [PATCH] fix _iter_insert_statements: persist scan_pos across chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The optimized parser silently dropped articles whose body contained an HTML decimal entity (&#NNNNN;) — affecting roughly any Wikipedia article referencing a Japanese/Korean/Chinese interwiki link. ~6,000 docs missing from the 128k cur ingest. The HTTP article was one casualty. Root cause: when the scanner ran out of pending data while inside a quoted string, flush_complete() returned with in_string=True. The outer read-loop trimmed pending and called flush_complete() again, which RESET i=0 and searched for the next "'" from the start of the buffer. The first quote found was the title-field open quote of the article we were already deep inside; bs=0 (no preceding backslash) so the parser incorrectly flipped in_string=False and treated the next ';' (typically inside an &#NNNNN; entity) as a statement terminator. Fix: persist scan_pos across read() chunks. After a trim, remap scan_pos = scan_pos - yield_from. The next call resumes scanning at the correct position rather than restarting at 0. Verified: optimized parser now yields identical statements to the canonical char-by-char reference on the first 60 MB of the dump. HTTP article correctly emitted at doc #7722 with full 2375-char body. Cost: ~50% slower on a 5000-doc benchmark (11.8s vs broken 7.8s) due to the correct cross-chunk state tracking. Worth the trade — a corpus that randomly drops articles is worse than a slightly slower complete one. Still ~3x faster than the original char-by-char parser. The existing 122k cur shard set is missing the same ~6k articles. To recover: rm -rf ~/.aborist/shards && make ingest-cur-attached SHARDS=4 takes ~3-4 min wall. 57 tests still passing. --- aborist/sources/wikipedia.py | 58 ++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/aborist/sources/wikipedia.py b/aborist/sources/wikipedia.py index b202fd0..9721fa1 100644 --- a/aborist/sources/wikipedia.py +++ b/aborist/sources/wikipedia.py @@ -142,23 +142,33 @@ def _split_values_tuples(payload: str) -> list[list[str | None]]: def _iter_insert_statements(file_obj: IO[str]) -> Iterator[str]: """Yield complete SQL statements using buffered string find. - Reads the file in 512 KB chunks, accumulates pending text, and uses - str.find for ';' and "'" to scan. String state crosses chunk boundaries - via the `in_string` flag. Avoids the per-char Python loop that dominated - the hot path before. + State persists across read() chunks via three indices: + yield_from position in `pending` where the next yielded statement starts + scan_pos position to resume the scanner from (do NOT restart at 0 + across chunks — would re-find the already-processed open `'`) + in_string are we inside a single-quoted string + + The previous version reset the scanner to position 0 on every chunk, + which silently flipped in_string=False as soon as the first `'` in the + new buffer was re-encountered (the title-field open quote of the + article we were already deep inside). Persisting scan_pos fixes it. """ pending = "" in_string = False + yield_from = 0 # start of current statement, in `pending` coords + scan_pos = 0 # next position to scan, in `pending` coords - def flush_complete(): - """Yield as many complete statements as exist in `pending`.""" - nonlocal pending, in_string - i = 0 - last_yield = 0 + while True: + chunk = file_obj.read(1 << 19) + if not chunk: + break + pending += chunk n = len(pending) + + i = scan_pos while i < n: if in_string: - # Find string terminator (handle escapes). + # Find string terminator (handle escape parity). end = i while True: pos = pending.find("'", end) @@ -166,8 +176,10 @@ def _iter_insert_statements(file_obj: IO[str]) -> Iterator[str]: i = n # need more data break bs = 0 + # Don't walk back past the start of the current statement; + # chars before yield_from belong to a previous statement. p = pos - 1 - while p >= 0 and pending[p] == "\\": + while p >= yield_from and pending[p] == "\\": bs += 1 p -= 1 if bs % 2 == 0: @@ -184,23 +196,23 @@ def _iter_insert_statements(file_obj: IO[str]) -> Iterator[str]: i = n break if qt < 0 or (sc >= 0 and sc < qt): - yield pending[last_yield : sc + 1] - last_yield = sc + 1 + yield pending[yield_from : sc + 1] + yield_from = sc + 1 i = sc + 1 else: in_string = True i = qt + 1 - if last_yield > 0: - pending = pending[last_yield:] - while True: - chunk = file_obj.read(1 << 19) - if not chunk: - break - pending += chunk - yield from flush_complete() - if pending.strip(): - yield pending + scan_pos = i + # Release yielded bytes; remap our two indices into the new buffer. + if yield_from > 0: + pending = pending[yield_from:] + scan_pos -= yield_from + yield_from = 0 + + tail = pending[yield_from:] + if tail.strip(): + yield tail def _extract_wikilinks(text: str, base_uri: str) -> list[Edge]: