From d36cdea5de3021cbd6a46cabb0fa506e63d56bad Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 1 Jun 2026 19:39:41 -0400 Subject: [PATCH] session: gate retrieval keywords on anaphora, scope to parent only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related fixes for fox-flagged "the convo gets narrowed": 1. Anaphora gate. The retrieval_keywords flow from ancestor cited titles into FTS5 was firing on every turn — even unrelated topic switches. Asking "when was batman comic created?" in a Sonic-heavy session pulled 8 Sonic titles into FTS5 and drowned the Batman article → UNGROUNDED. Now gated on the question containing a pronoun ("it", "him", "her", "they", "this", "that", "one"). For subject-bearing questions, retrieval stands on the question text alone — the LLM-side conversation_history (added 832297f) gives the model pronoun-resolution context, but retrieval doesn't need keywords when the new question already has a subject. 2. Parent-only scoping. When the question IS anaphoric, use ONLY the immediate parent's cited titles — not the union across all ancestors. Pronouns almost always refer to the most recent subject; accumulating across a long Sonic-then-Batman thread let Sonic dominate keyword weight and the model resolved "it" to Sonic instead of Batman. Parent-scoped keywords align with how English antecedent resolution actually works. Live verified (3-turn smoke): Sonic created? → HYBRID, Sonic article cited Batman comic created? → HYBRID, Batman article cited (was UNGROUNDED with bare-conv polluter) who created it? → HYBRID, Bob Kane + Bill Finger via List of Batman creators (was Sonic Naoto Ōshima — wrong topic) 24 session+providence+run-query tests pass. --- arborist/cli.py | 59 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/arborist/cli.py b/arborist/cli.py index 7cc302c..fc88114 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -7524,6 +7524,35 @@ _POINTER_LINE_RE = re.compile( ) _INLINE_POINTER_RE = re.compile(r'\[E\d+(?:\s*\|[^\]]*?)?\]') +# Pronouns / anaphoric markers the session-keyword gate looks for. +# Conservative set — only words whose referent is almost certainly +# in a prior turn. "the" / "a" / "what" are NOT here (those start +# fresh-subject questions like "what is the capital of Paris?"). +_ANAPHORIC_WORDS = frozenset({ + "it", "its", "itself", + "he", "him", "his", "himself", + "she", "her", "hers", "herself", + "they", "them", "their", "theirs", "themselves", + "this", "that", "these", "those", + "one", "ones", # "which one was first?" +}) +_WORD_RE = re.compile(r"[A-Za-z']+") + + +def _question_needs_anaphor_keywords(question: str) -> bool: + """True iff the question contains a pronoun / anaphoric word whose + antecedent is presumed to live in the conversation history. + + When True: ancestor cited-titles flow to FTS5 retrieval as keywords. + When False: retrieval stands on the question text alone — prevents + a Sonic-heavy prior conversation from polluting an unrelated + follow-up like "when was batman comic created?". + """ + for m in _WORD_RE.finditer(question or ""): + if m.group(0).lower() in _ANAPHORIC_WORDS: + return True + return False + def _session_strip_pointers(answer_text: str) -> str: """Strip claim-lattice pointer markers from a prior turn's answer @@ -7870,7 +7899,6 @@ def _session_repl(args: argparse.Namespace, sess) -> int: # questions ("who created it?" after "what is # permaculture?"). Folds into conversation_hash so each # distinct chat path gets its own cache_key. - seen_titles: list[str] = [] history: list[dict] = [] # current_bates is the PARENT of the soon-to-be-minted # node, so its path-to-root IS the conversation history. @@ -7880,9 +7908,6 @@ def _session_repl(args: argparse.Namespace, sess) -> int: for n in reversed(ancestors): if n.seq == 0: continue # skip synthetic root - for t in n.cited_titles: - if t and t not in seen_titles: - seen_titles.append(t) history.append({"role": "user", "content": n.question}) # Strip claim-lattice pointer-line excerpts before # passing prior answers to the model — the [E#] bindings @@ -7894,8 +7919,30 @@ def _session_repl(args: argparse.Namespace, sess) -> int: "content": _session_strip_pointers(n.answer_text), }) policy_local = dict(policy) - if seen_titles: - policy_local["retrieval_keywords"] = " ".join(seen_titles[:8]) + # Retrieval keyword flow is GATED on the new question being + # anaphoric — only inject titles when the question itself + # has no subject (pronouns "it", "him", "her", "they", + # "this", "that"). Without the gate, asking "when was batman + # comic created?" in a session whose prior turns are about + # Sonic pulls Sonic-titled keywords into FTS5 and drowns + # the Batman article. The LLM-side conversation_history + # carries antecedent context for pronoun questions anyway; + # keywords are only needed when the question text alone is + # subject-less. + # + # When anaphoric, use ONLY the immediate parent turn's + # cited titles — pronouns almost always refer to the most + # recent subject. Accumulating titles across all ancestors + # lets a long Sonic thread dominate keyword weight when the + # parent is actually about Batman. (2026-06-01 fox-flagged + # conversation narrowing + antecedent drift.) + if _question_needs_anaphor_keywords(line): + parent = sess.get_node(sess.current_bates) + parent_titles = list(parent.cited_titles) if parent else [] + if parent_titles: + policy_local["retrieval_keywords"] = " ".join( + parent_titles[:8] + ) try: result = providence_query(