diff --git a/bench/control_sweep.py b/bench/control_sweep.py index 1e0d04e..ed02372 100644 --- a/bench/control_sweep.py +++ b/bench/control_sweep.py @@ -2,12 +2,13 @@ """#000057 — control-arm characterization sweep (fox: "we both do not know which framing is right, measure all benchmarks and bring the results forward for review"; "we have qwen with and without reasoning -to use"). +to use"; "make the n huge huge and check in every ~7 turns"). -This is NOT a verdict generator. It is the wide sweep the bench-maxing -discipline demands when a config/framing decision is genuinely open: -sweep model × question-framing, log everything, present the SAME judge -verdicts under BOTH narrative framings, and hand the table to review. +This is NOT a verdict generator. It is the wide, parallel, incremental +sweep the bench-maxing discipline demands when a config/framing +decision is genuinely open: sweep model × question-framing at HUGE N, +log every item the instant it lands, present the SAME judge verdicts +under BOTH narrative framings, and hand the table to review. WHY this sweep (the scientific question fox is probing): Is "Hermes-solo confidently fabricates the current officeholder" a @@ -17,58 +18,57 @@ WHY this sweep (the scientific question fox is probing): {Hermes-3-8B, Qwen3.6-27B reasoning, Qwen3.6-27B no-reasoning} × {plain, source-relative, as-of-corpus-era} questions, judged vs the FIXED corpus-vintage gold, decides it empirically: - * if Qwen-27B-reasoning honestly ABSTAINS ("I cannot verify the - current officeholder") where Hermes-8B fabricates → the gap is - largely a weak-small-model artefact; the Arborist value claim - weakens and review must hear that; - * if even Qwen-27B-reasoning confidently asserts a post-corpus - answer → the gap is real and scale-independent (no bare - parametric model can be source-faithful) — the necessary- - substrate claim strengthens; - * the `as_of_corpus` variant ("As of 2010, who was…") separates - "model can't recall the corpus era" from "model won't constrain - to a source" — the exact ambiguity fox flagged. + * if Qwen-27B-reasoning honestly ABSTAINS where Hermes-8B + fabricates → the gap is largely a weak-small-model artefact; + * if even Qwen-27B-reasoning confidently asserts post-corpus → + the gap is real and scale-independent; + * `as_of_corpus` ("As of 2010, who was…") separates "can't recall + the corpus era" from "won't constrain to a source" — the exact + ambiguity fox flagged. (Corpus ~2010-2011, verified from the + artefact — see mine_questions.py — NOT the 2003 dump CLAUDE.md + names.) + +DESIGN for huge N: + * the huge N goes on the CONTROL (solo × 3 models × 3 framings) — + that is where fox's open statistical question lives. The Arborist + treatment arm is a FIXED smaller A/B reference (--arborist-n, + default 40): re-measuring Arborist 386× does not buy power on the + control question, and it is the heaviest call (retrieval over + ~40GB shards + proof path). Power belongs where the question is. + * parallel: independent hermetic judge calls + independent solo HTTP + calls fanned across a bounded worker pool (CLAUDE.md bench-maxing: + "serial-by-caution is halting in disguise"). Work unit = one + fixture item (all its variants/models) so each Arborist call gets + a per-unit qa_db — zero SQLite write contention; shards are + read-only and safe under concurrent readers. + * incremental: every record is appended+flushed the instant it + lands. `--report-only PATH.jsonl` aggregates a PARTIAL file with + NO LLM/judge spend — that is how the loop checks in every ~7 + turns while the sweep is still running. FRAMINGS, measured not chosen (all three from the AskUserQuestion): - * accuracy — WRONG+FABRICATED read as "model is wrong" - (the naive read fox correctly called unfair as a - *truth* claim — included precisely so review can - see why it misleads); - * grounding — the identical verdicts relabelled: WRONG/ - FABRICATED = "ungrounded confident assertion", - ABSTAINED = "honest about ungroundedness" (good), - CORRECT_GROUNDED = "matched the designated - source". Deterministic relabelling — zero extra - judge spend. - * faithfulness — judge each answer vs the context IT was given - (Arborist-arm-only, different judge call). Ticket - §4b scopes this v2/ext-ii; NOT run here (it is a - different instrument, not a relabelling) — listed - so the omission is explicit, not silent. + accuracy / grounding = two deterministic relabellings of the SAME + judge verdicts (zero extra spend). faithfulness-ablation = a + different instrument (judge vs the context each arm was given, + Arborist-only); ticket §4b scopes it v2/ext-ii — NOT run here, so + the omission is explicit not silent. Arborist×Qwen needs proof-path + guided_json+extra_body surgery — the coupled follow-up, also not + done in a measurement harness. -Arborist arm: the Hermes treatment reference is run alongside for -direct same-N comparison. Arborist×Qwen needs a guided_json+extra_body -merge inside the proof path (runner.py) — a distinct, careful piece of -work; flagged in the report as the next step, deliberately NOT done -here (no proof-path surgery in a measurement harness). - -Gate: judge-self-test must be 4/4 before any cell is trusted (run -once up front here, not per-cell — the gate, not 9× the gate). - -Spend is bounded and printed up front; N is a parameter so review can -scale it. Hermetic Opus judge, blinded, fixed gold — same hygiene as -control_ab.py (whose helpers this reuses; DRY). +Gate: judge-self-test 4/4 in-script, once, before any cell (aborts on +fail). `--report-only` skips it (read-only aggregation, no judge). """ from __future__ import annotations import argparse -import hashlib import json import os import re import sys +import threading import time from collections import Counter +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -76,14 +76,6 @@ sys.path.insert(0, str(ROOT)) from bench.control_ab import _SOLO_SYS, _descaffold, _gold # noqa: E402 from bench.judge import judge, self_test # noqa: E402 -# Model arms. `extra` merges into the chat payload root (the client -# forwards it); unrecognised keys are silently dropped server-side. -# qwen-think keeps reasoning ON (Qwen3.6 puts the clean answer in -# message.content and the chain-of-thought in a separate -# reasoning_content field the client does not surface — logged as a -# known limitation); its token budget is raised so the answer survives -# after the thinking span. qwen-nothink disables thinking via the -# chat-template kwarg probed live on the deployment. MODELS: dict[str, dict] = { "hermes": dict( endpoint="https://hermes.ai.unturf.com/v1", @@ -110,22 +102,10 @@ def _v_plain(q: str) -> str: def _v_source_relative(q: str) -> str: - # The FAIR framing: explicitly ask for a source-grounded answer. - # A bare model has no such source — honoring this means abstaining - # ("I have no reference knowledge base"); still asserting a - # remembered name is ungrounded-confident-assertion, not an unfair - # "you're out of date" penalty. Both arms get the identical text. return "According to the reference knowledge base, " + _lower_first(q) def _v_as_of_corpus(q: str) -> str: - # Separates "can't recall the corpus era" from "won't constrain to - # a source": names the snapshot explicitly. If the model CAN say - # the 2010 holder when asked "as of 2010", the gap is snapshot - # ambiguity; if it still can't, parametric memory genuinely cannot - # serve the corpus era. (Corpus vintage ~2010-2011, verified from - # the artefact — see mine_questions.py; NOT the 2003 dump CLAUDE.md - # names.) q2 = re.sub(r"^who is\b", "As of 2010, who was", q, flags=re.I) return q2 if q2 != q else f"As of 2010: {q}" @@ -137,144 +117,98 @@ VARIANTS = { } -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--fixture", - default="bench/qa_questions_stale_map.json") - ap.add_argument("--n", type=int, default=3) - ap.add_argument("--shards-dir", - default=str(Path.home() / ".arborist" / "shards")) - ap.add_argument("--models", default="hermes,qwen-think,qwen-nothink") - ap.add_argument("--variants", - default="plain,source_relative,as_of_corpus") - ap.add_argument("--arborist-ref", default="hermes", - help="model key for the Arborist treatment " - "reference column (proof path = Hermes; " - "Arborist×Qwen is the flagged follow-up)") - ap.add_argument("--out-dir", default="bench/qa_results") - ap.add_argument("--skip-self-test", action="store_true", - help="DANGER: bypasses the instrument gate; only " - "for offline harness development") - a = ap.parse_args() - - models = [m for m in a.models.split(",") if m in MODELS] - variants = [v for v in a.variants.split(",") if v in VARIANTS] - items = json.loads(Path(a.fixture).read_text())[:a.n] - - n_solo = len(models) * len(variants) * len(items) - n_arb = len(variants) * len(items) - print(f"#000057 control sweep — fixture={a.fixture} N={len(items)}") - print(f" models={models} variants={variants}") - print(f" bounded spend: {n_solo} solo LLM + {n_arb} Arborist " - f"query + ~{n_solo + n_arb} claude -p judge + 4 self-test") - - # The instrument gate — once, up front. Not 9× the gate. - if a.skip_self_test: - print(" !! self-test SKIPPED (--skip-self-test) — verdicts " - "are UNTRUSTED") - else: - print(" running judge self-test (instrument gate) …") - if self_test() != 0: - print("ABORT: judge unreliable — no cell may be trusted") - return 1 +# ---------------------------------------------------------------- work +def _process_item(idx: int, it: dict, variants: list[str], + models: list[str], shards_dir: Path, + arborist_on: bool, arborist_ref: str, + ts: str) -> list[dict]: + """All variants × models for ONE fixture item. Self-contained: + its own qa_db, its own clients — safe to run concurrently.""" from arborist.qa.client import OpenAICompatibleClient from arborist.qa.query import DEFAULT_QUERY_POLICY, query - ts = time.strftime("%Y-%m-%dT%H-%M-%SZ", time.gmtime()) - outp = Path(a.out_dir) / f"control_sweep_{ts}.jsonl" - outp.parent.mkdir(parents=True, exist_ok=True) - shards_dir = Path(a.shards_dir) - qa_db = Path("/tmp") / f"control_sweep_{ts}.db" + out: list[dict] = [] + q0 = it["question"] + gold = _gold(shards_dir, it.get("shard", ""), it["target_root"]) + if not gold: + return [{"arm": "skip", "i": idx, "question_orig": q0, + "reason": "no gold"}] clients: dict[str, OpenAICompatibleClient] = {} - def client_for(mkey: str) -> OpenAICompatibleClient: + def cl(mkey: str) -> OpenAICompatibleClient: if mkey not in clients: clients[mkey] = OpenAICompatibleClient( base_url=MODELS[mkey]["endpoint"], api_key=os.environ.get("ARBORIST_LLM_API_KEY")) return clients[mkey] - # tally[(arm_label, variant)] -> Counter(verdict) - tally: dict[tuple, Counter] = {} - log = open(outp, "w") - - def record(arm, mkey, variant, i, q_orig, q_asked, answer, - verdict, judge_raw, audit_mode=None, eff=None): - key = (arm if arm == "solo" else f"arborist", mkey, variant) - tally.setdefault(key, Counter())[eff or verdict] += 1 - rec = {"arm": arm, "model": mkey, "variant": variant, "i": i, - "question_orig": q_orig, "question_asked": q_asked, - "answer": (answer or "")[:1500], "verdict": verdict, - "verdict_eff": eff or verdict, - "arb_audit_mode": audit_mode, - "judge_raw": (judge_raw or "")[:700]} - log.write(json.dumps(rec, ensure_ascii=False) + "\n") - log.flush() - - for i, it in enumerate(items, 1): - q0 = it["question"] - gold = _gold(shards_dir, it.get("shard", ""), it["target_root"]) - if not gold: - print(f" [{i}] SKIP (no gold): {q0[:50]}") - continue + qa_db = Path("/tmp") / f"control_sweep_{ts}_{idx}.db" + try: for variant in variants: q_asked = VARIANTS[variant](q0) - # ---- control arms: each model, solo ---- for mkey in models: cfg = MODELS[mkey] - cl = client_for(mkey) try: - ans = cl.chat_completion( + ans = cl(mkey).chat_completion( [{"role": "system", "content": _SOLO_SYS}, {"role": "user", "content": q_asked}], - model=cfg["model"], max_tokens=cfg["max_tokens"], + model=cfg["model"], + max_tokens=cfg["max_tokens"], extra_body=cfg["extra"]) except Exception as e: # noqa: BLE001 ans = f"[solo-error: {type(e).__name__}: {e}]" v = judge(q_asked, _descaffold(ans), gold) - record("solo", mkey, variant, i, q0, q_asked, ans, - v.label, v.raw) - print(f" [{i}] solo/{mkey}/{variant} -> {v.label}") - # ---- treatment reference: Arborist on the proof-path - # model (Hermes) ---- - ak = a.arborist_ref + out.append({"arm": "solo", "model": mkey, + "variant": variant, "i": idx, + "question_orig": q0, + "question_asked": q_asked, + "answer": (ans or "")[:1500], + "verdict": v.label, + "verdict_eff": v.label, + "arb_audit_mode": None, + "judge_raw": (v.raw or "")[:700]}) + if arborist_on: + try: + r = query(question=q_asked, qa_db=qa_db, + chat_client=cl(arborist_ref), + model_id=MODELS[arborist_ref]["model"], + shards_dir=shards_dir, + policy=dict(DEFAULT_QUERY_POLICY, + answer_mode="claim_lattice")) + arb_raw = (r.get("raw_answer") + or r.get("answer_text") or "") + arb_mode = r.get("audit_mode") + except Exception as e: # noqa: BLE001 + arb_raw = f"[arborist-error: {type(e).__name__}: {e}]" + arb_mode = "ERROR" + va = judge(q_asked, _descaffold(arb_raw), gold) + eff = ("ABSTAINED" if arb_mode == "UNGROUNDED" + and va.label in ("WRONG", "FABRICATED") + else va.label) + out.append({"arm": "arborist", "model": arborist_ref, + "variant": variant, "i": idx, + "question_orig": q0, + "question_asked": q_asked, + "answer": (arb_raw or "")[:1500], + "verdict": va.label, "verdict_eff": eff, + "arb_audit_mode": arb_mode, + "judge_raw": (va.raw or "")[:700]}) + finally: + for c in clients.values(): try: - r = query(question=q_asked, qa_db=qa_db, - chat_client=client_for(ak), - model_id=MODELS[ak]["model"], - shards_dir=shards_dir, - policy=dict(DEFAULT_QUERY_POLICY, - answer_mode="claim_lattice")) - arb_raw = r.get("raw_answer") or r.get("answer_text") or "" - arb_mode = r.get("audit_mode") - except Exception as e: # noqa: BLE001 - arb_raw = f"[arborist-error: {type(e).__name__}: {e}]" - arb_mode = "ERROR" - va = judge(q_asked, _descaffold(arb_raw), gold) - eff = ("ABSTAINED" if arb_mode == "UNGROUNDED" - and va.label in ("WRONG", "FABRICATED") - else va.label) - record("arborist", ak, variant, i, q0, q_asked, arb_raw, - va.label, va.raw, audit_mode=arb_mode, eff=eff) - print(f" [{i}] arborist/{ak}/{variant} -> {va.label}" - f" (mode={arb_mode}, eff={eff})") - - log.close() - for c in clients.values(): + c.close() + except Exception: # noqa: BLE001 + pass try: - c.close() + qa_db.unlink(missing_ok=True) except Exception: # noqa: BLE001 pass + return out - md = _report(tally, items, models, variants, a, outp, ts) - mdp = Path(a.out_dir) / f"control_sweep_{ts}.md" - mdp.write_text(md) - print("\n" + md) - print(f"\nreview deliverables:\n {outp}\n {mdp}") - return 0 +# -------------------------------------------------------------- report def _bucket(c: Counter) -> str: tot = sum(c.values()) or 1 @@ -283,18 +217,36 @@ def _bucket(c: Counter) -> str: f"JE={c['JUDGE_ERROR']} (n={tot})") -def _report(tally, items, models, variants, a, outp, ts) -> str: - n = len(items) - L = [f"# #000057 control sweep — {ts}", +def _aggregate(jsonl_path: Path): + tally: dict[tuple, Counter] = {} + seen_items: set = set() + n_recs = 0 + for ln in jsonl_path.read_text().splitlines(): + if not ln.strip(): + continue + r = json.loads(ln) + if r.get("arm") == "skip": + continue + n_recs += 1 + seen_items.add(r["i"]) + key = (r["arm"], r["model"], r["variant"]) + tally.setdefault(key, Counter())[r["verdict_eff"]] += 1 + return tally, len(seen_items), n_recs + + +def _report(tally, n_items, n_recs, args, jsonl_path, ts, + done: bool) -> str: + L = [f"# #000057 control sweep — {ts}" + f"{'' if done else ' (INTERIM — sweep still running)'}", "", - f"Fixture `{a.fixture}` · N={n} · models={models} · " - f"variants={variants}", - f"Raw per-item JSONL: `{outp}`", + f"Fixture `{args.fixture}` · items scored so far={n_items} · " + f"records={n_recs}", + f"Raw per-item JSONL: `{jsonl_path}`", "", - "**This is review input, not a verdict.** fox: measure all " + "**Review input, not a verdict.** fox: measure all " "benchmarks, bring forward for review.", "", - "Verdict codes: CG=CORRECT_GROUNDED W=WRONG F=FABRICATED " + "Codes: CG=CORRECT_GROUNDED W=WRONG F=FABRICATED " "A=ABSTAINED JE=JUDGE_ERROR. Gold = fixed corpus-vintage " "article (~2010-2011, verified from the artefact, NOT the " "2003 dump CLAUDE.md names). Judge = hermetic blinded Opus.", @@ -305,43 +257,140 @@ def _report(tally, items, models, variants, a, outp, ts) -> str: "|-----|-------|---------|----------|"] for (arm, mkey, variant), c in sorted(tally.items()): L.append(f"| {arm} | {mkey} | {variant} | {_bucket(c)} |") + # Framing B headline numbers, computed deterministically. L += ["", - "## Framing A — accuracy (the naive read; shown so review " - "sees why it misleads)", - "", - "WRONG+FABRICATED counted as \"model is wrong\". This " - "penalises a model for knowing post-corpus reality " - "(\"Macron\" is world-true); fox correctly flagged it is " - "NOT a fair *truth* claim. Reported only for contrast.", - "", "## Framing B — grounding-fidelity (the defensible read)", "", - "Identical verdicts, relabelled: CG = matched the " - "designated source; A = honest about ungroundedness " - "(GOOD — the capability bare models lack); W+F = " - "ungrounded confident assertion (the failure that is " - "structurally impossible to avoid without a retrieval+" - "verify substrate). The headline question for review: does " - "a 27B *reasoning* control ABSTAIN where the 8B fabricates?", + "A = honest about ungroundedness (GOOD); W+F = ungrounded " + "confident assertion; CG = matched the designated source. " + "Solo abstention-rate per (model,variant):", + ""] + for (arm, mkey, variant), c in sorted(tally.items()): + if arm != "solo": + continue + tot = sum(c.values()) or 1 + a = c["ABSTAINED"] + wf = c["WRONG"] + c["FABRICATED"] + L.append(f"- `{mkey}/{variant}`: abstain " + f"{a}/{tot} ({a/tot:.0%}) · ungrounded-assert " + f"{wf}/{tot} ({wf/tot:.0%})") + L += ["", + "## Framing A — accuracy (shown only so review sees why it " + "misleads: it penalises a model for knowing post-corpus " + "reality; \"Macron\" is world-true).", "", - "## Framing C — faithfulness ablation (NOT run here)", - "", - "Judge each answer vs the context IT was given (Arborist-" - "arm-only, a different judge call, not a relabelling). " - "Ticket §4b scopes it v2/ext-ii. Listed so the omission is " - "explicit. Arborist×Qwen (proof-path guided_json+extra_body " - "merge) is the coupled follow-up — also deliberately not " - "done in a measurement harness.", - "", - "## Open for fox / review", - "", - "- bump `--n` (smallest-proof N here; denominators tiny);", - "- Arborist×Qwen arm (proof-path surgery — separate);", - "- which framing becomes the whitepaper headline;", - "- Qwen reasoning_content is not logged (client doesn't " - "surface it) — add if review wants the CoT audited."] + "## Not run here (explicit, not silent)", + "- faithfulness-ablation (judge vs each arm's own context) " + "— different instrument, ticket §4b v2/ext-ii;", + "- Arborist×Qwen — proof-path guided_json+extra_body merge, " + "coupled follow-up.", + ""] return "\n".join(L) + "\n" +def _emit_report(jsonl_path: Path, args, ts, done: bool) -> Path: + tally, n_items, n_recs = _aggregate(jsonl_path) + md = _report(tally, n_items, n_recs, args, jsonl_path, ts, done) + mdp = jsonl_path.with_suffix(".md") + mdp.write_text(md) + print(md) + return mdp + + +# ----------------------------------------------------------------- main + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--fixture", + default="bench/qa_questions_stale_map.json") + ap.add_argument("--n", type=int, default=386, + help="solo (control) item count — the HUGE axis") + ap.add_argument("--arborist-n", type=int, default=40, + help="Arborist A/B reference item count (fixed " + "small — power belongs on the control)") + ap.add_argument("--max-workers", type=int, default=6) + ap.add_argument("--shards-dir", + default=str(Path.home() / ".arborist" / "shards")) + ap.add_argument("--models", default="hermes,qwen-think,qwen-nothink") + ap.add_argument("--variants", + default="plain,source_relative,as_of_corpus") + ap.add_argument("--arborist-ref", default="hermes") + ap.add_argument("--out-dir", default="bench/qa_results") + ap.add_argument("--report-only", default="", + help="aggregate a (partial) JSONL with NO spend " + "and exit — the interim check-in path") + ap.add_argument("--skip-self-test", action="store_true") + a = ap.parse_args() + + if a.report_only: + jp = Path(a.report_only) + ts = jp.stem.replace("control_sweep_", "") + done = not any( # crude: running if a sweep proc still alive + True for _ in []) + _emit_report(jp, a, ts, done=False) + return 0 + + models = [m for m in a.models.split(",") if m in MODELS] + variants = [v for v in a.variants.split(",") if v in VARIANTS] + items = json.loads(Path(a.fixture).read_text())[:a.n] + + n_solo = len(models) * len(variants) * len(items) + arb_units = min(a.arborist_n, len(items)) + n_arb = len(variants) * arb_units + print(f"#000057 control sweep — fixture={a.fixture}") + print(f" solo N={len(items)} arborist-ref N={arb_units} " + f"models={models} variants={variants} " + f"workers={a.max_workers}") + print(f" bounded spend: {n_solo} solo + {n_arb} Arborist + " + f"~{n_solo + n_arb} claude -p judge + 4 self-test") + + if a.skip_self_test: + print(" !! self-test SKIPPED — verdicts UNTRUSTED") + else: + print(" judge self-test (instrument gate) …") + if self_test() != 0: + print("ABORT: judge unreliable") + return 1 + + ts = time.strftime("%Y-%m-%dT%H-%M-%SZ", time.gmtime()) + outp = Path(a.out_dir) / f"control_sweep_{ts}.jsonl" + outp.parent.mkdir(parents=True, exist_ok=True) + shards_dir = Path(a.shards_dir) + lock = threading.Lock() + t0 = time.time() + done_units = 0 + + with open(outp, "w") as log, ThreadPoolExecutor( + max_workers=a.max_workers) as ex: + futs = { + ex.submit(_process_item, i, it, variants, models, + shards_dir, i <= arb_units, a.arborist_ref, + ts): i + for i, it in enumerate(items, 1) + } + for fut in as_completed(futs): + i = futs[fut] + try: + recs = fut.result() + except Exception as e: # noqa: BLE001 + recs = [{"arm": "skip", "i": i, + "reason": f"unit-crash {type(e).__name__}: {e}"}] + with lock: + for rec in recs: + log.write(json.dumps(rec, ensure_ascii=False) + "\n") + log.flush() + done_units += 1 + el = time.time() - t0 + rate = done_units / el if el else 0 + eta = (len(items) - done_units) / rate / 60 if rate else 0 + print(f" unit {done_units}/{len(items)} (item {i}) " + f"· {el/60:.1f}m elapsed · ETA {eta:.0f}m") + + print(f"\nsweep complete in {(time.time()-t0)/60:.1f}m") + mdp = _emit_report(outp, a, ts, done=True) + print(f"review deliverables:\n {outp}\n {mdp}") + return 0 + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/bench/qa_questions_stale.txt b/bench/qa_questions_stale.txt index 19cae51..accb5ac 100644 --- a/bench/qa_questions_stale.txt +++ b/bench/qa_questions_stale.txt @@ -41,3 +41,349 @@ who is the governor-general of Fiji? who is the premier of the Northwest Territories? who is the president of Suriname? who is the president of Cuba? +who is the president of Portugal? +who is the prime minister of the Faroe Islands? +who is the prime minister of Angola? +who is the prime minister of Turkey? +who is the prime minister of Afghanistan? +who is the prime minister of Yemen? +who is the prime minister of Tuvalu? +who is the prime minister of Yemen Arab Republic? +who is the governor-general of Tuvalu? +who is the prime minister of Zambia? +who is the president of Serbia? +who is the president of Serbia and Montenegro? +who is the chancellor of Justice? +who is the president of Spain? +who is the prime minister of Iran? +who is the prime minister of Uzbekistan? +who is the premier of the Cayman Islands? +who is the chancellor of Uppsala University? +who is the prime minister of Kosovo? +who is the president of Hawaii? +who is the president of Mongolia? +who is the prime minister of Mauritius? +who is the prime minister of Somalia? +who is the president of the European Central Bank? +who is the prime minister of the Palestinian National Authority? +who is the president of the Indian National Congress? +who is the president of the Sahrawi Arab Democratic Republic? +who is the prime minister of the Philippines? +who is the president of the Maldives? +who is the chief minister of the Isle of Man? +who is the premier of the Northern Cape? +who is the prime minister of Azerbaijan? +who is the prime minister of Cambodia? +who is the prime minister of Kazakhstan? +who is the governor-general of the West Indies Federation? +who is the premier of the Turks and Caicos Islands? +who is the president of Palau? +who is the prime minister of Solomon Islands? +who is the president of Abkhazia? +who is the chancellor of the Exchequer of Ireland? +who is the prime minister of Yugoslavia? +who is the president of the Canary Islands? +who is the president of the University of Southern California? +who is the governor-general of the Federation of Rhodesia and Nyasaland? +who is the prime minister of Zimbabwe? +who is the president of the Chechen Republic? +who is the prime minister of Crimea? +who is the prime minister of Saudi Arabia? +who is the president of Chile? +who is the prime minister of China? +who is the president of French Polynesia? +who is the president of the Associação Brasileira de Estatística? +who is the prime minister of Serbia and Montenegro? +who is the president of the Federation of Bosnia and Herzegovina? +who is the chief minister of Singapore? +who is the president of the University of Richmond? +who is the chief minister of Sindh? +who is the president of the Autonomous Region of Bougainville? +who is the prime minister of Seychelles? +who is the president of Ingushetia? +who is the president of the Komi Republic? +who is the president of the Sakha Republic? +who is the president of the Republic of Buryatia? +who is the president of Dagestan? +who is the president of Karachay-Cherkessia? +who is the prime minister of the West Indies Federation? +who is the president of the National Congress of Honduras? +who is the president of Lombardy? +who is the prime minister of Iraqi Kurdistan? +who is the chancellor of the University of Cape Town? +who is the prime minister of Lower Saxony? +who is the governor-general of Australia? +who is the president of the United States? +who is the prime minister of India? +who is the premier of Western Australia? +who is the prime minister of Japan? +who is the president of Poland? +who is the chancellor of the Exchequer? +who is the president of the People's Republic of China? +who is the prime minister of Northern Ireland? +who is the prime minister of France? +who is the prime minister of the United States? +who is the president of the Republic? +who is the governor-general of India? +who is the first minister of Scotland? +who is the prime minister of Greece? +who is the prime minister of Italy? +who is the president of Nauru? +who is the president of Croatia? +who is the premier of Ontario? +who is the president of Malta? +who is the president of Georgia? +who is the president of Ghana? +who is the president of Bangladesh? +who is the prime minister of Vietnam? +who is the prime minister of Mongolia? +who is the president of Uganda? +who is the prime minister of Antigua and Barbuda? +who is the governor-general of Belize? +who is the prime minister of Lithuania? +who is the premier of Manitoba? +who is the premier of Prince Edward Island? +who is the prime minister of Fiji? +who is the prime minister of Armenia? +who is the governor-general of the Philippines? +who is the prime minister of Burma? +who is the president of Haiti? +who is the president of North Korea? +who is the president of Rhodesia? +who is the prime minister of Dominica? +who is the prime minister of the Sahrawi Arab Democratic Republic? +who is the prime minister of Cameroon? +who is the president of the Government? +who is the president of Harvard University? +who is the prime minister of Estonia? +who is the president of Azerbaijan? +who is the chancellor of Florence? +who is the prime minister of Egypt? +who is the president of Moldova? +who is the president of the Palestinian National Authority? +who is the president of the German Bundesrat? +who is the president of Kyrgyzstan? +who is the prime minister of Kyrgyzstan? +who is the president of Zanzibar? +who is the prime minister of Rwanda? +who is the president of Turkmenistan? +who is the president of Belarus? +who is the prime minister of Ghana? +who is the president of the Institutional Revolutionary Party? +who is the president of Italy? +who is the president of Kiribati? +who is the prime minister of Samoa? +who is the president of Nagorno-Karabakh Republic? +who is the president of Uzbekistan? +who is the president of the RSUA? +who is the chancellor of the University of St Andrews? +who is the president of Northern Cyprus? +who is the chief minister of Khyber Pakhtunkhwa? +who is the president of the United Arab Emirates? +who is the president of Burma? +who is the president of Madagascar's Small Grants Scheme? +who is the president of Savannah State University? +who is the chancellor of Keele University? +who is the prime minister of the Republic of Adygea? +who is the prime minister of Zimbabwe Rhodesia? +who is the premier of Niue? +who is the president of Veneto? +who is the president of Grinnell College? +who is the prime minister of South Ossetia? +who is the premier of Dominica? +who is the president of the Dewan Negara? +who is the chief minister of Sri Lanka? +who is the chancellor of the University of Glasgow? +who is the president of Bashkortostan? +who is the president of the Marshall Islands? +who is the president of Iraqi Kurdistan? +who is the prime minister of Slovenia? +who is the prime minister of Curaçao? +who is the prime minister of Sint Maarten? +who is the chief minister of the Northern Territory? +who is the premier of Victoria? +who is the president of Ireland? +who is the president of South Korea? +who is the president of the Irish Republic? +who is the president of Israel? +who is the prime minister of Iceland? +who is the president of Tajikistan? +who is the president of Bolivia? +who is the prime minister of Belgium? +who is the prime minister of Finland? +who is the president of Iran? +who is the president of Russia? +who is the president of the Philippines? +who is the prime minister of Thailand? +who is the prime minister of the Netherlands? +who is the president of Nicaragua? +who is the prime minister of Jamaica? +who is the president of South Africa? +who is the president of Sri Lanka? +who is the premier of China? +who is the president of Sierra Leone? +who is the prime minister of Pakistan? +who is the president of Fiji? +who is the premier of Nunavut? +who is the prime minister of Latvia? +who is the president of Trinidad and Tobago? +who is the prime minister of Georgia? +who is the prime minister of South Korea? +who is the president of Madrid? +who is the president of the Confederate States of America? +who is the prime minister of Rhodesia? +who is the president of the Federated States of Micronesia? +who is the president of the European Parliament? +who is the president of Kosovo? +who is the governor-general of Taiwan? +who is the president of East Timor? +who is the prime minister of East Timor? +who is the president of South Ossetia? +who is the president of Princeton University? +who is the president of Vanuatu? +who is the prime minister of Moldova? +who is the premier of the Eastern Cape? +who is the premier of Mpumalanga? +who is the prime minister of Bahrain? +who is the prime minister of Kenya? +who is the chancellor of the Order of the Thistle? +who is the president of Tynwald? +who is the president of the Municipal Chamber? +who is the prime minister of Brunei? +who is the prime minister of Vanuatu? +who is the president of the Gaelic Athletic Association? +who is the president of Montenegro? +who is the president of the Family Division? +who is the prime minister of Cuba? +who is the president of the Geological Society of London? +who is the premier of Nevis? +who is the prime minister of Tajikistan? +who is the chancellor of the Tang Dynasty? +who is the chancellor of Justice of Finland? +who is the president of Crimea? +who is the president of the Republic of Adygea? +who is the president of the Chamber of Deputies of Romania? +who is the president of Nepal? +who is the president of the Republic of Sierra Leone? +who is the president of the Government of Vojvodina? +who is the president of Ukraine? +who is the chief minister of the Australian Capital Territory? +who is the president of Mexico? +who is the premier of Gauteng? +who is the premier of KwaZulu-Natal? +who is the president of the Parliament of Montenegro? +who is the president of Arellano University? +who is the prime minister of Canada? +who is the prime minister of the United Kingdom? +who is the prime minister of Israel? +who is the president of Germany? +who is the chancellor of Austria? +who is the president of the Republic of China? +who is the prime minister of New Zealand? +who is the governor-general of the Irish Free State? +who is the premier of the People's Republic of China? +who is the president of Turkey? +who is the prime minister of Sweden? +who is the prime minister of Spain? +who is the prime minister of Norway? +who is the president of Argentina? +who is the president of Colombia? +who is the prime minister of Romania? +who is the president of Brazil? +who is the president of Romania? +who is the governor-general of Finland? +who is the prime minister of Papua New Guinea? +who is the prime minister of South Africa? +who is the president of Slovenia? +who is the premier of British Columbia? +who is the president of Egypt? +who is the president of the Continental Congress? +who is the president of Singapore? +who is the prime minister of Sri Lanka? +who is the governor-general of Papua New Guinea? +who is the premier of Quebec? +who is the president of Angola? +who is the governor-general of Kenya? +who is the prime minister of Algeria? +who is the president of Pakistan? +who is the president of Mauritius? +who is the president of El Salvador? +who is the president of Vietnam? +who is the premier of Nova Scotia? +who is the premier of Saskatchewan? +who is the premier of Newfoundland and Labrador? +who is the premier of Yukon? +who is the governor-general of Pakistan? +who is the prime minister of Malaysia? +who is the president of Greece? +who is the president of Nigeria? +who is the prime minister of Abkhazia? +who is the prime minister of Serbia? +who is the president of Paraguay? +who is the governor-general of the Dutch East Indies? +who is the governor-general of Solomon Islands? +who is the prime minister of Uganda? +who is the prime minister of Ukraine? +who is the president of the Republic of Macedonia? +who is the president of Estonia? +who is the prime minister of Namibia? +who is the president of Cyprus? +who is the prime minister of Senegal? +who is the premier of the Kingdom of Viti? +who is the president of Ichkeria? +who is the prime minister of Madagascar? +who is the governor-general of Ceylon? +who is the president of Laos? +who is the prime minister of Tunisia? +who is the prime minister of Aruba? +who is the chief minister of Anguilla? +who is the chief minister of Jersey? +who is the premier of North West? +who is the chief minister of Papua and New Guinea? +who is the prime minister of Laos? +who is the prime minister of Tonga? +who is the chief minister of Montserrat? +who is the prime minister of the Republic of Macedonia? +who is the governor-general of New Zealand? +who is the prime minister of Nagorno-Karabakh Republic? +who is the president of Transnistria? +who is the chief minister of Balochistan? +who is the prime minister of Montenegro? +who is the president of the Legislative Yuan? +who is the president of Bucknell University? +who is the president of the Queen's Bench Division? +who is the chief minister of Guernsey? +who is the prime minister of Northern Cyprus? +who is the president of Grande Comore? +who is the president of Anjouan? +who is the president of Mohéli? +who is the president of the Bundestag? +who is the president of the Moldovan Parliament? +who is the chief minister of Punjab? +who is the president of Indonesia? +who is the president of Cantabria? +who is the president of the University of Michigan? +who is the chancellor of the University of Edinburgh? +who is the prime minister of the Federation of Rhodesia and Nyasaland? +who is the president of the Congress of Colombia? +who is the premier of the Western Cape? +who is the president of Congress? +who is the president of Zimbabwe Rhodesia? +who is the president of the World Esperanto Association? +who is the prime minister of Turkmenistan? +who is the chancellor of Germany? +who is the president of the States of Alderney? +who is the president of the Government of New Caledonia? +who is the chancellor of the University of Dallas? +who is the president of the Naval War College? +who is the president of the University of Southern Maine? +who is the president of Kabardino-Balkaria? +who is the president of the University of the Philippines? +who is the president of Honduras? +who is the premier of the Free State? +who is the premier of Limpopo? +who is the president of Uruguay? +who is the president of the Xunta of Galicia? +who is the president of Umbria? +who is the prime minister of Hyderabad? +who is the premier of Montserrat? diff --git a/bench/qa_questions_stale_map.json b/bench/qa_questions_stale_map.json index 803a33d..6ec6592 100644 --- a/bench/qa_questions_stale_map.json +++ b/bench/qa_questions_stale_map.json @@ -238,5 +238,2081 @@ "target_title": "President of Cuba", "target_root": "98af74f9b96f3f679a4f671f719ad8562c3f70d2416f5c6230485f3cc5faca79", "shard": "000.db" + }, + { + "question": "who is the president of Portugal?", + "target_title": "President of Portugal", + "target_root": "28f7d5baac745ae153e8009b5eef5a3d76a560e61ea5f6c287980abf89955229", + "shard": "000.db" + }, + { + "question": "who is the prime minister of the Faroe Islands?", + "target_title": "Prime Minister of the Faroe Islands", + "target_root": "8c9069eb5b6bbb79dcd1e6754b1f74a442150996c8bb89c864ae8070166e2884", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Angola?", + "target_title": "Prime Minister of Angola", + "target_root": "e79ad135a3c6a31ff2b1de66d25186f53864962b39151808ce1a844289d565bf", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Turkey?", + "target_title": "Prime Minister of Turkey", + "target_root": "8174d01a83c7ac6cb8979bbd572930ee2c1bf1bb9b81509430b6e0569f26ab2a", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Afghanistan?", + "target_title": "Prime Minister of Afghanistan", + "target_root": "30f674bd8dd83e0e8cb547f3bdf341f2cf2db754ee26f575f47bd167de396fbd", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Yemen?", + "target_title": "Prime Minister of Yemen", + "target_root": "d391be990086f85418f642de5f8896742af444194e1dd1eaffc76b433d930dc3", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Tuvalu?", + "target_title": "Prime Minister of Tuvalu", + "target_root": "164bc82ec830ab0446d00e993460804719246df24d44e47ff48fc81703f2ff6a", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Yemen Arab Republic?", + "target_title": "Prime Minister of Yemen Arab Republic", + "target_root": "fa92fea4ce43440713d418354ae806891ae5e14c3236f7d5e09d44cecf85b588", + "shard": "000.db" + }, + { + "question": "who is the governor-general of Tuvalu?", + "target_title": "Governor-General of Tuvalu", + "target_root": "908d2ffdc1b8095eae6f73b3e0a888910fa7725514a38494bb761a825afbcc09", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Zambia?", + "target_title": "Prime Minister of Zambia", + "target_root": "cf3caede827c540b40934c6b04042cbc4d92b1072e440570ba0d5b12ca8df287", + "shard": "000.db" + }, + { + "question": "who is the president of Serbia?", + "target_title": "President of Serbia", + "target_root": "63944014cd3d5e691906c885cf1430845bf4ff4a9165b60e01874269d816c42f", + "shard": "000.db" + }, + { + "question": "who is the president of Serbia and Montenegro?", + "target_title": "President of Serbia and Montenegro", + "target_root": "f764e9d95b1cbd216c11bbca8c3eaf0e8bdb12b231dc1f0d1a5cdaaf0f917126", + "shard": "000.db" + }, + { + "question": "who is the chancellor of Justice?", + "target_title": "Chancellor of Justice", + "target_root": "7a8f7feab6937f2d7bad4b5cb5e2b092540f4ff41d1023e2738e314f31afc0f9", + "shard": "000.db" + }, + { + "question": "who is the president of Spain?", + "target_title": "President of Spain", + "target_root": "4fb675d806a13b811e3dc0ffaf4364ff13155b9e3c5f2a8a31088136513bd11d", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Iran?", + "target_title": "Prime Minister of Iran", + "target_root": "327dc53efa726a73ead535709a629f1ef52c2485d54214aa911d3965999bfc38", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Uzbekistan?", + "target_title": "Prime Minister of Uzbekistan", + "target_root": "6d484fd63e4aaa1cd4ce1dc25c85d72543677f10db3c9b79342ff0f9ce93fe67", + "shard": "000.db" + }, + { + "question": "who is the premier of the Cayman Islands?", + "target_title": "Premier of the Cayman Islands", + "target_root": "a71a5ffa226e02569c07a35e8974a2ed72a6faa1250a807c6e6bccfb2b97dbd0", + "shard": "000.db" + }, + { + "question": "who is the chancellor of Uppsala University?", + "target_title": "Chancellor of Uppsala University", + "target_root": "4af7d4ba8c5799c72ae4f3c3e532f8329104abac3dac507d77c1a4bb15df58f8", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Kosovo?", + "target_title": "Prime Minister of Kosovo", + "target_root": "39dd126cda1b7f0c134939fcd51c871ab1e0c4a71bc3b7f5a67eb6f768290058", + "shard": "000.db" + }, + { + "question": "who is the president of Hawaii?", + "target_title": "President of Hawaii", + "target_root": "7d2d7be34530470f0714714f763310c3ebf23916e6aa7f818e2b1b1cfd5d930b", + "shard": "000.db" + }, + { + "question": "who is the president of Mongolia?", + "target_title": "President of Mongolia", + "target_root": "007b63d9d5427cca8c8266a3efe0baa8dc47c072fe4e1ad95f59dadce169d1ab", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Mauritius?", + "target_title": "Prime Minister of Mauritius", + "target_root": "d0ab4849cc522a394f91c449db4930eb251f306298cd194f7a64822839a38025", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Somalia?", + "target_title": "Prime Minister of Somalia", + "target_root": "97f165183bc020827f3389c4f1c236438474523f705bc6482819365faac9d56e", + "shard": "000.db" + }, + { + "question": "who is the president of the European Central Bank?", + "target_title": "President of the European Central Bank", + "target_root": "cc0ae22622d1a11f0bfd4dd4ab6cfe0e7587e73207a2249c71942aff7c37b987", + "shard": "000.db" + }, + { + "question": "who is the prime minister of the Palestinian National Authority?", + "target_title": "Prime Minister of the Palestinian National Authority", + "target_root": "bf7f72ba11587006ea69a15b368370bfc69fe7e21fdb477f2c4ec46aafaee668", + "shard": "000.db" + }, + { + "question": "who is the president of the Indian National Congress?", + "target_title": "President of the Indian National Congress", + "target_root": "429f2fd4654b6b0ec8c3da0722fcdf8e4325f1c1766dd2b8d7be55a35675e07a", + "shard": "000.db" + }, + { + "question": "who is the president of the Sahrawi Arab Democratic Republic?", + "target_title": "President of the Sahrawi Arab Democratic Republic", + "target_root": "540e38682284f423cc5ff1cfa098f0d5b457a97cd68262454c50c83155269469", + "shard": "000.db" + }, + { + "question": "who is the prime minister of the Philippines?", + "target_title": "Prime Minister of the Philippines", + "target_root": "fe6d87b1ecaaac6e542090deff9677d9f1d06514ff9ae5433499d1366981866f", + "shard": "000.db" + }, + { + "question": "who is the president of the Maldives?", + "target_title": "President of the Maldives", + "target_root": "7dbab3f83ce83d736cb224ac4ef1fb7827ae5212c2ea5ddf72d225d6ca55ef79", + "shard": "000.db" + }, + { + "question": "who is the chief minister of the Isle of Man?", + "target_title": "Chief Minister of the Isle of Man", + "target_root": "3f748eea2f67c67f1cf3d0de466242f66e5a2243562ae453116ea5b8ac09b869", + "shard": "000.db" + }, + { + "question": "who is the premier of the Northern Cape?", + "target_title": "Premier of the Northern Cape", + "target_root": "7a8aa8c734848aa0cfcef611b4fb5cb961e422c56f66c861dc9b76cd861d9ced", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Azerbaijan?", + "target_title": "Prime Minister of Azerbaijan", + "target_root": "16712d3b87fed32814d7e43864f3235c02db257a7f1d375d15c9869c30ef1533", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Cambodia?", + "target_title": "Prime Minister of Cambodia", + "target_root": "e1cbb42a73f6a07fe21a41e9244f26322b24490a42dfffbbb984c2761e527e4d", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Kazakhstan?", + "target_title": "Prime Minister of Kazakhstan", + "target_root": "7d3a0fd87155a8d0f537f65bb52e42d4d8a7cef095345b58058920fdbe18b67f", + "shard": "000.db" + }, + { + "question": "who is the governor-general of the West Indies Federation?", + "target_title": "Governor-General of the West Indies Federation", + "target_root": "e3943906bbc7ad14385697fe34030070396059608380c5b52216286baa744120", + "shard": "000.db" + }, + { + "question": "who is the premier of the Turks and Caicos Islands?", + "target_title": "Premier of the Turks and Caicos Islands", + "target_root": "e01a7ea6c71da5b94b3ba408dfc2ad4611e9ea0c3a0c1e32a26b845b8686fb16", + "shard": "000.db" + }, + { + "question": "who is the president of Palau?", + "target_title": "President of Palau", + "target_root": "833612b0dd3e823fe136adc72c1c5b328bf230578a13560181e18eddd309f949", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Solomon Islands?", + "target_title": "Prime Minister of Solomon Islands", + "target_root": "3024c5753e63de48fe3409cc28a011caea83399f62bbf8dc2b0f1e8a9e29a26f", + "shard": "000.db" + }, + { + "question": "who is the president of Abkhazia?", + "target_title": "President of Abkhazia", + "target_root": "e24b481e215f4d705db7995b5bb7644371db8206fd258acf7154f6b6c7d36b22", + "shard": "000.db" + }, + { + "question": "who is the chancellor of the Exchequer of Ireland?", + "target_title": "Chancellor of the Exchequer of Ireland", + "target_root": "554c1187ac1540cadc5e876d9e309c68d3552540f0a65591f6d12862a5657f1e", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Yugoslavia?", + "target_title": "Prime Minister of Yugoslavia", + "target_root": "27d211151bfba600858beea8a674c6dbda5bcd34e8f7dbe39beb3608158b0fb7", + "shard": "000.db" + }, + { + "question": "who is the president of the Canary Islands?", + "target_title": "President of the Canary Islands", + "target_root": "1d67fc1e1d71bdfa98ad5c9cb990355dfb17187cdff7416bb09594286c895fbf", + "shard": "000.db" + }, + { + "question": "who is the president of the University of Southern California?", + "target_title": "President of the University of Southern California", + "target_root": "b0a7d06f7ff1155cd8b1609703b0c441501f72598056acc451f838c98d580b31", + "shard": "000.db" + }, + { + "question": "who is the governor-general of the Federation of Rhodesia and Nyasaland?", + "target_title": "Governor-General of the Federation of Rhodesia and Nyasaland", + "target_root": "e236fbd96f103b04a59ad1bacf322d377b2dfa61ebbf91cd3c2e03cdb4bb904b", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Zimbabwe?", + "target_title": "Prime Minister of Zimbabwe", + "target_root": "a9014821e24e09ed1e04aba45c8ea4a269a61d9c3538cc919143302595ff4d83", + "shard": "000.db" + }, + { + "question": "who is the president of the Chechen Republic?", + "target_title": "President of the Chechen Republic", + "target_root": "dd8153a2d94828faaaeb55407b61bdb81b9531410a78062a40481eb8cb881631", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Crimea?", + "target_title": "Prime Minister of Crimea", + "target_root": "6ca1349e0159e8fd86964bbec37c809978d8c1f7412ff789b69e25c6b7343c3c", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Saudi Arabia?", + "target_title": "Prime Minister of Saudi Arabia", + "target_root": "00cfc115cf9fd321f1ec2e99c512c43b6af8a2ceee54a6a6061b3327f3258dae", + "shard": "000.db" + }, + { + "question": "who is the president of Chile?", + "target_title": "President of Chile", + "target_root": "fdebd83c353ff2d40b46994420a9b650b99a17332f1a21ef11e53eece52f4d00", + "shard": "000.db" + }, + { + "question": "who is the prime minister of China?", + "target_title": "Prime Minister of China", + "target_root": "91b10caf8c1f594002151270f792dcddc62464700262736c806220954808e03b", + "shard": "000.db" + }, + { + "question": "who is the president of French Polynesia?", + "target_title": "President of French Polynesia", + "target_root": "e793e369d56660a2c0804c6dc14cc94c04c1d148db472d1bc9cf763c92a9ca44", + "shard": "000.db" + }, + { + "question": "who is the president of the Associação Brasileira de Estatística?", + "target_title": "President of the Associação Brasileira de Estatística", + "target_root": "9ffc8e27508aceb05dd5e4a368233303d53cc8b4ad65a947815ef051826ae140", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Serbia and Montenegro?", + "target_title": "Prime Minister of Serbia and Montenegro", + "target_root": "bf9daaa86600d89ef4ff459799be810243228665e3770a57fb693cb6e3348084", + "shard": "000.db" + }, + { + "question": "who is the president of the Federation of Bosnia and Herzegovina?", + "target_title": "President of the Federation of Bosnia and Herzegovina", + "target_root": "8a7df74f6ba00a6a4720d9db00ec074ec508a8fd68e79e670723a6d47126890e", + "shard": "000.db" + }, + { + "question": "who is the chief minister of Singapore?", + "target_title": "Chief Minister of Singapore", + "target_root": "bca32a06d92c461033f51fdf30996477472a305b304f850cf66c06b0b6a3a287", + "shard": "000.db" + }, + { + "question": "who is the president of the University of Richmond?", + "target_title": "President of the University of Richmond", + "target_root": "c1bd8ffba83c4de5da1a4839fbdfe373138f62759fd99474ef06c953acb1d65a", + "shard": "000.db" + }, + { + "question": "who is the chief minister of Sindh?", + "target_title": "Chief Minister of Sindh", + "target_root": "b69ba0d5528c56016995413bca89a44c93004d73c63e3a28261d5a06466471a8", + "shard": "000.db" + }, + { + "question": "who is the president of the Autonomous Region of Bougainville?", + "target_title": "President of the Autonomous Region of Bougainville", + "target_root": "2b5d5ea5eaee733457b1d196cd547aa6db655be62ffa3a941e7a2660fea4436e", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Seychelles?", + "target_title": "Prime Minister of Seychelles", + "target_root": "49183296420e40f60e1fff630b9d0a1d70694a146e39db4961dccca1889e157c", + "shard": "000.db" + }, + { + "question": "who is the president of Ingushetia?", + "target_title": "President of Ingushetia", + "target_root": "8b2b89e35478156a8b0186344dadc0bc156330b9629ac106abf2add30077ccf9", + "shard": "000.db" + }, + { + "question": "who is the president of the Komi Republic?", + "target_title": "President of the Komi Republic", + "target_root": "998c96c9ec26563561467015f5e4f24711f48cbb6edbc5f60faba975601429ad", + "shard": "000.db" + }, + { + "question": "who is the president of the Sakha Republic?", + "target_title": "President of the Sakha Republic", + "target_root": "8af6b233a86876471e18b0aa88137a41670a33e4819ccde27b1b4854ca3e45ab", + "shard": "000.db" + }, + { + "question": "who is the president of the Republic of Buryatia?", + "target_title": "President of the Republic of Buryatia", + "target_root": "04c30c5cc637960d9eb030fc68d989c6df9cb9a05734e67626174e64524b185c", + "shard": "000.db" + }, + { + "question": "who is the president of Dagestan?", + "target_title": "President of Dagestan", + "target_root": "00b3c23a96a13ccc9162b2b69c0ba5999a3b2f556d1c017460bfc9751674b880", + "shard": "000.db" + }, + { + "question": "who is the president of Karachay-Cherkessia?", + "target_title": "President of Karachay-Cherkessia", + "target_root": "1edf55f09d247b35296524d5df0f230c86b573a06ea370e9c9828d2d84a9c36d", + "shard": "000.db" + }, + { + "question": "who is the prime minister of the West Indies Federation?", + "target_title": "Prime Minister of the West Indies Federation", + "target_root": "7095e0939246f065fed2f9c6a913d2f2df25a4a65e8740f5be4ec4229efbb905", + "shard": "000.db" + }, + { + "question": "who is the president of the National Congress of Honduras?", + "target_title": "President of the National Congress of Honduras", + "target_root": "f9b5c298b8fca252107f121e670f860fe361594d516a41756a725600f992e4c7", + "shard": "000.db" + }, + { + "question": "who is the president of Lombardy?", + "target_title": "President of Lombardy", + "target_root": "64705ba8d4d48606c34b34b3377d0c164487f72aed8219fe3a01f6cac0eddae1", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Iraqi Kurdistan?", + "target_title": "Prime Minister of Iraqi Kurdistan", + "target_root": "c186817a0c23e83c5e1db6b42bb276ce96084504a94dad837a1933bc73114bf4", + "shard": "000.db" + }, + { + "question": "who is the chancellor of the University of Cape Town?", + "target_title": "Chancellor of the University of Cape Town", + "target_root": "0a859a12ba119f310ed9debf9d40ae9ed6a7159f46866b28c257ce5dfeebe275", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Lower Saxony?", + "target_title": "Prime Minister of Lower Saxony", + "target_root": "ae18e63b026c730882783abee9e354905c1ab96ccb9b730bc632e4c29f39e050", + "shard": "000.db" + }, + { + "question": "who is the governor-general of Australia?", + "target_title": "Governor-General of Australia", + "target_root": "474e706cdf01428a44316e9f3854681370630da980820e99215c63873f96acfe", + "shard": "001.db" + }, + { + "question": "who is the president of the United States?", + "target_title": "President of the United States", + "target_root": "4908c1242cb0d06558076f652dde341090a26ad414aa77c65207f63c287dcb90", + "shard": "001.db" + }, + { + "question": "who is the prime minister of India?", + "target_title": "Prime Minister of India", + "target_root": "7fd75ee87bc32f0945b27e6fe67b2ee747b9dc1430e095e1ed2445c94ec3ef05", + "shard": "001.db" + }, + { + "question": "who is the premier of Western Australia?", + "target_title": "Premier of Western Australia", + "target_root": "7e888924b33f2405a14083c9cb0148c364d41b2441cd00a5d5c8ece3ec0033f2", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Japan?", + "target_title": "Prime Minister of Japan", + "target_root": "1c41d680b5199de5517c8991204240c6bfd2df88871bf5497012624047ac38b8", + "shard": "001.db" + }, + { + "question": "who is the president of Poland?", + "target_title": "President of Poland", + "target_root": "e71984c722db7c39046046263a0df2ec4b64b56bef5f8ac67f5efeacedf083b6", + "shard": "001.db" + }, + { + "question": "who is the chancellor of the Exchequer?", + "target_title": "Chancellor of the Exchequer", + "target_root": "65fb4aadfa4902c86393563168311646ed2ac636b5b81e5d840142b232cd6f1f", + "shard": "001.db" + }, + { + "question": "who is the president of the People's Republic of China?", + "target_title": "President of the People's Republic of China", + "target_root": "172bf3e5d07f06415d60ea2f1ee00cdac82e6d72cecdae2245180023c8a04f79", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Northern Ireland?", + "target_title": "Prime Minister of Northern Ireland", + "target_root": "531cc5f25f07c13742c42b0af135f5417b3e995485c41f78c2d781809c070bf3", + "shard": "001.db" + }, + { + "question": "who is the prime minister of France?", + "target_title": "Prime Minister of France", + "target_root": "dda76c98f64b2b69ffe508367e647eed01ae85be9e5325d105a34828f4942240", + "shard": "001.db" + }, + { + "question": "who is the prime minister of the United States?", + "target_title": "Prime Minister of the United States", + "target_root": "eee29a269d99744265c29a71ec1e79d418ace5494ef25d6fa538ac097534f49a", + "shard": "001.db" + }, + { + "question": "who is the president of the Republic?", + "target_title": "President of the Republic", + "target_root": "01ee45cbc66c1696a6c36d497e62ae61608b39f0c80f0525b9216d376fc0284d", + "shard": "001.db" + }, + { + "question": "who is the governor-general of India?", + "target_title": "Governor-General of India", + "target_root": "eb298c3c73d4d3f9ad830ee984deac581fae8d3d3d4940b8a826a3ba660e6527", + "shard": "001.db" + }, + { + "question": "who is the first minister of Scotland?", + "target_title": "First Minister of Scotland", + "target_root": "5fba063e9c6f08928d4cad09b102368c34f8ea667777eb58c59a50f01ded4ee2", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Greece?", + "target_title": "Prime Minister of Greece", + "target_root": "c760d8afc62027e53d19832fa562e25cac773b991a7cb36dfb4b8f4f9cb6346f", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Italy?", + "target_title": "Prime Minister of Italy", + "target_root": "a928dacee6ee47927edf8a55d76eb17dc2fc9763a75710ce923a6bbcd568a2f4", + "shard": "001.db" + }, + { + "question": "who is the president of Nauru?", + "target_title": "President of Nauru", + "target_root": "ff63efb24021dd719738c20b92ccc348cba98aba6c1b98e0ece0f8cc277d3244", + "shard": "001.db" + }, + { + "question": "who is the president of Croatia?", + "target_title": "President of Croatia", + "target_root": "bc47acbb2b7592e17d4e8ee05b502171460ef8a0192c4fea66334b6e68547375", + "shard": "001.db" + }, + { + "question": "who is the premier of Ontario?", + "target_title": "Premier of Ontario", + "target_root": "493326da3d268452b18314e302d389ad58efd335eca4f26f84194aef2134b5a1", + "shard": "001.db" + }, + { + "question": "who is the president of Malta?", + "target_title": "President of Malta", + "target_root": "80c9403818330c33f72dda98d032aae72da18692c25df7913af2dfa072e3c26f", + "shard": "001.db" + }, + { + "question": "who is the president of Georgia?", + "target_title": "President of Georgia", + "target_root": "63e1e2e81ee30d4cbb1dbfd55f16c1bbdfbe3f8c805e461b256704125f3776ae", + "shard": "001.db" + }, + { + "question": "who is the president of Ghana?", + "target_title": "President of Ghana", + "target_root": "5d2fa3d7a0cc8c4dd8ffe3b7bd65d978f8561c6172582581826ef3c2c2af5a82", + "shard": "001.db" + }, + { + "question": "who is the president of Bangladesh?", + "target_title": "President of Bangladesh", + "target_root": "b432c0c0da43315518723b6c3ad12975e67f40d1300cb85fe60c6fb5191aabf8", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Vietnam?", + "target_title": "Prime Minister of Vietnam", + "target_root": "909d3fbb0c60c2bbc968fb2e262f05c07af6d6372ed9c1b3cdb63b1727495848", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Mongolia?", + "target_title": "Prime Minister of Mongolia", + "target_root": "2694b2a37964ad6695422071dc5835adc4c79f5b754c7fb9feb2b4cb27159063", + "shard": "001.db" + }, + { + "question": "who is the president of Uganda?", + "target_title": "President of Uganda", + "target_root": "44be0bb35c4d971c54e9ea54549728f402f6263cb073147c9676c64abb466453", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Antigua and Barbuda?", + "target_title": "Prime Minister of Antigua and Barbuda", + "target_root": "ca3c85be1ce3a782ded79f97189934e15f44dc931106e5a74d65034109c69c50", + "shard": "001.db" + }, + { + "question": "who is the governor-general of Belize?", + "target_title": "Governor-General of Belize", + "target_root": "f9f569045a53c953a426f8364168245032c66e6d2a32f5e0dcc10269da88e73c", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Lithuania?", + "target_title": "Prime Minister of Lithuania", + "target_root": "0fe3ee320e4deaf3beac23dfd158f42a29747d9d3b99522cce205a8675459a15", + "shard": "001.db" + }, + { + "question": "who is the premier of Manitoba?", + "target_title": "Premier of Manitoba", + "target_root": "8c7b8c127939f96ff4af40832bf2dcde2d530e34eecd07547f376f6d6ac083d9", + "shard": "001.db" + }, + { + "question": "who is the premier of Prince Edward Island?", + "target_title": "Premier of Prince Edward Island", + "target_root": "4b948e51935556260e7773867989513996274cfd32a983733ea9d7686aa00e8e", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Fiji?", + "target_title": "Prime Minister of Fiji", + "target_root": "f3e995aeb77058e018a4ef32574f8938b10805e6afb7a8e97d87d3088423a5c7", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Armenia?", + "target_title": "Prime Minister of Armenia", + "target_root": "d37bfaaf2799f4b4f06e366283fe3383f25372e48d195e76a094b4233323bdd1", + "shard": "001.db" + }, + { + "question": "who is the governor-general of the Philippines?", + "target_title": "Governor-General of the Philippines", + "target_root": "37576e02cdba4af2578ac551b79a0e2bb23bf25551611ac12d2042e3514b93e3", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Burma?", + "target_title": "Prime Minister of Burma", + "target_root": "22bf19003432f1873c6bf697d56535bd352227483d2c9022d9c2216f0d9533f6", + "shard": "001.db" + }, + { + "question": "who is the president of Haiti?", + "target_title": "President of Haiti", + "target_root": "dbc84e4ffbae10c6223f66c09417a977fe9755c44737294f939268a6b9e09604", + "shard": "001.db" + }, + { + "question": "who is the president of North Korea?", + "target_title": "President of North Korea", + "target_root": "2ef02165731f102ee38833c18249877cd65469ebf6fb6842da129bbc927ba859", + "shard": "001.db" + }, + { + "question": "who is the president of Rhodesia?", + "target_title": "President of Rhodesia", + "target_root": "9e0e3876a3cf96f9a83d0227c40d19d03d664249f7133b10e0b77003b7d61703", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Dominica?", + "target_title": "Prime Minister of Dominica", + "target_root": "73155053b873c55aef0de296da985311d0cb4b1a5e6067807f7d7e926277e99c", + "shard": "001.db" + }, + { + "question": "who is the prime minister of the Sahrawi Arab Democratic Republic?", + "target_title": "Prime Minister of the Sahrawi Arab Democratic Republic", + "target_root": "7574da2d3ed56abccc0a2433162dab09ccc8ddfbbf0cc8d40a7abd7470e63226", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Cameroon?", + "target_title": "Prime Minister of Cameroon", + "target_root": "44f6e537e5bc2458a6354c29ac518f14a619b5976edacf3d4729986c292f5841", + "shard": "001.db" + }, + { + "question": "who is the president of the Government?", + "target_title": "President of the Government", + "target_root": "b576c76162ac2e7937295f1956be2f9a6b93fb9179f4f88383e63a1d14beb4a5", + "shard": "001.db" + }, + { + "question": "who is the president of Harvard University?", + "target_title": "President of Harvard University", + "target_root": "0667f8c69dc484c328c94b81869a0850776d3cbf94929b7e343c25129171c0cc", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Estonia?", + "target_title": "Prime Minister of Estonia", + "target_root": "dc34b5bc7d6059537b779fba3e0c111a44a728abdbb7d5fc60140d03126ba17c", + "shard": "001.db" + }, + { + "question": "who is the president of Azerbaijan?", + "target_title": "President of Azerbaijan", + "target_root": "73bdb87b80c24b6f3d727958c57b6bccc0964a48dd402b67805d969839f91175", + "shard": "001.db" + }, + { + "question": "who is the chancellor of Florence?", + "target_title": "Chancellor of Florence", + "target_root": "1045f5db15f2250371de087cd8fb7cbd84e0b4cd7d54eeadb8e5fc1d6b205ce2", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Egypt?", + "target_title": "Prime Minister of Egypt", + "target_root": "90c449094140c34f03f3f8c6a04eff0bc12418315d5badb362eb6207d2a43266", + "shard": "001.db" + }, + { + "question": "who is the president of Moldova?", + "target_title": "President of Moldova", + "target_root": "d99477c19a38b60cb90497a2e3898f63f90c0c85d0374233389690dc535815d3", + "shard": "001.db" + }, + { + "question": "who is the president of the Palestinian National Authority?", + "target_title": "President of the Palestinian National Authority", + "target_root": "ec1024148425e8b7f3a30d9855161df647e78fc86eb5d18070af044fd6a1b8fd", + "shard": "001.db" + }, + { + "question": "who is the president of the German Bundesrat?", + "target_title": "President of the German Bundesrat", + "target_root": "8421960b77bf8ecd1c5bdf5bf62574ba3233a08fd25b41a33325fe563597ca1b", + "shard": "001.db" + }, + { + "question": "who is the president of Kyrgyzstan?", + "target_title": "President of Kyrgyzstan", + "target_root": "cab64744367c2fbc431f99670eb5ea230788cd230115f9688a1678eadbf420a5", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Kyrgyzstan?", + "target_title": "Prime Minister of Kyrgyzstan", + "target_root": "4c3dc20494f18b7a38017cd3e3299c6f71b97b5afd025d6c41c1bf75151ca78f", + "shard": "001.db" + }, + { + "question": "who is the president of Zanzibar?", + "target_title": "President of Zanzibar", + "target_root": "0d2224099e483fbf14c8192ea81d1ece7b3052bbe930667594a72b199889bd05", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Rwanda?", + "target_title": "Prime Minister of Rwanda", + "target_root": "cbae08fb4e727765d3d940cd86e6233ba29678b1004534b4aedd805535b476be", + "shard": "001.db" + }, + { + "question": "who is the president of Turkmenistan?", + "target_title": "President of Turkmenistan", + "target_root": "932758c769ec79bc45d16ef8856488ef2aed5446b292c7ed00df082f8aca9dcb", + "shard": "001.db" + }, + { + "question": "who is the president of Belarus?", + "target_title": "President of Belarus", + "target_root": "6321333ce71b7bebe789766c33b4fa2be4636564cdf1ea1030bf01649058504c", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Ghana?", + "target_title": "Prime Minister of Ghana", + "target_root": "8c7046f28b538f860a11900cfe9ef89389692e6d6c9d03798bb2db7277effa72", + "shard": "001.db" + }, + { + "question": "who is the president of the Institutional Revolutionary Party?", + "target_title": "President of the Institutional Revolutionary Party", + "target_root": "760a1f7b41b09bee01506602e38ca1c9f86f737f45ca8454f9be803c5a2f8c5b", + "shard": "001.db" + }, + { + "question": "who is the president of Italy?", + "target_title": "President of Italy", + "target_root": "9a3a6e81e81a26779218c8c93590e2d719691bb695dd99f329fe1f33e043bd0f", + "shard": "001.db" + }, + { + "question": "who is the president of Kiribati?", + "target_title": "President of Kiribati", + "target_root": "a71f767a228bb8ed040c03928e5cdc7a85dee72f85c0fe07c8e587cbf34bbdd4", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Samoa?", + "target_title": "Prime Minister of Samoa", + "target_root": "b553a3e4cb9713293b0ebf7cdaa002b02ecbb64e52fbdfe10ad0e5d2bc5f5b6e", + "shard": "001.db" + }, + { + "question": "who is the president of Nagorno-Karabakh Republic?", + "target_title": "President of Nagorno-Karabakh Republic", + "target_root": "f468b24c099de6bb7da4e829fc16bac461cc416f22169090fd97c2eeaf07d93d", + "shard": "001.db" + }, + { + "question": "who is the president of Uzbekistan?", + "target_title": "President of Uzbekistan", + "target_root": "3e5c05c27d669675e2017f457014855b5ad48b422b99c9a6bde3d939f15cbb67", + "shard": "001.db" + }, + { + "question": "who is the president of the RSUA?", + "target_title": "President of the RSUA", + "target_root": "edd80959d78aa795d81424bee995be3ba1eeda59e01e27f6931e2be7565cd55a", + "shard": "001.db" + }, + { + "question": "who is the chancellor of the University of St Andrews?", + "target_title": "Chancellor of the University of St Andrews", + "target_root": "893b673295d48623c8ec2b15db7f41189139340a4c52ba2509950749df04dcc4", + "shard": "001.db" + }, + { + "question": "who is the president of Northern Cyprus?", + "target_title": "President of Northern Cyprus", + "target_root": "0399be405c6792b791751aa8f08d87cd8871833f4f5d990c5528b644ce7fd9f2", + "shard": "001.db" + }, + { + "question": "who is the chief minister of Khyber Pakhtunkhwa?", + "target_title": "Chief Minister of Khyber Pakhtunkhwa", + "target_root": "7a65509f487f50c1395bb4d17db0daee6a9608fdde7d2b6d007d09046a9d7d74", + "shard": "001.db" + }, + { + "question": "who is the president of the United Arab Emirates?", + "target_title": "President of the United Arab Emirates", + "target_root": "54cdcf083ffb15f556a3576d42468b3f5938bfb55bf785e1c63973f7bdc7b1dd", + "shard": "001.db" + }, + { + "question": "who is the president of Burma?", + "target_title": "President of Burma", + "target_root": "01be2f7fdf52e4316b1bbe7d38dafe7fc38efeede28f89cc2ae664e8d2559234", + "shard": "001.db" + }, + { + "question": "who is the president of Madagascar's Small Grants Scheme?", + "target_title": "President of Madagascar's Small Grants Scheme", + "target_root": "2f25708c6e1c288a58ef26a05c431f2a1dc3dce90cf7cddec381a9a27d6003a9", + "shard": "001.db" + }, + { + "question": "who is the president of Savannah State University?", + "target_title": "President of Savannah State University", + "target_root": "fe62b0a88543878747f23aa3ed6fc1b4775f6f3cc2499be3e0a389d2cb1949c0", + "shard": "001.db" + }, + { + "question": "who is the chancellor of Keele University?", + "target_title": "Chancellor of Keele University", + "target_root": "270ea2e6ccada6045f690d266f846a36e4befb1f33df401a32a4ad1074725d86", + "shard": "001.db" + }, + { + "question": "who is the prime minister of the Republic of Adygea?", + "target_title": "Prime Minister of the Republic of Adygea", + "target_root": "1a4a6e5ba3a3f023efdccd546703d65426faadff996afb2cde5c84c84ca0cb6e", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Zimbabwe Rhodesia?", + "target_title": "Prime Minister of Zimbabwe Rhodesia", + "target_root": "64e7b87d89abc6d9cbe6cb7583034f7531be9c6d75489fbcfaca1b884037dc89", + "shard": "001.db" + }, + { + "question": "who is the premier of Niue?", + "target_title": "Premier of Niue", + "target_root": "6151f7f61b8a6428a4daaea5a5b1dbbdaddb81958bc123cf0a7de54833206f9a", + "shard": "001.db" + }, + { + "question": "who is the president of Veneto?", + "target_title": "President of Veneto", + "target_root": "9fa2a3e4a5db5440c0442905b258979db2a2bf137965eacf76cd464de8d0a31d", + "shard": "001.db" + }, + { + "question": "who is the president of Grinnell College?", + "target_title": "President of Grinnell College", + "target_root": "cfb01da14c6ec0a8e25e77dfa21bad7f8273e0d7cebafd8aac907e156b8dac1b", + "shard": "001.db" + }, + { + "question": "who is the prime minister of South Ossetia?", + "target_title": "Prime Minister of South Ossetia", + "target_root": "c9f3097abad9d256a8345d8918d2f59812a7ee5e739dbdf6e9ef1c58ccdd215c", + "shard": "001.db" + }, + { + "question": "who is the premier of Dominica?", + "target_title": "Premier of Dominica", + "target_root": "58a11a83082b9475331756f30010ba75c1f328dfd3b9e90380e60a68bcc72115", + "shard": "001.db" + }, + { + "question": "who is the president of the Dewan Negara?", + "target_title": "President of the Dewan Negara", + "target_root": "3af66e8aaccf1f19e8808f4839e0171c3b22f3768092a2b1df015e538408c7b5", + "shard": "001.db" + }, + { + "question": "who is the chief minister of Sri Lanka?", + "target_title": "Chief Minister of Sri Lanka", + "target_root": "c1c042764b21fccfce8cfb1c2c53537285555d26f4548a3b09d675fa77df2658", + "shard": "001.db" + }, + { + "question": "who is the chancellor of the University of Glasgow?", + "target_title": "Chancellor of the University of Glasgow", + "target_root": "ea5e53e94e2c8998398dbb64693af96c0b93a5c1afd9e437c4d4e0708f5c355b", + "shard": "001.db" + }, + { + "question": "who is the president of Bashkortostan?", + "target_title": "President of Bashkortostan", + "target_root": "d196334a7d4fb3f2c32b7a8c092681119022574e2ffa6c9c1963b3ccfcee3fc5", + "shard": "001.db" + }, + { + "question": "who is the president of the Marshall Islands?", + "target_title": "President of the Marshall Islands", + "target_root": "3e6e66f43a8ad4d1fa81b6c119b3d571f4f4f8573b09260ecd66f188f353578e", + "shard": "001.db" + }, + { + "question": "who is the president of Iraqi Kurdistan?", + "target_title": "President of Iraqi Kurdistan", + "target_root": "785ad19e7116784912ec698c4601c0647e83a434021b1aad3c9d70dd10de9070", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Slovenia?", + "target_title": "Prime Minister of Slovenia", + "target_root": "2fe6b8eff65ed7f7e09436d05952ba6908c268d7cc5b052cef863de752e7c774", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Curaçao?", + "target_title": "Prime Minister of Curaçao", + "target_root": "0cee26f9ac32d9d9ccdfb2c6609ab5df48535188269684d5017ff2cac47ff958", + "shard": "001.db" + }, + { + "question": "who is the prime minister of Sint Maarten?", + "target_title": "Prime Minister of Sint Maarten", + "target_root": "8c389805b324000d4a7eae225f8c0524f9e1eb36aa78336c45a2775519740462", + "shard": "001.db" + }, + { + "question": "who is the chief minister of the Northern Territory?", + "target_title": "Chief Minister of the Northern Territory", + "target_root": "0220115f90b5edfa9d5422fc0fdf9647d51b338f23a6ed16fa56bf01f999ae01", + "shard": "002.db" + }, + { + "question": "who is the premier of Victoria?", + "target_title": "Premier of Victoria", + "target_root": "bcea0805f576c48bcfd3535ee07b83821de85a863ea092a68ca278a3b9ba38a5", + "shard": "002.db" + }, + { + "question": "who is the president of Ireland?", + "target_title": "President of Ireland", + "target_root": "aa14e62f2ee2eb1a2524a5dc861ee9405e008b52fdee1a172233dbbd5773fbfe", + "shard": "002.db" + }, + { + "question": "who is the president of South Korea?", + "target_title": "President of South Korea", + "target_root": "849e564e05495be7e2938b2c661ce14bdd7d14bcb1b5464041953aa72fd03873", + "shard": "002.db" + }, + { + "question": "who is the president of the Irish Republic?", + "target_title": "President of the Irish Republic", + "target_root": "b554f79187466ef3187194e72ae306dbf8d57f7e908b92503045acf250104206", + "shard": "002.db" + }, + { + "question": "who is the president of Israel?", + "target_title": "President of Israel", + "target_root": "7517eb98208236167d00c00cf917997df2e75f8594f135e49009c3125cf93f38", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Iceland?", + "target_title": "Prime Minister of Iceland", + "target_root": "87480c4acb3b7e52061a2f0a31d677565c55be687702e103b6fbd6e36db2af75", + "shard": "002.db" + }, + { + "question": "who is the president of Tajikistan?", + "target_title": "President of Tajikistan", + "target_root": "8dfe268b5570934f72fb54eb6b26b09f42309f57100499930905021dc43db80b", + "shard": "002.db" + }, + { + "question": "who is the president of Bolivia?", + "target_title": "President of Bolivia", + "target_root": "08f492ce9ce061d669146be4117395ac69b402e1c5a931bef7faf9400ff35e82", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Belgium?", + "target_title": "Prime Minister of Belgium", + "target_root": "1a42195ac981a1ea1b62adfd843d6ece0cbc5a70a466d77f067c015e0fc4258b", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Finland?", + "target_title": "Prime Minister of Finland", + "target_root": "d564f365be8aba57f832b35efbbdc7aa78d9c88964b326c6bac25500b5d89e88", + "shard": "002.db" + }, + { + "question": "who is the president of Iran?", + "target_title": "President of Iran", + "target_root": "b039e088a759b39e4bfb7d4e0738573e712e45b2ccdc93feeee6920702bda4d9", + "shard": "002.db" + }, + { + "question": "who is the president of Russia?", + "target_title": "President of Russia", + "target_root": "1d4e8fb57423e4d5bf936f1ce7ebabb2ab8c3378ac4252ee503abe38eae106ff", + "shard": "002.db" + }, + { + "question": "who is the president of the Philippines?", + "target_title": "President of the Philippines", + "target_root": "812c2db6c1f20f6edec67b99e261305bb746285c346f9291d7fa6d68c2c54a1c", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Thailand?", + "target_title": "Prime Minister of Thailand", + "target_root": "a2db7006030db84c43ea5e80cefc0623f9eb27608a9fd0f7bedcbf4d88f17bd0", + "shard": "002.db" + }, + { + "question": "who is the prime minister of the Netherlands?", + "target_title": "Prime Minister of the Netherlands", + "target_root": "c22af468d86f3833f24364bfa2fe736e4018dc6e3b27ec87e0a89e0da0ca53a0", + "shard": "002.db" + }, + { + "question": "who is the president of Nicaragua?", + "target_title": "President of Nicaragua", + "target_root": "0859af17bce24a4a88d88d8921bcfacc6f7c54dea92aecd49f7b39f5817164f0", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Jamaica?", + "target_title": "Prime Minister of Jamaica", + "target_root": "490e1f75f7833818b32d4ecce690fb53e16b7aec85eff54b3acc170c79b38417", + "shard": "002.db" + }, + { + "question": "who is the president of South Africa?", + "target_title": "President of South Africa", + "target_root": "6bf52237d167fd3ef94d7468713a2cd495b7b8e9e27a4976c12ca3daf5882a1e", + "shard": "002.db" + }, + { + "question": "who is the president of Sri Lanka?", + "target_title": "President of Sri Lanka", + "target_root": "e1eb4868981b5eed8dacc81f0d10577a453054a6fcc7ef628c166aa542f3d83f", + "shard": "002.db" + }, + { + "question": "who is the premier of China?", + "target_title": "Premier of China", + "target_root": "62ccf6956615947f72e8659277fed99bbcc0282a627235ee0b174c5ccedcbf8d", + "shard": "002.db" + }, + { + "question": "who is the president of Sierra Leone?", + "target_title": "President of Sierra Leone", + "target_root": "f996123c19e7a9cee10456c09d66f5939ca20b7405ec59dfaef27c069628ddf6", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Pakistan?", + "target_title": "Prime Minister of Pakistan", + "target_root": "97d4139b7efccc597bb9a9ad82a0b7f49b60d963d710056e795a9efcd217f455", + "shard": "002.db" + }, + { + "question": "who is the president of Fiji?", + "target_title": "President of Fiji", + "target_root": "3e953762d984c20ca6d55b5f87d596948da26239c4489ac34ecb017e28ddf6b3", + "shard": "002.db" + }, + { + "question": "who is the premier of Nunavut?", + "target_title": "Premier of Nunavut", + "target_root": "421708cd222af17150567e9c05a67e4bb328e62f6d4f135dbe8f0c9da82959c8", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Latvia?", + "target_title": "Prime Minister of Latvia", + "target_root": "e9c287947d859ce911df1830642f2548143616ecce2ced35d7d2853f64bd56f7", + "shard": "002.db" + }, + { + "question": "who is the president of Trinidad and Tobago?", + "target_title": "President of Trinidad and Tobago", + "target_root": "50460536aabd98b4854fc75c8802c7a6b4dff9b0ee9dbfb4b18a806ac096bb70", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Georgia?", + "target_title": "Prime Minister of Georgia", + "target_root": "abf0df2ccf61ba0a9cf3fdb8f58ada085ed1c90f9ce29fbe58bcd24b6751b914", + "shard": "002.db" + }, + { + "question": "who is the prime minister of South Korea?", + "target_title": "Prime Minister of South Korea", + "target_root": "67a369158a95c56d6e66a9107b9b6981042453f65f9eadca809fca83cb4d18b4", + "shard": "002.db" + }, + { + "question": "who is the president of Madrid?", + "target_title": "President of Madrid", + "target_root": "720407f62a9c6297761e39b55e4bf97b341c4dcd1f357564b90d1b48f0b82de3", + "shard": "002.db" + }, + { + "question": "who is the president of the Confederate States of America?", + "target_title": "President of the Confederate States of America", + "target_root": "c0bdd8a458931f280f0e7a96f222f29fa0486071ec7bc4c0d6637d065368130d", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Rhodesia?", + "target_title": "Prime Minister of Rhodesia", + "target_root": "15cec5c07b67f7659fe0a3a6147e8c3286ddcfe30364f16e19415a352bccafa7", + "shard": "002.db" + }, + { + "question": "who is the president of the Federated States of Micronesia?", + "target_title": "President of the Federated States of Micronesia", + "target_root": "23f2c1dca71ebb00477a7a75fdcaf23fff8973a57a3f35043a08fd385baeb875", + "shard": "002.db" + }, + { + "question": "who is the president of the European Parliament?", + "target_title": "President of the European Parliament", + "target_root": "a512bf9ef733af8987d688f51356f56834f42ab87252dd271642a8717176b07e", + "shard": "002.db" + }, + { + "question": "who is the president of Kosovo?", + "target_title": "President of Kosovo", + "target_root": "cb30180fb753268d760c17582cacd6ed278871cd53a4ad3f1b2fdd20e72ac87d", + "shard": "002.db" + }, + { + "question": "who is the governor-general of Taiwan?", + "target_title": "Governor-General of Taiwan", + "target_root": "cd1bf9a70f4bf1ddb50d39ffe6c417a686c3bdbb395824d151a95ac260474456", + "shard": "002.db" + }, + { + "question": "who is the president of East Timor?", + "target_title": "President of East Timor", + "target_root": "5ddce9ad10e53d27681ccf065873ae931fd694ecbd2a36060927a2f63242721a", + "shard": "002.db" + }, + { + "question": "who is the prime minister of East Timor?", + "target_title": "Prime Minister of East Timor", + "target_root": "37d54680bbb0216e7313f63a09a8c4fc95c68e6ae5e8e6844bbc7a23439b6c11", + "shard": "002.db" + }, + { + "question": "who is the president of South Ossetia?", + "target_title": "President of South Ossetia", + "target_root": "b31875c92f4b4219508eede4eb8c4c0599d431c0516a8ddaba556370ba5de620", + "shard": "002.db" + }, + { + "question": "who is the president of Princeton University?", + "target_title": "President of Princeton University", + "target_root": "b88123d940b33df6fa54f830019e3c9e604bade051bbf25d35275414442b1a3f", + "shard": "002.db" + }, + { + "question": "who is the president of Vanuatu?", + "target_title": "President of Vanuatu", + "target_root": "04c7616f40f3a4d6b62dd1223d0604108606d8394d03bf3173486234dec69125", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Moldova?", + "target_title": "Prime Minister of Moldova", + "target_root": "4811587ed851f9673935e281f665092a95acb544246274addf193ea522beaf9e", + "shard": "002.db" + }, + { + "question": "who is the premier of the Eastern Cape?", + "target_title": "Premier of the Eastern Cape", + "target_root": "bcbce04b8f30772410656d69c5486fda096f719afb8a20113f378d5f5f0f434f", + "shard": "002.db" + }, + { + "question": "who is the premier of Mpumalanga?", + "target_title": "Premier of Mpumalanga", + "target_root": "49fb67569afe4e7b87c9355ba31c4160049d89ff649fb5f8aa0ad24845510f13", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Bahrain?", + "target_title": "Prime Minister of Bahrain", + "target_root": "c8549a1c90d17f07e439789450d5d9af7344b588a5d086bd50bc1eecb2c4c539", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Kenya?", + "target_title": "Prime Minister of Kenya", + "target_root": "1b0f7620ebb5a32c56c5cc6e09d83b5778fd8b018ea50fb5a650682a20febb34", + "shard": "002.db" + }, + { + "question": "who is the chancellor of the Order of the Thistle?", + "target_title": "Chancellor of the Order of the Thistle", + "target_root": "c3d3da3396814a0a996d912eb8d5a7e3ac6c49287ec53bd1a0a503b8b7084c0d", + "shard": "002.db" + }, + { + "question": "who is the president of Tynwald?", + "target_title": "President of Tynwald", + "target_root": "be130d02785353033f3d6aad4527b9a35e4f9e7dfcbca2369561b579632a131e", + "shard": "002.db" + }, + { + "question": "who is the president of the Municipal Chamber?", + "target_title": "President of the Municipal Chamber", + "target_root": "abbf53f0055f3f29af5836858c83bcedc696a245f6cd8b834b1ae3177e063faf", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Brunei?", + "target_title": "Prime Minister of Brunei", + "target_root": "fe575292cc3ebcc7de64f9701d31ad15d60aecc158ae324a3bf55d6b89dc9f12", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Vanuatu?", + "target_title": "Prime Minister of Vanuatu", + "target_root": "59dc5b0ebf49835b8daf57ba2c55092a1c5cdd6249b98211343c8ed8994e1485", + "shard": "002.db" + }, + { + "question": "who is the president of the Gaelic Athletic Association?", + "target_title": "President of the Gaelic Athletic Association", + "target_root": "afc3bfadd0a2137f6885029e216d3f3d89054039d122673833938a233165051d", + "shard": "002.db" + }, + { + "question": "who is the president of Montenegro?", + "target_title": "President of Montenegro", + "target_root": "a0c0ed85b028432fc3bf1053ea26c5664f36b31884c1988302ca2d413c80dbc1", + "shard": "002.db" + }, + { + "question": "who is the president of the Family Division?", + "target_title": "President of the Family Division", + "target_root": "681b9df62cc7639dd3bda9532fec2b152d9ba97603e6fa7975ab33e8f9a33871", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Cuba?", + "target_title": "Prime Minister of Cuba", + "target_root": "5eeb486a9bb18bb41250af747b6dbedb59c59d2330b9f09c7810aec4701623f8", + "shard": "002.db" + }, + { + "question": "who is the president of the Geological Society of London?", + "target_title": "President of the Geological Society of London", + "target_root": "f3c7449a6f697e4c58a55ca81d59aa4542f9c5b160ec527302e88480cee82ecb", + "shard": "002.db" + }, + { + "question": "who is the premier of Nevis?", + "target_title": "Premier of Nevis", + "target_root": "e18c84b840b8fcb673576d48a49e010a42897fb411c6870d3bf74da033060a3d", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Tajikistan?", + "target_title": "Prime Minister of Tajikistan", + "target_root": "6b7879ff5aa12c59da11f54b05c8e4e89b17447e01a1928ee16e2634311ac541", + "shard": "002.db" + }, + { + "question": "who is the chancellor of the Tang Dynasty?", + "target_title": "Chancellor of the Tang Dynasty", + "target_root": "a961165f242ce4fc0c3e99ff09eae6d566e47fd559fc838ddec171286d1851df", + "shard": "002.db" + }, + { + "question": "who is the chancellor of Justice of Finland?", + "target_title": "Chancellor of Justice of Finland", + "target_root": "65b19bd29edae1d26a51b4a5192e5e4acc8162932ddcc94c3a96029843424ef0", + "shard": "002.db" + }, + { + "question": "who is the president of Crimea?", + "target_title": "President of Crimea", + "target_root": "3f9c43f895b61ca8d4b9f93ab3814c1069eab1c241e1a8308ec5f41ed99f7160", + "shard": "002.db" + }, + { + "question": "who is the president of the Republic of Adygea?", + "target_title": "President of the Republic of Adygea", + "target_root": "e8357469bbe682e036fcdcea8d50196db9a2314e25b35c0bacb2fb23a77d3fb8", + "shard": "002.db" + }, + { + "question": "who is the president of the Chamber of Deputies of Romania?", + "target_title": "President of the Chamber of Deputies of Romania", + "target_root": "cc685bd3f00b928fcbdd9792a6abd01f3f60504e59d1c2a7f2eca2377641c506", + "shard": "002.db" + }, + { + "question": "who is the president of Nepal?", + "target_title": "President of Nepal", + "target_root": "cef934fb43e7353c62bf9e2aa3c54baceef169413f39726da9c04f84a6da85ad", + "shard": "002.db" + }, + { + "question": "who is the president of the Republic of Sierra Leone?", + "target_title": "President of the Republic of Sierra Leone", + "target_root": "280fb190d48c6d8ccbe7a05134e200f6f7fae63b42e7653cb432348ba6e18a68", + "shard": "002.db" + }, + { + "question": "who is the president of the Government of Vojvodina?", + "target_title": "President of the Government of Vojvodina", + "target_root": "663e9f34af16913ee67022d681766cdd6a2af954b5b0f652341c55cc06bd8cdb", + "shard": "002.db" + }, + { + "question": "who is the president of Ukraine?", + "target_title": "President of Ukraine", + "target_root": "5d1bd51c7738a7cd37f35568fd7d52b5639ebf4bbd75b071169fc7b59a119d1c", + "shard": "002.db" + }, + { + "question": "who is the chief minister of the Australian Capital Territory?", + "target_title": "Chief Minister of the Australian Capital Territory", + "target_root": "cd61d292b24b810bb16aeb80964bec89e13658654688eaa9eb7ad43f36df1903", + "shard": "002.db" + }, + { + "question": "who is the president of Mexico?", + "target_title": "President of Mexico", + "target_root": "33d4ed250fb64ed28770356917d12952274a9a7cf2c2ee526193950c2ca13b5f", + "shard": "002.db" + }, + { + "question": "who is the premier of Gauteng?", + "target_title": "Premier of Gauteng", + "target_root": "f64022e2851a6538dac9e8cef05ea693f4cf40d7f47db4f1944cbc6b7d0f8e9a", + "shard": "002.db" + }, + { + "question": "who is the premier of KwaZulu-Natal?", + "target_title": "Premier of KwaZulu-Natal", + "target_root": "957291346304f21248d34ec711ce22023cedf4757124ba1db5ab706665ee638c", + "shard": "002.db" + }, + { + "question": "who is the president of the Parliament of Montenegro?", + "target_title": "President of the Parliament of Montenegro", + "target_root": "feb85707242a3cdf8643af431f0faa4b70b60082980f8024ead313a9ae116c74", + "shard": "002.db" + }, + { + "question": "who is the president of Arellano University?", + "target_title": "President of Arellano University", + "target_root": "dfb4f8a795e48c1237d763fb09b5cb9fa3d2df0a896f4d30942469a7311253ac", + "shard": "002.db" + }, + { + "question": "who is the prime minister of Canada?", + "target_title": "Prime Minister of Canada", + "target_root": "ff079821d866e3f85b9c88ec0635d3ec050e73bd527a617ce54b847604ebb95b", + "shard": "003.db" + }, + { + "question": "who is the prime minister of the United Kingdom?", + "target_title": "Prime Minister of the United Kingdom", + "target_root": "c7bbd8611062fcf1e94de2c51b16476e09e3b6bd8955c5fca4c68858266033ef", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Israel?", + "target_title": "Prime Minister of Israel", + "target_root": "6fe6998985e6cb3eff94a3bbf5a6da2e76ce9fda70a0a4a470d423940ec6eaa0", + "shard": "003.db" + }, + { + "question": "who is the president of Germany?", + "target_title": "President of Germany", + "target_root": "313edbeb06489df339d04933f814b62bd242592084cace3f1fa6e6b9410bd1cc", + "shard": "003.db" + }, + { + "question": "who is the chancellor of Austria?", + "target_title": "Chancellor of Austria", + "target_root": "9703e4ec37ce9d48ab189ae8043912bb7f5eb51795b4a3d8eb47b8b4cd45b067", + "shard": "003.db" + }, + { + "question": "who is the president of the Republic of China?", + "target_title": "President of the Republic of China", + "target_root": "262b1fe7d0dc490f5c60c8fcc6290ff3b047d5c58d1404606978857bb929654a", + "shard": "003.db" + }, + { + "question": "who is the prime minister of New Zealand?", + "target_title": "Prime Minister of New Zealand", + "target_root": "99fad3fb7d46e1e93cf658edc142385cf801294ca68729a29cac7a11fd1f1354", + "shard": "003.db" + }, + { + "question": "who is the governor-general of the Irish Free State?", + "target_title": "Governor-General of the Irish Free State", + "target_root": "4e8d49059c68d55a891272adf1f8fa607beafac00be9fee82f515962de98e747", + "shard": "003.db" + }, + { + "question": "who is the premier of the People's Republic of China?", + "target_title": "Premier of the People's Republic of China", + "target_root": "f690542d183297b41da0e297302e93f6d3eb1599ad96519bd30facae7e0c0512", + "shard": "003.db" + }, + { + "question": "who is the president of Turkey?", + "target_title": "President of Turkey", + "target_root": "52abb9f18c53af632426a97ec60e72c0572ee2c1e4b53404b8d5adf2bc8497ca", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Sweden?", + "target_title": "Prime Minister of Sweden", + "target_root": "e77a4b31f1334bba08258d3a5f9727d271f8443514d9de4cfbf1c8ef4478ea04", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Spain?", + "target_title": "Prime Minister of Spain", + "target_root": "4f851e77f4064fabc249500e0e8cfdea842652a9917bcd77d0e6a7ab7312d4c1", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Norway?", + "target_title": "Prime Minister of Norway", + "target_root": "2d2b415084da2556c77140a86b48569657cd4cc66453900e7a855f9662bb5ca4", + "shard": "003.db" + }, + { + "question": "who is the president of Argentina?", + "target_title": "President of Argentina", + "target_root": "5b776fabc3c6062641b8106cd27e86e7033effd21bebd94627f9157a31cb80a9", + "shard": "003.db" + }, + { + "question": "who is the president of Colombia?", + "target_title": "President of Colombia", + "target_root": "5703b1c1fa269465d29f40d6be2cfe455edbc4623ab2c50fe5a95debe0621ce8", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Romania?", + "target_title": "Prime Minister of Romania", + "target_root": "41f4cc18f3bf8ca468622298b79b5c58a9432e02583c70db9f316bcbfeb3e782", + "shard": "003.db" + }, + { + "question": "who is the president of Brazil?", + "target_title": "President of Brazil", + "target_root": "891c2eee9fd66e2644d5d07ddee3f085584dea9e0991c0941f6c8b5485f587c8", + "shard": "003.db" + }, + { + "question": "who is the president of Romania?", + "target_title": "President of Romania", + "target_root": "e3a038404f28d9f9b218e8c791dbff3533a90ccb0a8f174e6af3319558f34d76", + "shard": "003.db" + }, + { + "question": "who is the governor-general of Finland?", + "target_title": "Governor-General of Finland", + "target_root": "d01d2bc4b9824d791da8230219d4ba589a88dd4486d717fd02263f5a718774e2", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Papua New Guinea?", + "target_title": "Prime Minister of Papua New Guinea", + "target_root": "feacb771e587eaf3cd284b06bbfafd578497886f5e27c4220c2242368b568c30", + "shard": "003.db" + }, + { + "question": "who is the prime minister of South Africa?", + "target_title": "Prime Minister of South Africa", + "target_root": "1f00a720f2549c2950c38008ac8cd5b43c1bd031f1c0372f04ead9e749214dd5", + "shard": "003.db" + }, + { + "question": "who is the president of Slovenia?", + "target_title": "President of Slovenia", + "target_root": "add4441b4cd528dcaa50a8c1b9ce873c134e3cfcbbb446e8b8f6d393be5603b9", + "shard": "003.db" + }, + { + "question": "who is the premier of British Columbia?", + "target_title": "Premier of British Columbia", + "target_root": "e755b7bb91069c4701ab1d82515a0062af22fcde170cfd1095ebed2040a8487f", + "shard": "003.db" + }, + { + "question": "who is the president of Egypt?", + "target_title": "President of Egypt", + "target_root": "067fe8b5d9cf788652fe78dab3a6c80d147bf5b11f960fc3a3f9ec7e0c092d2d", + "shard": "003.db" + }, + { + "question": "who is the president of the Continental Congress?", + "target_title": "President of the Continental Congress", + "target_root": "5988d5a9ab416dc4e70e60bb248926f1555050751a0fce3fc8475ea724ec3e64", + "shard": "003.db" + }, + { + "question": "who is the president of Singapore?", + "target_title": "President of Singapore", + "target_root": "11c591aa9054cf37eb8c0705eb0fd623a280b579f87fc8f7f0e8d70c6a0c33d9", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Sri Lanka?", + "target_title": "Prime Minister of Sri Lanka", + "target_root": "b8cc87758ee0929f9451fc7945783dd23a283d7540163c6c0cd094b3418575b3", + "shard": "003.db" + }, + { + "question": "who is the governor-general of Papua New Guinea?", + "target_title": "Governor-General of Papua New Guinea", + "target_root": "dee138e9a4b7fb8dc5a6d9a6b53a7239f690f59ccd780165c2040ecdf6c80520", + "shard": "003.db" + }, + { + "question": "who is the premier of Quebec?", + "target_title": "Premier of Quebec", + "target_root": "0035e535e9d2987f89616497ed17541711528d284c4bc33acbfa104d2e236e09", + "shard": "003.db" + }, + { + "question": "who is the president of Angola?", + "target_title": "President of Angola", + "target_root": "ab04390d1ab67002fbc270a72fb39ff2cdb12532eab9a6c2c33b71041272141b", + "shard": "003.db" + }, + { + "question": "who is the governor-general of Kenya?", + "target_title": "Governor-General of Kenya", + "target_root": "9e64e9408c237ecc4315f440c69cd08ae8492017ee5174aee2e145d5dcf7eec4", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Algeria?", + "target_title": "Prime Minister of Algeria", + "target_root": "b8d4b0168d7c27e0fbbf7314c1dfc76e158ff93386e948d43bc376606ace9b69", + "shard": "003.db" + }, + { + "question": "who is the president of Pakistan?", + "target_title": "President of Pakistan", + "target_root": "951d4ea7faea094cef771c5215d312491b842cecfe736f9ce48b40c6af198519", + "shard": "003.db" + }, + { + "question": "who is the president of Mauritius?", + "target_title": "President of Mauritius", + "target_root": "9961ebcbc17f682d463b3b465983a28832bb21f74809647bff6eaf9c92530906", + "shard": "003.db" + }, + { + "question": "who is the president of El Salvador?", + "target_title": "President of El Salvador", + "target_root": "d362a630cb089dead536515b95bf6d2dd31f9dac452c94db28d6fbff86f61970", + "shard": "003.db" + }, + { + "question": "who is the president of Vietnam?", + "target_title": "President of Vietnam", + "target_root": "c50b32da6beb495b99fb0e4d95cd180cb4b0ffb37d67dae4f9dc2ce36acac4f4", + "shard": "003.db" + }, + { + "question": "who is the premier of Nova Scotia?", + "target_title": "Premier of Nova Scotia", + "target_root": "05fd76ea2afd9fd6ee3604caa63cffaaa9eed9c9ee9e23ecedd72439f205f66a", + "shard": "003.db" + }, + { + "question": "who is the premier of Saskatchewan?", + "target_title": "Premier of Saskatchewan", + "target_root": "998e92b5cbf1b2e4aa914e2aaa5ebb24706a9e220cc2dba3b2938403180bd553", + "shard": "003.db" + }, + { + "question": "who is the premier of Newfoundland and Labrador?", + "target_title": "Premier of Newfoundland and Labrador", + "target_root": "fa31c92962d1b7e2c18a5aa973a8268e50900a8047b5be21542fa6e1969b303c", + "shard": "003.db" + }, + { + "question": "who is the premier of Yukon?", + "target_title": "Premier of Yukon", + "target_root": "e3d76c3da3906676710c9cf16d1d8e9ddf77643a6c49c319892a8cf3106fcf0d", + "shard": "003.db" + }, + { + "question": "who is the governor-general of Pakistan?", + "target_title": "Governor-General of Pakistan", + "target_root": "7ac5129e1363401a26002cb886eee9259ff69c3ca7f4b2939e1709d4822c3995", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Malaysia?", + "target_title": "Prime Minister of Malaysia", + "target_root": "3f2af933c6a03d6ab469f606469470e96c53efdde7b71755ca9ee58891d1c3a2", + "shard": "003.db" + }, + { + "question": "who is the president of Greece?", + "target_title": "President of Greece", + "target_root": "4e32d1f9fd3036f0a9cd49ff0bf9518d6811c9b119386fe44465a2468ae90598", + "shard": "003.db" + }, + { + "question": "who is the president of Nigeria?", + "target_title": "President of Nigeria", + "target_root": "130501845a4267b5b639ce5d743fb4e4d9acb4ace24bb0d0a72a1b706b9ea5ce", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Abkhazia?", + "target_title": "Prime Minister of Abkhazia", + "target_root": "a3917affe4e170e79f15cd66a8934349a8b27520619e5e566adbdf345f53f5df", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Serbia?", + "target_title": "Prime Minister of Serbia", + "target_root": "7dd82f8e8dd073a4a562f6f15a70dc96e959d6204c92ec9818663b7caa0fbc9c", + "shard": "003.db" + }, + { + "question": "who is the president of Paraguay?", + "target_title": "President of Paraguay", + "target_root": "bc2f66cbefa388105dae46ba740b8bd71625f00b71601709c43b6b7626bee49f", + "shard": "003.db" + }, + { + "question": "who is the governor-general of the Dutch East Indies?", + "target_title": "Governor-General of the Dutch East Indies", + "target_root": "26b507afbe9ef31c230d0d38faf272c3e796cdc4b7bde5e33e6ddabd372d4b9a", + "shard": "003.db" + }, + { + "question": "who is the governor-general of Solomon Islands?", + "target_title": "Governor-General of Solomon Islands", + "target_root": "838051a535ea26da3c7bde490ab73f10aed26ed6314c91b3c38d998e62723fca", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Uganda?", + "target_title": "Prime Minister of Uganda", + "target_root": "d15886ca28b0e68821010852f993d09bdd4fcd7ae7cd7a560874999cdb1be3f4", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Ukraine?", + "target_title": "Prime Minister of Ukraine", + "target_root": "da3e04bf44df7dd302504d2843a80463f97870f55a1b44667bdc3e27efe836ac", + "shard": "003.db" + }, + { + "question": "who is the president of the Republic of Macedonia?", + "target_title": "President of the Republic of Macedonia", + "target_root": "0d74ba6da047c557c99cc64bc83b530f0c8260c581c2c86a88493ca316884abf", + "shard": "003.db" + }, + { + "question": "who is the president of Estonia?", + "target_title": "President of Estonia", + "target_root": "c4cb1c9b14bfc36166a8c9557e966c8c81ba4c7cb7a83e00166162f52198d562", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Namibia?", + "target_title": "Prime Minister of Namibia", + "target_root": "463afa4d72fd22ccafea6ebc512c6911b3a016f9251541637f073d8dcf881c49", + "shard": "003.db" + }, + { + "question": "who is the president of Cyprus?", + "target_title": "President of Cyprus", + "target_root": "5097316b387f117d3e43c83ecae2b5240cb83c9e5531cfa00303d4d725ca1516", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Senegal?", + "target_title": "Prime Minister of Senegal", + "target_root": "fe13b678bf31e3e701dc4c02fb601bc92fca75970cf9277b8cfec9823b4a2be6", + "shard": "003.db" + }, + { + "question": "who is the premier of the Kingdom of Viti?", + "target_title": "Premier of the Kingdom of Viti", + "target_root": "2e72fc1cf394e6b4f2b3724709ef107206f1307464d1c6d179e0295c80abf5ec", + "shard": "003.db" + }, + { + "question": "who is the president of Ichkeria?", + "target_title": "President of Ichkeria", + "target_root": "eb97fd0d0d77a14a185da1875dfe9f73da60ec1945ae6e131d2ed438abaa3bdc", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Madagascar?", + "target_title": "Prime Minister of Madagascar", + "target_root": "569fa9dcc5624dba1ea21f363ac327700740409a95ea22fd34a144b9479443ff", + "shard": "003.db" + }, + { + "question": "who is the governor-general of Ceylon?", + "target_title": "Governor-General of Ceylon", + "target_root": "328b73a3833f74d9f1678142bc194175a28f163d50b4c9bfacd829bb6eba35da", + "shard": "003.db" + }, + { + "question": "who is the president of Laos?", + "target_title": "President of Laos", + "target_root": "a27bf2f9c798ec6a8831b16597040a91bbfd728707cf1208c5a1fc5bcb001318", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Tunisia?", + "target_title": "Prime Minister of Tunisia", + "target_root": "6337b9aa3bbcaa6afd3cb3ad80612f496393820444e3a866d000edd4802ce884", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Aruba?", + "target_title": "Prime Minister of Aruba", + "target_root": "0beff589113bc1df60cc0010d1f7e04458ab1c6901634dac283bb8f116727bca", + "shard": "003.db" + }, + { + "question": "who is the chief minister of Anguilla?", + "target_title": "Chief Minister of Anguilla", + "target_root": "dad0093fb9ba6fb40bff81ff337d3b1dc0979612f7de5d2c788f61b1e5acfef4", + "shard": "003.db" + }, + { + "question": "who is the chief minister of Jersey?", + "target_title": "Chief Minister of Jersey", + "target_root": "0bd78564b54e972e2c465768d3387f7c7cc4efb825f8460cadad1637acc028f1", + "shard": "003.db" + }, + { + "question": "who is the premier of North West?", + "target_title": "Premier of North West", + "target_root": "17acb03826859363d84dd59abbfddfba4b3a10b63e0f96a3f39e4abba8549535", + "shard": "003.db" + }, + { + "question": "who is the chief minister of Papua and New Guinea?", + "target_title": "Chief Minister of Papua and New Guinea", + "target_root": "0078851ba7118ad535e2dca86d1bc8735504d94dc61b65f68a88f6c4f7f9a59b", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Laos?", + "target_title": "Prime Minister of Laos", + "target_root": "940c02698a77e62129c10d6d77720adb6bcd35c8e7ea1c30bdd569cf29370e51", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Tonga?", + "target_title": "Prime Minister of Tonga", + "target_root": "00bcd105ddb296a8eb215dc06dc07ab31cd0562dcf8a85b2506ea5be48c26da8", + "shard": "003.db" + }, + { + "question": "who is the chief minister of Montserrat?", + "target_title": "Chief Minister of Montserrat", + "target_root": "b3449be00e301031cf0b7c94968df008be3f0d6a88d3ef742ff67440d22ed674", + "shard": "003.db" + }, + { + "question": "who is the prime minister of the Republic of Macedonia?", + "target_title": "Prime Minister of the Republic of Macedonia", + "target_root": "676d8616d82c83d5fc3a5b82800f76dfc0e1d48f9c72bb87746c3824b097cb1a", + "shard": "003.db" + }, + { + "question": "who is the governor-general of New Zealand?", + "target_title": "Governor-General of New Zealand", + "target_root": "78a386731a76ac730221c0023e1e0eac2e83c817f41c19ac3b827923082306fb", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Nagorno-Karabakh Republic?", + "target_title": "Prime Minister of Nagorno-Karabakh Republic", + "target_root": "5c66335f5f397f1da2d4aa9f12f406ad63f564ab758a75aa047a5b4cbacc6216", + "shard": "003.db" + }, + { + "question": "who is the president of Transnistria?", + "target_title": "President of Transnistria", + "target_root": "76275b343e3de568486e7e433306a5412a5dab868c0b014198de1316c9b1b4bb", + "shard": "003.db" + }, + { + "question": "who is the chief minister of Balochistan?", + "target_title": "Chief Minister of Balochistan", + "target_root": "b609fade8e56d2551bf81325a34b1b7e4737ba3543b3f433e0ec11277c1f7687", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Montenegro?", + "target_title": "Prime Minister of Montenegro", + "target_root": "15e0c5a196d0e61b52eedd0290fbc1bd18d8a917a60f18f46d4ba0735c8f0770", + "shard": "003.db" + }, + { + "question": "who is the president of the Legislative Yuan?", + "target_title": "President of the Legislative Yuan", + "target_root": "27f06d435b5661b51f5327793b31545c889f57ce94995b04a10b9a96006f9ac7", + "shard": "003.db" + }, + { + "question": "who is the president of Bucknell University?", + "target_title": "President of Bucknell University", + "target_root": "fb54f56954d30df06bb75ab793609f36a7bdc4ca05f42e97051f47ba7f4b8db4", + "shard": "003.db" + }, + { + "question": "who is the president of the Queen's Bench Division?", + "target_title": "President of the Queen's Bench Division", + "target_root": "74696d67142f5a43b2ad4f2b00bd1bc2b74a8caa8cbaf3530e48f5cb61983ae2", + "shard": "003.db" + }, + { + "question": "who is the chief minister of Guernsey?", + "target_title": "Chief Minister of Guernsey", + "target_root": "18b6ac4f491e4bee2800f44a8e85671a6cc260d241a87ef201bbbb871c1f30e1", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Northern Cyprus?", + "target_title": "Prime Minister of Northern Cyprus", + "target_root": "87dd2d941b013ef61eff91d3f14829e7a3f901331ad73c0afc9a4a80d277b6db", + "shard": "003.db" + }, + { + "question": "who is the president of Grande Comore?", + "target_title": "President of Grande Comore", + "target_root": "61af107e57683026b2537119a61b1d23ae267342dcdafcd9c3e68f7f7f00d965", + "shard": "003.db" + }, + { + "question": "who is the president of Anjouan?", + "target_title": "President of Anjouan", + "target_root": "dc5fc1f25ab8afc883854958eac273653bda8510837c13bfc0cf5095eb3351bc", + "shard": "003.db" + }, + { + "question": "who is the president of Mohéli?", + "target_title": "President of Mohéli", + "target_root": "560b992bc0f0ddd89c1d6a02add6ce61a49eda5466df2b972c4342c514ce1344", + "shard": "003.db" + }, + { + "question": "who is the president of the Bundestag?", + "target_title": "President of the Bundestag", + "target_root": "351b2cc92f708e743d545dee1db0148f9abc44a4f4bb38b510828fc9e9378a7f", + "shard": "003.db" + }, + { + "question": "who is the president of the Moldovan Parliament?", + "target_title": "President of the Moldovan Parliament", + "target_root": "9aff54c20bb1924aaad8969c76eb1d598b66477cce699ec6307ce074c3e9f693", + "shard": "003.db" + }, + { + "question": "who is the chief minister of Punjab?", + "target_title": "Chief Minister of Punjab", + "target_root": "9c714c282daf03811c587c3bfc797dbddf67fad054828fea5fb0650419807623", + "shard": "003.db" + }, + { + "question": "who is the president of Indonesia?", + "target_title": "President of Indonesia", + "target_root": "3337108e37f52b873124133b0e504103c8d01885963a0d8e07696609b6a9ebbd", + "shard": "003.db" + }, + { + "question": "who is the president of Cantabria?", + "target_title": "President of Cantabria", + "target_root": "621516b31787a1a5b404beb151df1b5e7bcb839105047f9894db41801de1154f", + "shard": "003.db" + }, + { + "question": "who is the president of the University of Michigan?", + "target_title": "President of the University of Michigan", + "target_root": "c21d38c22199db355d171b2262506aaefc3434d6809038ab2f5a2970146e9f08", + "shard": "003.db" + }, + { + "question": "who is the chancellor of the University of Edinburgh?", + "target_title": "Chancellor of the University of Edinburgh", + "target_root": "ed6a9ccb25baf857e08cf10addef5bb5ef0d54fecedfc02530d3ee3ab608e5fa", + "shard": "003.db" + }, + { + "question": "who is the prime minister of the Federation of Rhodesia and Nyasaland?", + "target_title": "Prime Minister of the Federation of Rhodesia and Nyasaland", + "target_root": "0c3ab16c1594a5c664322cb8b98264c945ea1aa48a63d78c7d7b555952d4df65", + "shard": "003.db" + }, + { + "question": "who is the president of the Congress of Colombia?", + "target_title": "President of the Congress of Colombia", + "target_root": "e4f16870f331c03b9ff48355611fd300e93bdd8e7ce63b4078addd0173a2de7f", + "shard": "003.db" + }, + { + "question": "who is the premier of the Western Cape?", + "target_title": "Premier of the Western Cape", + "target_root": "4b4237b2ce88b33aee3f8fcfca3dac0f94df78adf374f67e824ec3ef5cbc342d", + "shard": "003.db" + }, + { + "question": "who is the president of Congress?", + "target_title": "President of Congress", + "target_root": "eeed99f03116347b6e72461167849dc5519710546aff0fc3b56ff30301a88f43", + "shard": "003.db" + }, + { + "question": "who is the president of Zimbabwe Rhodesia?", + "target_title": "President of Zimbabwe Rhodesia", + "target_root": "fe186e243239613a81f472ad43454e8d8f8c999f79af2bd7884af5871b95ec9c", + "shard": "003.db" + }, + { + "question": "who is the president of the World Esperanto Association?", + "target_title": "President of the World Esperanto Association", + "target_root": "83fd4a1abe6a906f08005c9950f87337e3ac409bb48faae88edf55dd980e10d7", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Turkmenistan?", + "target_title": "Prime Minister of Turkmenistan", + "target_root": "9b001c3bfc9b73a590ba6ef6a9a0b99cb779586bdffb1dc468f6fb2f4794a77f", + "shard": "003.db" + }, + { + "question": "who is the chancellor of Germany?", + "target_title": "Chancellor of Germany", + "target_root": "e2c1c1b0e1c2298c5d3f20d1811f7dcd71de912ad71160387ddc213b29b7d79f", + "shard": "003.db" + }, + { + "question": "who is the president of the States of Alderney?", + "target_title": "President of the States of Alderney", + "target_root": "35451c0ce9b4727e58803ebc56335c23e811dd98171d57f7ebb07643a7c5bc83", + "shard": "003.db" + }, + { + "question": "who is the president of the Government of New Caledonia?", + "target_title": "President of the Government of New Caledonia", + "target_root": "b91eb6c65ef90744e6a6a1ca85666abfca53dd63a2ff0ff1e395beb2028f8e8e", + "shard": "003.db" + }, + { + "question": "who is the chancellor of the University of Dallas?", + "target_title": "Chancellor of the University of Dallas", + "target_root": "084b84fe49f1fd6282d1c7b2d50d78658e1fffb1a2d10efbc4a4b9ea59f329d2", + "shard": "003.db" + }, + { + "question": "who is the president of the Naval War College?", + "target_title": "President of the Naval War College", + "target_root": "e821ed40d44bdebd823c998e179ae7765c0241c6ae11ba9bef919198ae0d9a87", + "shard": "003.db" + }, + { + "question": "who is the president of the University of Southern Maine?", + "target_title": "President of the University of Southern Maine", + "target_root": "a008b3c88f1e9dae51c65f39730581f5a69d386ab9167d81354b251189c0254f", + "shard": "003.db" + }, + { + "question": "who is the president of Kabardino-Balkaria?", + "target_title": "President of Kabardino-Balkaria", + "target_root": "5fd7ee2f04a1dae3ad012356a91cc419a082fcf9c9d149a9a1477c6ed6c8d1ff", + "shard": "003.db" + }, + { + "question": "who is the president of the University of the Philippines?", + "target_title": "President of the University of the Philippines", + "target_root": "3f9477397a9041de8ff817a793203ba3cdbaa350f7221045824cba9b8437ffb0", + "shard": "003.db" + }, + { + "question": "who is the president of Honduras?", + "target_title": "President of Honduras", + "target_root": "205b27b874b9812a51cd3111592dade588048a2b78d7dc6272fc8ac71339434d", + "shard": "003.db" + }, + { + "question": "who is the premier of the Free State?", + "target_title": "Premier of the Free State", + "target_root": "d58c3d677b5507f6d0ec08038f1cd1a3c614a355818fd0c0e12651a0a3c07738", + "shard": "003.db" + }, + { + "question": "who is the premier of Limpopo?", + "target_title": "Premier of Limpopo", + "target_root": "8b5fcab737add179e8c5fa203bdef6511a6c60a5e511ba31ef9e37ecb8527f91", + "shard": "003.db" + }, + { + "question": "who is the president of Uruguay?", + "target_title": "President of Uruguay", + "target_root": "7dc648d7228158882b746c09f94c6f88b7f3f6cbec4f529419aaf5542a165208", + "shard": "003.db" + }, + { + "question": "who is the president of the Xunta of Galicia?", + "target_title": "President of the Xunta of Galicia", + "target_root": "855ef5bc8af930c23b98d34ae97098fa7a5f3342e4ca41b94ac60a6d3e705a28", + "shard": "003.db" + }, + { + "question": "who is the president of Umbria?", + "target_title": "President of Umbria", + "target_root": "4a18956b29d80905207ffc38528ed9181133f585faf24e7aec8db5efc30a36de", + "shard": "003.db" + }, + { + "question": "who is the prime minister of Hyderabad?", + "target_title": "Prime Minister of Hyderabad", + "target_root": "3d13af7582247d72814331aebfb3073e2b37096720e378a069def33af85f3933", + "shard": "003.db" + }, + { + "question": "who is the premier of Montserrat?", + "target_title": "Premier of Montserrat", + "target_root": "8fdf9d46790b8215ad22e41aca2460423b3f4bfb0c25bd71ee8a21fc6b27d03e", + "shard": "003.db" } ]