From 76c4271e771a8d55fadbaef36897eba3e3acc5bf Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 5 Jun 2026 11:30:11 -0400 Subject: [PATCH] new file: bench/legacy_vs_providence_results/2026-05-31T23-14-10Z.jsonl new file: bench/legacy_vs_providence_results/2026-05-31T23-27-07Z.jsonl --- .../2026-05-31T23-14-10Z.jsonl | 15 +++++++++++++++ .../2026-05-31T23-27-07Z.jsonl | 8 ++++++++ 2 files changed, 23 insertions(+) create mode 100644 bench/legacy_vs_providence_results/2026-05-31T23-14-10Z.jsonl create mode 100644 bench/legacy_vs_providence_results/2026-05-31T23-27-07Z.jsonl diff --git a/bench/legacy_vs_providence_results/2026-05-31T23-14-10Z.jsonl b/bench/legacy_vs_providence_results/2026-05-31T23-14-10Z.jsonl new file mode 100644 index 0000000..5f3015d --- /dev/null +++ b/bench/legacy_vs_providence_results/2026-05-31T23-14-10Z.jsonl @@ -0,0 +1,15 @@ +{"theme": "accent", "question": "what is Casa Batllo?", "default": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 19.01, "error": "exit 1"}, "legacy": {"audit_mode": "STRICT", "n_verified": 4, "primary": {"title": "Casa Batlló", "uri": "https://en.wikipedia.org/wiki/Casa_Batlló", "used": false}, "elapsed_s": 5.53, "error": null}, "same_primary": false, "audit_delta": -3} +{"theme": "accent", "question": "what is Andre-Marie Ampere?", "default": {"audit_mode": "HYBRID", "n_verified": 10, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 25.76, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 4.82, "error": null}, "same_primary": true, "audit_delta": 1} +{"theme": "accent", "question": "what is Antoni Gaudi?", "default": {"audit_mode": "HYBRID", "n_verified": 10, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": true}, "elapsed_s": 18.57, "error": null}, "legacy": {"audit_mode": "HYBRID", "n_verified": 5, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": false}, "elapsed_s": 5.42, "error": null}, "same_primary": true, "audit_delta": 0} +{"theme": "hyphen", "question": "what is The Amazing Spider Man (comic book)?", "default": {"audit_mode": "HYBRID", "n_verified": 8, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": true}, "elapsed_s": 38.17, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": false}, "elapsed_s": 8.25, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "hyphen", "question": "what is Augustin Jean Fresnel?", "default": {"audit_mode": "HYBRID", "n_verified": 17, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": true}, "elapsed_s": 23.14, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": false}, "elapsed_s": 4.98, "error": null}, "same_primary": true, "audit_delta": 1} +{"theme": "hyphen", "question": "what is André Marie Ampère?", "default": {"audit_mode": "HYBRID", "n_verified": 10, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 21.67, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 2, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 4.81, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "honorific", "question": "what is Dr Who?", "default": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Indian Association of Pathologists and Microbiologists", "uri": "https://en.wikipedia.org/wiki/Indian_Association_of_Pathologists_and_Microbiologists", "used": false}, "elapsed_s": 22.19, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 3, "primary": {"title": "Doctor (Doctor Who)", "uri": "https://en.wikipedia.org/wiki/Doctor_(Doctor_Who)", "used": false}, "elapsed_s": 7.01, "error": null}, "same_primary": false, "audit_delta": 0} +{"theme": "honorific", "question": "what is Dr Syn?", "default": {"audit_mode": "HYBRID", "n_verified": 16, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": true}, "elapsed_s": 22.44, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 2, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": false}, "elapsed_s": 6.2, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "honorific", "question": "what is Dr V64?", "default": {"audit_mode": "HYBRID", "n_verified": 16, "primary": {"title": "Doctor V64", "uri": "https://en.wikipedia.org/wiki/Doctor_V64", "used": true}, "elapsed_s": 26.91, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 19, "primary": {"title": "Doctor V64", "uri": "https://en.wikipedia.org/wiki/Doctor_V64", "used": false}, "elapsed_s": 10.53, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "brit", "question": "what is Finnish defense Forces?", "default": {"audit_mode": "HYBRID", "n_verified": 20, "primary": {"title": "Finnish Defence Forces", "uri": "https://en.wikipedia.org/wiki/Finnish_Defence_Forces", "used": true}, "elapsed_s": 32.71, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 11, "primary": {"title": "Finnish Defence Forces", "uri": "https://en.wikipedia.org/wiki/Finnish_Defence_Forces", "used": false}, "elapsed_s": 9.44, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "brit", "question": "what is Hopewell center, Hong Kong?", "default": {"audit_mode": "HYBRID", "n_verified": 22, "primary": {"title": "Hopewell Centre, Hong Kong", "uri": "https://en.wikipedia.org/wiki/Hopewell_Centre,_Hong_Kong", "used": true}, "elapsed_s": 34.15, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "Hopewell Centre, Hong Kong", "uri": "https://en.wikipedia.org/wiki/Hopewell_Centre,_Hong_Kong", "used": false}, "elapsed_s": 8.06, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "brit", "question": "what is labor economics?", "default": {"audit_mode": "HYBRID", "n_verified": 3, "primary": {"title": "Journal of Labor Economics", "uri": "https://en.wikipedia.org/wiki/Journal_of_Labor_Economics", "used": true}, "elapsed_s": 21.15, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 11, "primary": {"title": "Journal of Labor Economics", "uri": "https://en.wikipedia.org/wiki/Journal_of_Labor_Economics", "used": false}, "elapsed_s": 9.76, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "numeral", "question": "who was Albert the third?", "default": {"audit_mode": "HYBRID", "n_verified": 9, "primary": {"title": "Inside the Third Reich", "uri": "https://en.wikipedia.org/wiki/Inside_the_Third_Reich", "used": false}, "elapsed_s": 26.76, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "The Third", "uri": "https://en.wikipedia.org/wiki/The_Third", "used": false}, "elapsed_s": 6.94, "error": null}, "same_primary": false, "audit_delta": 1} +{"theme": "numeral", "question": "who was Ahmed the third?", "default": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Ahmed Samir", "uri": "https://en.wikipedia.org/wiki/Ahmed_Samir", "used": false}, "elapsed_s": 21.33, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "III", "uri": "https://en.wikipedia.org/wiki/III", "used": false}, "elapsed_s": 5.66, "error": null}, "same_primary": false, "audit_delta": 2} +{"theme": "numeral", "question": "who was Alaric the first?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "Alaric", "uri": "https://en.wikipedia.org/wiki/Alaric", "used": true}, "elapsed_s": 72.75, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Alaric I", "uri": "https://en.wikipedia.org/wiki/Alaric_I", "used": false}, "elapsed_s": 12.71, "error": null}, "same_primary": false, "audit_delta": -1} diff --git a/bench/legacy_vs_providence_results/2026-05-31T23-27-07Z.jsonl b/bench/legacy_vs_providence_results/2026-05-31T23-27-07Z.jsonl new file mode 100644 index 0000000..3e414c4 --- /dev/null +++ b/bench/legacy_vs_providence_results/2026-05-31T23-27-07Z.jsonl @@ -0,0 +1,8 @@ +{"theme": "accent", "question": "what is Casa Batllo?", "default": {"audit_mode": "?", "n_verified": null, "primary": {"title": "", "uri": "", "used": false}, "elapsed_s": 46.7, "error": "exit 1"}, "legacy": {"audit_mode": "STRICT", "n_verified": 3, "primary": {"title": "Casa Batlló", "uri": "https://en.wikipedia.org/wiki/Casa_Batlló", "used": false}, "elapsed_s": 12.55, "error": null}, "same_primary": false, "audit_delta": -3} +{"theme": "accent", "question": "what is Andre-Marie Ampere?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 61.5, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 22.04, "error": null}, "same_primary": true, "audit_delta": 1} +{"theme": "accent", "question": "what is Antoni Gaudi?", "default": {"audit_mode": "HYBRID", "n_verified": 10, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": true}, "elapsed_s": 44.99, "error": null}, "legacy": {"audit_mode": "HYBRID", "n_verified": 5, "primary": {"title": "Antoni Gaudí", "uri": "https://en.wikipedia.org/wiki/Antoni_Gaudí", "used": false}, "elapsed_s": 24.15, "error": null}, "same_primary": true, "audit_delta": 0} +{"theme": "hyphen", "question": "what is The Amazing Spider Man (comic book)?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": true}, "elapsed_s": 93.64, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 6, "primary": {"title": "The Amazing Spider-Man (comic book)", "uri": "https://en.wikipedia.org/wiki/The_Amazing_Spider-Man_(comic_book)", "used": false}, "elapsed_s": 16.34, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "hyphen", "question": "what is Augustin Jean Fresnel?", "default": {"audit_mode": "HYBRID", "n_verified": 8, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": true}, "elapsed_s": 48.78, "error": null}, "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "primary": {"title": "Augustin-Jean Fresnel", "uri": "https://en.wikipedia.org/wiki/Augustin-Jean_Fresnel", "used": false}, "elapsed_s": 11.19, "error": null}, "same_primary": true, "audit_delta": 1} +{"theme": "hyphen", "question": "what is André Marie Ampère?", "default": {"audit_mode": "HYBRID", "n_verified": 4, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": true}, "elapsed_s": 48.35, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "André-Marie Ampère", "uri": "https://en.wikipedia.org/wiki/André-Marie_Ampère", "used": false}, "elapsed_s": 6.91, "error": null}, "same_primary": true, "audit_delta": -1} +{"theme": "honorific", "question": "what is Dr Who?", "default": {"audit_mode": "STRICT", "n_verified": 1, "primary": {"title": "Indian Association of Pathologists and Microbiologists", "uri": "https://en.wikipedia.org/wiki/Indian_Association_of_Pathologists_and_Microbiologists", "used": false}, "elapsed_s": 25.34, "error": null}, "legacy": {"audit_mode": "STRICT", "n_verified": 3, "primary": {"title": "Doctor (Doctor Who)", "uri": "https://en.wikipedia.org/wiki/Doctor_(Doctor_Who)", "used": false}, "elapsed_s": 8.57, "error": null}, "same_primary": false, "audit_delta": 0} +{"theme": "honorific", "question": "what is Dr Syn?", "default": {"audit_mode": "HYBRID", "n_verified": 18, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": true}, "elapsed_s": 54.58, "error": null}, "legacy": {"audit_mode": "HYBRID", "n_verified": 3, "primary": {"title": "Doctor Syn", "uri": "https://en.wikipedia.org/wiki/Doctor_Syn", "used": false}, "elapsed_s": 13.31, "error": null}, "same_primary": true, "audit_delta": 0}