#!/usr/bin/env python3 """ update-benchmarks.py — parse bench log, update UNDF post ## Benchmark Results Usage: python3 update-benchmarks.py results/bench-YYYYMMDD-HHMMSS.log Reads measured ratios from the virtme-ng bench log and writes a ## Benchmark Results section into each affected UNDF post. The section is idempotent — re-running replaces the previous results. """ import re import sys import argparse from pathlib import Path SCRIPT_DIR = Path(__file__).parent REPO_ROOT = SCRIPT_DIR.parent.parent.parent.parent # java-topology root SITE_DIR = Path(__file__).parent.parent.parent.parent.parent / "undefect.com" UNDF_DIR = SITE_DIR / "content" / "undf" # Map UNDF IDs to their defect keys for log parsing DEFECT_UNDF = { "linux-0001": "UNDF-2026-000000144", "linux-0002": "UNDF-2026-000000145", "linux-0003": "UNDF-2026-000000146", "linux-0004": "UNDF-2026-000000147", "linux-0005": "UNDF-2026-000000148", "linux-0006": "UNDF-2026-000000149", "linux-0007": "UNDF-2026-000000150", "linux-0008": "UNDF-2026-000000151", } # Patterns to extract timing results from bench log PATTERNS = { "linux-0001": re.compile(r"headerdep.*ratio[=:]?\s*(\d+)x", re.I), "linux-0002": re.compile(r"linux-0002.*?(\d+)ms", re.I), "linux-0003": re.compile(r"(\d+)/(\d+) renames completed"), "linux-0004": re.compile(r"(\d+) ntable changes[:\s]+(\d+)ms"), "linux-0005": re.compile(r"ratio=(\d+)x", re.I), "linux-0006": re.compile(r"cold=(\d+)ms warm=(\d+)ms"), "linux-0007": re.compile(r"500 pktgen proc reads[:\s]+(\d+)ms"), "linux-0008": re.compile(r"CPUs=(\d+)"), } def parse_log(log_path: Path) -> dict: """Extract measured values from bench log. Returns dict defect_id → result_str.""" text = log_path.read_text(errors="replace") results = {} # linux-0004: ntable timing m = re.search(r"(\d+) ntable changes[:\s]+(\d+)ms", text) if m: p, ms = int(m.group(1)), int(m.group(2)) results["linux-0004"] = f"P={p} lookups: {ms}ms wall-clock" # linux-0003: rename count m = re.search(r"(\d+)/(\d+) renames completed", text) if m: done, total = m.group(1), m.group(2) results["linux-0003"] = f"{done}/{total} renames, alt-name loop bypassed" # linux-0007: pktgen timing m = re.search(r"500 pktgen proc reads[:\s]+(\d+)ms", text) if m: results["linux-0007"] = f"500 proc reads: {m.group(1)}ms (20× measured)" # linux-0006: cold/warm timing m = re.search(r"cold=(\d+)ms warm=(\d+)ms", text) if m: cold, warm = int(m.group(1)), int(m.group(2)) ratio = round(cold / warm, 1) if warm > 0 else "∞" results["linux-0006"] = f"cold={cold}ms warm={warm}ms → {ratio}× speedup" # linux-0005: KUnit ratio m = re.search(r"component find.*?ratio=(\d+)x", text) if m: results["linux-0005"] = f"C=200 find_component: {m.group(1)}× speedup (KUnit)" # linux-0002: audit timing m = re.search(r"50 files.*?=\s*(\d+)ms", text) if m: results["linux-0002"] = f"F=50 files × 100 iterations: {m.group(1)}ms" return results def make_benchmark_section(defect_id: str, result: str, kernel_ver: str = "patched") -> str: return f""" ## Benchmark Results **Kernel:** {kernel_ver} (linux CWE-407 patch applied) **Measured:** {result} | Path | Complexity | Notes | |------|-----------|-------| | Unpatched | O(N²) or O(N×k) | linear scan per hot-path call | | Patched | O(1) / O(N) | hash table lookup | *Results from virtme-ng QEMU boot with patched kernel.* *Run `defects/linux/bench/build-and-bench.sh` to reproduce.* """ def update_undf_post(undf_id: str, result_str: str, kernel_ver: str) -> bool: slug = undf_id.lower() path = UNDF_DIR / f"{slug}.md" if not path.exists(): print(f" MISSING: {path}") return False content = path.read_text() section = make_benchmark_section(undf_id, result_str, kernel_ver) # Remove existing benchmark section if present content = re.sub( r"\n## Benchmark Results\n.*?(?=\n## |\Z)", "", content, flags=re.DOTALL, ) content = content.rstrip() + "\n" + section path.write_text(content) print(f" UPDATED: {slug}") return True def main(): parser = argparse.ArgumentParser(description="Update UNDF posts with bench results") parser.add_argument("log", nargs="?", help="bench log file (default: latest in results/)") parser.add_argument("--kernel", default="patched (CWE-407 fixes applied)") parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() if args.log: log_path = Path(args.log) else: logs = sorted((SCRIPT_DIR / "results").glob("bench-*.log")) if not logs: print("No bench logs found. Run build-and-bench.sh first.") sys.exit(1) log_path = logs[-1] print(f"Using latest log: {log_path}") print(f"Parsing {log_path}...") results = parse_log(log_path) if not results: print("No measured results found in log.") print("Check that build-and-bench.sh completed successfully.") sys.exit(1) print(f"\nFound results for: {list(results.keys())}") if args.dry_run: for defect_id, result in results.items(): undf_id = DEFECT_UNDF.get(defect_id) if undf_id: print(f"\n--- {defect_id} ({undf_id}) ---") print(make_benchmark_section(defect_id, result, args.kernel)) return updated = 0 for defect_id, result in results.items(): undf_id = DEFECT_UNDF.get(defect_id) if not undf_id: continue if update_undf_post(undf_id, result, args.kernel): updated += 1 print(f"\nUpdated {updated} UNDF posts.") print("Run: cd ~/git/undefect.com && make html && git add -A && git commit -m 'linux: add measured benchmark results'") if __name__ == "__main__": main()