feat: add scan_verify.py + check_coverage.py for all 1258 UNDF indicators
scan_verify.py: structural patch verification for all UNDF registry entries.
- Reads UNDF-REGISTRY.json, walks defects/*/patch/*.patch
- Checks: UNDF header comment, defective pattern in removed lines,
fix signature in added lines (HashSet/unordered_set/HashMap/etc.)
- Slug-to-patch filtering prevents cross-contamination in multi-entry dirs
- Additive patches (guard insertion) treated as WARN not FAIL
- Status: 454 PASS / 441 WARN / 12 FAIL / 300 NO_PATCH / 51 PENDING
- 74.2% structural coverage of clonable entries
check_coverage.py: CI gate — fails when new registry entries lack patch files.
- Any NO_PATCH entry (dir exists but no patch) causes exit code 1
- Designed to run on every commit to catch regressions early
- Quality (WARN/PASS) delegated to scan_verify.py
Makefile targets added:
make scan-verify — verbose structural verification (all 1258 entries)
make scan-verify-report — same + writes tests/SCAN-VERIFY-REPORT.md
make scan-verify-strict -- also fail on WARN
make coverage-check — CI gate for patch presence
Completes the three-tier coverage system:
unit/integration/functional Java tests (MOADs 0001-0011)
+ structural patch verification (1258+ UNDF indicators)
+ coverage CI gate (enforces no indicator left without a patch)
This commit is contained in:
parent
4888153e40
commit
0ac47a5b75
4 changed files with 2331 additions and 1 deletions
145
tests/check_coverage.py
Normal file
145
tests/check_coverage.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
check_coverage.py — CI gate for UNDF registry patch coverage.
|
||||
|
||||
Enforces the rule: every UNDF entry in UNDF-REGISTRY.json must have at least
|
||||
a patch file present in its defect directory. This is the minimum bar for a
|
||||
new indicator to be considered "covered." Structural quality (UNDF header,
|
||||
defective/fix signatures) is enforced separately by scan_verify.py.
|
||||
|
||||
This script is designed to run on every commit to catch regressions early:
|
||||
- New entry added to UNDF-REGISTRY.json without a patch file → FAIL
|
||||
- Existing patch deleted without removing the registry entry → FAIL
|
||||
- Entry has a patch file (even a stub) → PASS (quality via scan_verify)
|
||||
|
||||
Exit codes:
|
||||
0 — all entries have at least one patch file (or are pending source clone)
|
||||
1 — one or more entries are missing patch files (NO_PATCH status)
|
||||
2 — registry not found or unreadable
|
||||
|
||||
Usage:
|
||||
python3 tests/check_coverage.py # from repo root
|
||||
python3 tests/check_coverage.py --verbose # list NO_PATCH entries
|
||||
python3 tests/check_coverage.py --include-pending # also fail on PENDING
|
||||
|
||||
Run via Makefile:
|
||||
make coverage-check
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
# ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
REPO_ROOT = Path(__file__).parent.parent # java-topology/
|
||||
REGISTRY = REPO_ROOT / "UNDF-REGISTRY.json"
|
||||
DEFECTS_DIR = REPO_ROOT / "defects"
|
||||
|
||||
|
||||
# ── Reuse patch-finding logic from scan_verify ─────────────────────────────────
|
||||
|
||||
def find_patch_files_for_slug(slug: str) -> list:
|
||||
"""Return patch files belonging to this slug, or [] if none found."""
|
||||
proj_dir = slug.rsplit("-", 1)[0]
|
||||
patch_dir = DEFECTS_DIR / proj_dir / "patch"
|
||||
if not patch_dir.is_dir():
|
||||
return [] # PENDING — source not cloned
|
||||
|
||||
all_patches = [p for p in patch_dir.iterdir() if p.suffix == ".patch"]
|
||||
if not all_patches:
|
||||
return []
|
||||
|
||||
entry_suffix = slug.rsplit("-", 1)[-1]
|
||||
matched = [p for p in all_patches if entry_suffix in p.stem]
|
||||
return sorted(matched)
|
||||
|
||||
|
||||
def proj_dir_exists(slug: str) -> bool:
|
||||
"""True if the project directory exists (source cloned)."""
|
||||
proj_dir = slug.rsplit("-", 1)[0]
|
||||
return (DEFECTS_DIR / proj_dir).is_dir()
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.strip().split("\n")[0])
|
||||
parser.add_argument("--verbose", action="store_true", help="List NO_PATCH entries")
|
||||
parser.add_argument("--include-pending", action="store_true", help="Also fail on PENDING (source not cloned)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not REGISTRY.exists():
|
||||
print(f"ERROR: Registry not found at {REGISTRY}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
with REGISTRY.open() as f:
|
||||
registry = json.load(f)
|
||||
|
||||
no_patch = []
|
||||
pending = []
|
||||
covered = []
|
||||
|
||||
for slug, undf_id in sorted(registry.items()):
|
||||
if not proj_dir_exists(slug):
|
||||
pending.append((slug, undf_id))
|
||||
continue
|
||||
|
||||
patches = find_patch_files_for_slug(slug)
|
||||
if patches:
|
||||
covered.append((slug, undf_id, patches))
|
||||
else:
|
||||
no_patch.append((slug, undf_id))
|
||||
|
||||
total = len(registry)
|
||||
n_covered = len(covered)
|
||||
n_nopatch = len(no_patch)
|
||||
n_pending = len(pending)
|
||||
|
||||
pct = 100 * n_covered / (total - n_pending) if total > n_pending else 100.0
|
||||
|
||||
print(f"COVERAGE CHECK — {total} UNDF entries")
|
||||
print(f" Covered (patch present): {n_covered}")
|
||||
print(f" NO_PATCH (dir exists, no patch): {n_nopatch}")
|
||||
print(f" PENDING (source not cloned): {n_pending}")
|
||||
print(f" Coverage: {n_covered}/{total - n_pending} ({pct:.1f}%) of clonable entries")
|
||||
print()
|
||||
|
||||
if args.verbose and no_patch:
|
||||
print("NO_PATCH entries (patch file required):")
|
||||
for slug, undf_id in no_patch:
|
||||
proj_dir = slug.rsplit("-", 1)[0]
|
||||
print(f" {undf_id} {slug} → defects/{proj_dir}/patch/")
|
||||
print()
|
||||
|
||||
if args.verbose and args.include_pending and pending:
|
||||
print("PENDING entries (source not yet cloned):")
|
||||
for slug, undf_id in pending:
|
||||
proj_dir = slug.rsplit("-", 1)[0]
|
||||
print(f" {undf_id} {slug} → defects/{proj_dir}/")
|
||||
print()
|
||||
|
||||
fail = n_nopatch > 0
|
||||
if args.include_pending:
|
||||
fail = fail or n_pending > 0
|
||||
|
||||
if fail:
|
||||
reasons = []
|
||||
if n_nopatch:
|
||||
reasons.append(f"{n_nopatch} entries missing patch files")
|
||||
if args.include_pending and n_pending:
|
||||
reasons.append(f"{n_pending} entries pending source clone")
|
||||
print(f"RESULT: FAIL — {'; '.join(reasons)}")
|
||||
print(" Add patch files for NO_PATCH entries to pass this gate.")
|
||||
print(" Structural quality: run 'make scan-verify' for header/sig checks.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"RESULT: PASS — all {n_covered} clonable entries have patch files")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue