From 8da77c6dd16ab654080db65ac740327cb8ffa91c Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 28 Jan 2026 18:40:10 -0500 Subject: [PATCH] feat(reports): add API health tracking to perf reports - generate-perf-report.sh now collects api-health.json from test artifacts - Aggregates retry counts by type (429, 5xx, timeout, connection) - Calculates API health score (100 = perfect, decreases with retries) - Adds API Health section to perf.md with interpretation guide - aggregate-performance-reports.py now shows API health trends over time - Tracks scientific integrity improvements starting from 4.2.34 --- scripts/aggregate-performance-reports.py | 70 ++++++++++++++++++ scripts/generate-perf-report.sh | 92 ++++++++++++++++++++++++ 2 files changed, 162 insertions(+) diff --git a/scripts/aggregate-performance-reports.py b/scripts/aggregate-performance-reports.py index fc485d0..59b8f4d 100644 --- a/scripts/aggregate-performance-reports.py +++ b/scripts/aggregate-performance-reports.py @@ -69,6 +69,44 @@ def extract_language_timings(perf_data): return langs +def analyze_api_health(reports): + """Analyze API health trends across releases""" + health_data = {} + + for version, perf_data in reports.items(): + api_health = perf_data.get("api_health", {}) + if api_health: + health_data[version] = { + "score": api_health.get("score", 100), + "total_retries": api_health.get("total_retries", 0), + "rate_limit_429": api_health.get("retries_by_type", {}).get("rate_limit_429", 0), + "server_error_5xx": api_health.get("retries_by_type", {}).get("server_error_5xx", 0), + "timeout": api_health.get("retries_by_type", {}).get("timeout", 0), + "connection": api_health.get("retries_by_type", {}).get("connection", 0), + "tests_with_retries": api_health.get("tests_with_retries", 0), + } + + if not health_data: + return None + + # Calculate trends + versions = sorted(health_data.keys()) + scores = [health_data[v]["score"] for v in versions] + retries = [health_data[v]["total_retries"] for v in versions] + + return { + "per_version": health_data, + "versions": versions, + "scores": scores, + "avg_score": mean(scores) if scores else 100, + "min_score": min(scores) if scores else 100, + "max_score": max(scores) if scores else 100, + "total_retries_all_versions": sum(retries), + "avg_retries_per_version": mean(retries) if retries else 0, + "trend": "improving" if len(scores) >= 2 and scores[-1] > scores[0] else "degrading" if len(scores) >= 2 and scores[-1] < scores[0] else "stable", + } + + def analyze_variance(reports): """Analyze performance variance across releases""" @@ -255,6 +293,7 @@ def generate_report(reports, output_file, analysis=None): if analysis is None: analysis = analyze_variance(reports) concurrency = detect_concurrency_pattern(reports) + api_health = analyze_api_health(reports) versions = sorted(reports.keys()) version_dates = {v: reports[v].get("timestamp", "unknown") for v in versions} @@ -352,6 +391,37 @@ The same language changes dramatically in rank between runs: --- +### 4. API Health Trends + +""" + + if api_health: + report += f"""**Overall API Health:** {api_health['avg_score']:.1f}/100 (avg across {len(api_health['versions'])} releases) +**Trend:** {api_health['trend'].upper()} +**Total Retries (all releases):** {api_health['total_retries_all_versions']} + +| Release | Health Score | Total Retries | 429 (Rate Limit) | 5xx (Server) | Timeout | Connection | +|---------|--------------|---------------|------------------|--------------|---------|------------| +""" + for v in api_health['versions']: + h = api_health['per_version'][v] + report += f"| {v} | {h['score']}/100 | {h['total_retries']} | {h['rate_limit_429']} | {h['server_error_5xx']} | {h['timeout']} | {h['connection']} |\n" + + report += f""" +**Interpretation:** +- **Score 95-100:** API healthy, tests pass on first attempt +- **Score 80-94:** Some transient errors, tests recovered via retry +- **Score < 80:** Significant API instability affecting test reliability + +**Scientific Integrity Note:** Prior to 4.2.34, tests used "soft passes" that masked failures. +Now tests retry transient errors and fail honestly if they can't verify results. +""" + else: + report += "*No API health data available for analyzed releases.*\n" + + report += f""" +--- + ## The Orchestrator Problem: DevOps 101 ### Why This Matters diff --git a/scripts/generate-perf-report.sh b/scripts/generate-perf-report.sh index 53f38f6..835fc37 100755 --- a/scripts/generate-perf-report.sh +++ b/scripts/generate-perf-report.sh @@ -161,6 +161,65 @@ MIN_DURATION=$(echo "$PERF_JSON" | jq '[.[].duration_seconds] | min') SLOWEST_LANG=$(echo "$PERF_JSON" | jq -r '.[0].language') FASTEST_LANG=$(echo "$PERF_JSON" | jq -r '.[-1].language') +# ============================================================================ +# Collect API Health Data from Test Artifacts +# ============================================================================ +log_info "Collecting API health data from test artifacts..." + +# Initialize API health counters +TOTAL_RETRIES=0 +RETRIES_429=0 +RETRIES_5XX=0 +RETRIES_TIMEOUT=0 +RETRIES_CONN=0 +TESTS_WITH_RETRIES=0 +API_HEALTH_DATA="[]" + +# Download api-health.json from each test job artifact +for job_id in $(echo "$JOBS_JSON" | jq -r '.[] | select(.name | startswith("test: [")) | .id'); do + LANG=$(echo "$JOBS_JSON" | jq -r ".[] | select(.id == $job_id) | .name | gsub(\"test: \\\\[|\\\\]\"; \"\")") + + # Try to download the api-health.json artifact + ARTIFACT_URL="$GITLAB_URL/$PROJECT_PATH/-/jobs/$job_id/artifacts/raw/test-results-$LANG/api-health.json" + HEALTH_JSON=$(curl -s "$ARTIFACT_URL" 2>/dev/null || echo "{}") + + if echo "$HEALTH_JSON" | jq -e '.retries' > /dev/null 2>&1; then + # Extract and accumulate retry counts + JOB_RETRIES=$(echo "$HEALTH_JSON" | jq -r '.retries.total // 0') + JOB_429=$(echo "$HEALTH_JSON" | jq -r '.retries.rate_limit_429 // 0') + JOB_5XX=$(echo "$HEALTH_JSON" | jq -r '.retries.server_error_5xx // 0') + JOB_TIMEOUT=$(echo "$HEALTH_JSON" | jq -r '.retries.timeout // 0') + JOB_CONN=$(echo "$HEALTH_JSON" | jq -r '.retries.connection // 0') + JOB_TESTS_WITH_RETRIES=$(echo "$HEALTH_JSON" | jq -r '.tests_with_retries // 0') + + TOTAL_RETRIES=$((TOTAL_RETRIES + JOB_RETRIES)) + RETRIES_429=$((RETRIES_429 + JOB_429)) + RETRIES_5XX=$((RETRIES_5XX + JOB_5XX)) + RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + JOB_TIMEOUT)) + RETRIES_CONN=$((RETRIES_CONN + JOB_CONN)) + TESTS_WITH_RETRIES=$((TESTS_WITH_RETRIES + JOB_TESTS_WITH_RETRIES)) + + # Add to per-language health data + API_HEALTH_DATA=$(echo "$API_HEALTH_DATA" | jq ". + [{ + \"language\": \"$LANG\", + \"retries\": $JOB_RETRIES, + \"rate_limit_429\": $JOB_429, + \"server_error_5xx\": $JOB_5XX, + \"timeout\": $JOB_TIMEOUT, + \"connection\": $JOB_CONN, + \"tests_with_retries\": $JOB_TESTS_WITH_RETRIES + }]") + fi +done + +# Calculate API health score (100 = perfect, decreases with retries) +API_HEALTH_SCORE=$(echo "scale=1; 100 - ($TOTAL_RETRIES * 2)" | bc 2>/dev/null || echo "100") +if [ "$(echo "$API_HEALTH_SCORE < 0" | bc)" -eq 1 ]; then + API_HEALTH_SCORE="0" +fi + +log_info "API Health: $TOTAL_RETRIES total retries (429: $RETRIES_429, 5xx: $RETRIES_5XX, timeout: $RETRIES_TIMEOUT, conn: $RETRIES_CONN)" + # Write JSON report cat > "$REPORT_DIR/perf.json" << EOF { @@ -180,6 +239,18 @@ cat > "$REPORT_DIR/perf.json" << EOF "slowest_language": "$SLOWEST_LANG", "fastest_language": "$FASTEST_LANG" }, + "api_health": { + "score": $API_HEALTH_SCORE, + "total_retries": $TOTAL_RETRIES, + "retries_by_type": { + "rate_limit_429": $RETRIES_429, + "server_error_5xx": $RETRIES_5XX, + "timeout": $RETRIES_TIMEOUT, + "connection": $RETRIES_CONN + }, + "tests_with_retries": $TESTS_WITH_RETRIES, + "per_language": $API_HEALTH_DATA + }, "languages": $PERF_JSON } EOF @@ -210,6 +281,27 @@ cat > "$REPORT_DIR/perf.md" << EOF --- +## API Health + +Tracks transient errors encountered during test execution. Tests retry on failures to ensure accurate results. + +| Metric | Value | +|--------|-------| +| Health Score | ${API_HEALTH_SCORE}/100 | +| Total Retries | $TOTAL_RETRIES | +| Rate Limit (429) | $RETRIES_429 | +| Server Error (5xx) | $RETRIES_5XX | +| Timeout | $RETRIES_TIMEOUT | +| Connection | $RETRIES_CONN | +| Tests Needing Retries | $TESTS_WITH_RETRIES | + +**Interpretation:** +- **Score 95-100:** API is healthy, minimal transient errors +- **Score 80-94:** Some API instability, but tests recovered via retry +- **Score < 80:** Significant API issues affecting test reliability + +--- + ## Test Duration by Language The primary performance metric - how long each language takes to run its full test suite (15 tests per language).