feat(reports): add API health tracking to perf reports
- generate-perf-report.sh now collects api-health.json from test artifacts - Aggregates retry counts by type (429, 5xx, timeout, connection) - Calculates API health score (100 = perfect, decreases with retries) - Adds API Health section to perf.md with interpretation guide - aggregate-performance-reports.py now shows API health trends over time - Tracks scientific integrity improvements starting from 4.2.34
This commit is contained in:
parent
ee43329a55
commit
8da77c6dd1
2 changed files with 162 additions and 0 deletions
|
|
@ -69,6 +69,44 @@ def extract_language_timings(perf_data):
|
|||
return langs
|
||||
|
||||
|
||||
def analyze_api_health(reports):
|
||||
"""Analyze API health trends across releases"""
|
||||
health_data = {}
|
||||
|
||||
for version, perf_data in reports.items():
|
||||
api_health = perf_data.get("api_health", {})
|
||||
if api_health:
|
||||
health_data[version] = {
|
||||
"score": api_health.get("score", 100),
|
||||
"total_retries": api_health.get("total_retries", 0),
|
||||
"rate_limit_429": api_health.get("retries_by_type", {}).get("rate_limit_429", 0),
|
||||
"server_error_5xx": api_health.get("retries_by_type", {}).get("server_error_5xx", 0),
|
||||
"timeout": api_health.get("retries_by_type", {}).get("timeout", 0),
|
||||
"connection": api_health.get("retries_by_type", {}).get("connection", 0),
|
||||
"tests_with_retries": api_health.get("tests_with_retries", 0),
|
||||
}
|
||||
|
||||
if not health_data:
|
||||
return None
|
||||
|
||||
# Calculate trends
|
||||
versions = sorted(health_data.keys())
|
||||
scores = [health_data[v]["score"] for v in versions]
|
||||
retries = [health_data[v]["total_retries"] for v in versions]
|
||||
|
||||
return {
|
||||
"per_version": health_data,
|
||||
"versions": versions,
|
||||
"scores": scores,
|
||||
"avg_score": mean(scores) if scores else 100,
|
||||
"min_score": min(scores) if scores else 100,
|
||||
"max_score": max(scores) if scores else 100,
|
||||
"total_retries_all_versions": sum(retries),
|
||||
"avg_retries_per_version": mean(retries) if retries else 0,
|
||||
"trend": "improving" if len(scores) >= 2 and scores[-1] > scores[0] else "degrading" if len(scores) >= 2 and scores[-1] < scores[0] else "stable",
|
||||
}
|
||||
|
||||
|
||||
def analyze_variance(reports):
|
||||
"""Analyze performance variance across releases"""
|
||||
|
||||
|
|
@ -255,6 +293,7 @@ def generate_report(reports, output_file, analysis=None):
|
|||
if analysis is None:
|
||||
analysis = analyze_variance(reports)
|
||||
concurrency = detect_concurrency_pattern(reports)
|
||||
api_health = analyze_api_health(reports)
|
||||
|
||||
versions = sorted(reports.keys())
|
||||
version_dates = {v: reports[v].get("timestamp", "unknown") for v in versions}
|
||||
|
|
@ -352,6 +391,37 @@ The same language changes dramatically in rank between runs:
|
|||
|
||||
---
|
||||
|
||||
### 4. API Health Trends
|
||||
|
||||
"""
|
||||
|
||||
if api_health:
|
||||
report += f"""**Overall API Health:** {api_health['avg_score']:.1f}/100 (avg across {len(api_health['versions'])} releases)
|
||||
**Trend:** {api_health['trend'].upper()}
|
||||
**Total Retries (all releases):** {api_health['total_retries_all_versions']}
|
||||
|
||||
| Release | Health Score | Total Retries | 429 (Rate Limit) | 5xx (Server) | Timeout | Connection |
|
||||
|---------|--------------|---------------|------------------|--------------|---------|------------|
|
||||
"""
|
||||
for v in api_health['versions']:
|
||||
h = api_health['per_version'][v]
|
||||
report += f"| {v} | {h['score']}/100 | {h['total_retries']} | {h['rate_limit_429']} | {h['server_error_5xx']} | {h['timeout']} | {h['connection']} |\n"
|
||||
|
||||
report += f"""
|
||||
**Interpretation:**
|
||||
- **Score 95-100:** API healthy, tests pass on first attempt
|
||||
- **Score 80-94:** Some transient errors, tests recovered via retry
|
||||
- **Score < 80:** Significant API instability affecting test reliability
|
||||
|
||||
**Scientific Integrity Note:** Prior to 4.2.34, tests used "soft passes" that masked failures.
|
||||
Now tests retry transient errors and fail honestly if they can't verify results.
|
||||
"""
|
||||
else:
|
||||
report += "*No API health data available for analyzed releases.*\n"
|
||||
|
||||
report += f"""
|
||||
---
|
||||
|
||||
## The Orchestrator Problem: DevOps 101
|
||||
|
||||
### Why This Matters
|
||||
|
|
|
|||
|
|
@ -161,6 +161,65 @@ MIN_DURATION=$(echo "$PERF_JSON" | jq '[.[].duration_seconds] | min')
|
|||
SLOWEST_LANG=$(echo "$PERF_JSON" | jq -r '.[0].language')
|
||||
FASTEST_LANG=$(echo "$PERF_JSON" | jq -r '.[-1].language')
|
||||
|
||||
# ============================================================================
|
||||
# Collect API Health Data from Test Artifacts
|
||||
# ============================================================================
|
||||
log_info "Collecting API health data from test artifacts..."
|
||||
|
||||
# Initialize API health counters
|
||||
TOTAL_RETRIES=0
|
||||
RETRIES_429=0
|
||||
RETRIES_5XX=0
|
||||
RETRIES_TIMEOUT=0
|
||||
RETRIES_CONN=0
|
||||
TESTS_WITH_RETRIES=0
|
||||
API_HEALTH_DATA="[]"
|
||||
|
||||
# Download api-health.json from each test job artifact
|
||||
for job_id in $(echo "$JOBS_JSON" | jq -r '.[] | select(.name | startswith("test: [")) | .id'); do
|
||||
LANG=$(echo "$JOBS_JSON" | jq -r ".[] | select(.id == $job_id) | .name | gsub(\"test: \\\\[|\\\\]\"; \"\")")
|
||||
|
||||
# Try to download the api-health.json artifact
|
||||
ARTIFACT_URL="$GITLAB_URL/$PROJECT_PATH/-/jobs/$job_id/artifacts/raw/test-results-$LANG/api-health.json"
|
||||
HEALTH_JSON=$(curl -s "$ARTIFACT_URL" 2>/dev/null || echo "{}")
|
||||
|
||||
if echo "$HEALTH_JSON" | jq -e '.retries' > /dev/null 2>&1; then
|
||||
# Extract and accumulate retry counts
|
||||
JOB_RETRIES=$(echo "$HEALTH_JSON" | jq -r '.retries.total // 0')
|
||||
JOB_429=$(echo "$HEALTH_JSON" | jq -r '.retries.rate_limit_429 // 0')
|
||||
JOB_5XX=$(echo "$HEALTH_JSON" | jq -r '.retries.server_error_5xx // 0')
|
||||
JOB_TIMEOUT=$(echo "$HEALTH_JSON" | jq -r '.retries.timeout // 0')
|
||||
JOB_CONN=$(echo "$HEALTH_JSON" | jq -r '.retries.connection // 0')
|
||||
JOB_TESTS_WITH_RETRIES=$(echo "$HEALTH_JSON" | jq -r '.tests_with_retries // 0')
|
||||
|
||||
TOTAL_RETRIES=$((TOTAL_RETRIES + JOB_RETRIES))
|
||||
RETRIES_429=$((RETRIES_429 + JOB_429))
|
||||
RETRIES_5XX=$((RETRIES_5XX + JOB_5XX))
|
||||
RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + JOB_TIMEOUT))
|
||||
RETRIES_CONN=$((RETRIES_CONN + JOB_CONN))
|
||||
TESTS_WITH_RETRIES=$((TESTS_WITH_RETRIES + JOB_TESTS_WITH_RETRIES))
|
||||
|
||||
# Add to per-language health data
|
||||
API_HEALTH_DATA=$(echo "$API_HEALTH_DATA" | jq ". + [{
|
||||
\"language\": \"$LANG\",
|
||||
\"retries\": $JOB_RETRIES,
|
||||
\"rate_limit_429\": $JOB_429,
|
||||
\"server_error_5xx\": $JOB_5XX,
|
||||
\"timeout\": $JOB_TIMEOUT,
|
||||
\"connection\": $JOB_CONN,
|
||||
\"tests_with_retries\": $JOB_TESTS_WITH_RETRIES
|
||||
}]")
|
||||
fi
|
||||
done
|
||||
|
||||
# Calculate API health score (100 = perfect, decreases with retries)
|
||||
API_HEALTH_SCORE=$(echo "scale=1; 100 - ($TOTAL_RETRIES * 2)" | bc 2>/dev/null || echo "100")
|
||||
if [ "$(echo "$API_HEALTH_SCORE < 0" | bc)" -eq 1 ]; then
|
||||
API_HEALTH_SCORE="0"
|
||||
fi
|
||||
|
||||
log_info "API Health: $TOTAL_RETRIES total retries (429: $RETRIES_429, 5xx: $RETRIES_5XX, timeout: $RETRIES_TIMEOUT, conn: $RETRIES_CONN)"
|
||||
|
||||
# Write JSON report
|
||||
cat > "$REPORT_DIR/perf.json" << EOF
|
||||
{
|
||||
|
|
@ -180,6 +239,18 @@ cat > "$REPORT_DIR/perf.json" << EOF
|
|||
"slowest_language": "$SLOWEST_LANG",
|
||||
"fastest_language": "$FASTEST_LANG"
|
||||
},
|
||||
"api_health": {
|
||||
"score": $API_HEALTH_SCORE,
|
||||
"total_retries": $TOTAL_RETRIES,
|
||||
"retries_by_type": {
|
||||
"rate_limit_429": $RETRIES_429,
|
||||
"server_error_5xx": $RETRIES_5XX,
|
||||
"timeout": $RETRIES_TIMEOUT,
|
||||
"connection": $RETRIES_CONN
|
||||
},
|
||||
"tests_with_retries": $TESTS_WITH_RETRIES,
|
||||
"per_language": $API_HEALTH_DATA
|
||||
},
|
||||
"languages": $PERF_JSON
|
||||
}
|
||||
EOF
|
||||
|
|
@ -210,6 +281,27 @@ cat > "$REPORT_DIR/perf.md" << EOF
|
|||
|
||||
---
|
||||
|
||||
## API Health
|
||||
|
||||
Tracks transient errors encountered during test execution. Tests retry on failures to ensure accurate results.
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Health Score | ${API_HEALTH_SCORE}/100 |
|
||||
| Total Retries | $TOTAL_RETRIES |
|
||||
| Rate Limit (429) | $RETRIES_429 |
|
||||
| Server Error (5xx) | $RETRIES_5XX |
|
||||
| Timeout | $RETRIES_TIMEOUT |
|
||||
| Connection | $RETRIES_CONN |
|
||||
| Tests Needing Retries | $TESTS_WITH_RETRIES |
|
||||
|
||||
**Interpretation:**
|
||||
- **Score 95-100:** API is healthy, minimal transient errors
|
||||
- **Score 80-94:** Some API instability, but tests recovered via retry
|
||||
- **Score < 80:** Significant API issues affecting test reliability
|
||||
|
||||
---
|
||||
|
||||
## Test Duration by Language
|
||||
|
||||
The primary performance metric - how long each language takes to run its full test suite (15 tests per language).
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue