arborist/.gitlab-ci.yml

# GitLab CI for arborist.
#
# Runs the unit test suite on every push. Lives at the same shape as
# the sibling repos (`unsandbox.com`, `unfirehose-nextjs-logger`):
# `tags: build` selects the in-house runner; `before_script` warms up
# the venv; `script` runs the actual gate.
#
# What's NOT in CI:
#   - `make test-live` — needs the live Hermes endpoint + populated
#     shards under ~/.arborist/shards. Run by hand via `make test-live`
#     when iterating on QA quality.
#   - `make test-crawler` — needs `[crawler]` extras + network access
#     to real HTML sites. Opt-in via `make test-crawler` locally.
#   - `make bench-qa` — runs the live LLM bench, ~30-70 min wall-clock.
#     Run on demand via `make bench-qa-{quick,smoke}` (10s / 30s) or
#     full `make bench-qa` (full sweep).

# Pipeline RE-ENABLED 2026-05-09 with `test-ci` (skips wikipedia
# ingest tests; full suite still available locally via `make test`).
# Originally disabled 2026-05-02 because bench infrastructure
# overlapped with the runner pool; mitigation today is the
# `test-ci` scope cap + per-job parallelism cap
# (PYTEST_XDIST_AUTO_NUM_WORKERS=4) so CI doesn't crowd live bench
# work.

stages:
  - test

variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.pip-cache"
  # Pin the parallelism factor so we don't oversubscribe on shared
  # runners. pytest-xdist's `-n auto` reads `os.cpu_count()` which on
  # cgroup-limited containers can over-report the host's cores. Setting
  # this var caps the worker count regardless. Local `make test` keeps
  # using -n auto (untouched by this var).
  PYTEST_XDIST_AUTO_NUM_WORKERS: "4"

# ----------------------------------------------------------------------
# Default test job — runs the CI-scoped suite via `make test-ci`,
# which is `make test` minus tests/crawler (network) and the
# wikipedia ingest tests (`test_wikipedia_old.py`,
# `test_wikipedia_xml.py`; their runtime path is exercised end-to-end
# by real ingest under `make ingest-*` so the synthetic fixtures
# duplicate coverage). pytest-xdist `-n auto` parallelism is capped
# by PYTEST_XDIST_AUTO_NUM_WORKERS so cgroup-limited containers
# don't oversubscribe the host. Local `make test` stays
# comprehensive for the dev loop.
# ----------------------------------------------------------------------
test:
  stage: test
  tags:
    - build
  cache:
    # Key on pyproject.toml so dependency churn invalidates the cache;
    # otherwise reuse the venv across CI runs to avoid re-installing
    # editable + dev extras on every push.
    key:
      files:
        - pyproject.toml
    paths:
      - .venv/
      - .pip-cache/
  before_script:
    - python3 --version
    - test -d .venv || python3 -m venv .venv
    - .venv/bin/pip install --upgrade pip wheel
    - .venv/bin/pip install -e '.[dev]'
  script:
    # `make test-ci` invokes pytest with --ignore=tests/crawler
    # plus --ignore on the two wikipedia ingest test files.
    # PYTEST_XDIST_AUTO_NUM_WORKERS env var caps the worker count on
    # the runner; locally `-n auto` uses the full machine.
    - make test-ci
  artifacts:
    when: on_failure
    paths:
      - .pytest_cache/
    expire_in: 1 week

# ----------------------------------------------------------------------
# Bench gate — runs the complete Dav1DPrometheus suite (5S + 5T + 5F
# + 5R = 21 sub-batteries, 462+ deterministic fixtures) on every push.
# Job fails if any fixture fails. Bench result JSON is stored as a
# pipeline artifact so the v8-score job (below) can compare branches.
# Wall-clock ~3-5s on the in-house runner; cheaper than `make test`.
#
# Closes the loop from "we have a fitness substrate" (#000021,
# #000023-25) to "every push is gated by it." Phase 1 of #000012 +
# the 2026-05-08 fox roadmap note.
# ----------------------------------------------------------------------
bench-suite:
  stage: test
  tags:
    - build
  cache:
    key:
      files:
        - pyproject.toml
    paths:
      - .venv/
      - .pip-cache/
  before_script:
    - python3 --version
    - test -d .venv || python3 -m venv .venv
    - .venv/bin/pip install --upgrade pip wheel
    - .venv/bin/pip install -e '.[dev]'
  script:
    - mkdir -p bench/results
    - .venv/bin/python -m bench.batteries.runner --all --out bench/results/bench-${CI_COMMIT_SHORT_SHA}.json
  artifacts:
    paths:
      - bench/results/
    expire_in: 30 days

# ----------------------------------------------------------------------
# v8 ForkScore gate — for merge requests, score the bench delta
# between this branch and main. ACCEPT or MARGINAL pass; REJECT fails
# the job. Phase 1 of #000012's selection protocol; the consensus
# layer (validators, slashing, fork-choice) lives in the v8 paper
# itself.
#
# This job is `manual` for now — auto-gating waits until the v8 paper
# pins weight-set defaults for arborist's deployment shape. Operators
# trigger it on demand from the MR UI.
# ----------------------------------------------------------------------
v8-score:
  stage: test
  tags:
    - build
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: manual
      allow_failure: true
    - when: manual
      allow_failure: true
  needs:
    - job: bench-suite
      artifacts: true
  cache:
    key:
      files:
        - pyproject.toml
    paths:
      - .venv/
      - .pip-cache/
  before_script:
    - test -d .venv || python3 -m venv .venv
    - .venv/bin/pip install --upgrade pip wheel
    - .venv/bin/pip install -e '.[dev]'
  script:
    # Find this branch's bench result.
    - CHILD=$(ls -t bench/results/bench-*.json | head -1)
    # Pull main's most recent bench result via the latest pipeline
    # artifact. The runner exposes CI_API_V4_URL + CI_PROJECT_ID;
    # we curl the artifacts endpoint for the latest main pipeline's
    # bench-suite job.
    - |
      curl -sf -H "PRIVATE-TOKEN: ${CI_JOB_TOKEN}" \
        "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/artifacts/main/raw/bench/results/?job=bench-suite" \
        -o /tmp/parent-bench.zip 2>/dev/null && \
        unzip -p /tmp/parent-bench.zip 'bench/results/bench-*.json' \
          > /tmp/parent.json || \
        echo '{"schema_version":"bench-result-v1","results":[]}' > /tmp/parent.json
    # Score child vs parent. Exit 1 on REJECT (gates the merge).
    - .venv/bin/arborist v8 score --parent /tmp/parent.json --child "$CHILD"

# ----------------------------------------------------------------------
# Optional: crawler extras + crawler-tagged tests. Opt-in via the
# `crawler` keyword so it doesn't gate every push (network access +
# heavier deps).
# ----------------------------------------------------------------------
test-crawler:
  stage: test
  tags:
    - build
  rules:
    # Only run when explicitly invoked or commit message asks for it.
    - if: '$CI_COMMIT_MESSAGE =~ /\[ci-crawler\]/'
    - when: manual
      allow_failure: true
  cache:
    key:
      files:
        - pyproject.toml
    paths:
      - .venv/
      - .pip-cache/
  before_script:
    - test -d .venv || python3 -m venv .venv
    - .venv/bin/pip install --upgrade pip wheel
    - .venv/bin/pip install -e '.[dev,crawler]'
  script:
    - make test-crawler
  artifacts:
    when: on_failure
    paths:
      - .pytest_cache/
    expire_in: 1 week