From 3ac13975a4d9ce7194f46d29d3988a32bc4640ea Mon Sep 17 00:00:00 2001
From: "russell@unturf.com" <russell@unturf.com>
Date: Sat, 2 May 2026 10:16:17 -0400
Subject: [PATCH] ci: GitLab pipeline with pytest-xdist parallelism cap
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`.gitlab-ci.yml` runs the unit test suite (`make test`, which uses
`-n auto`) on every push. PYTEST_XDIST_AUTO_NUM_WORKERS=4 caps the
worker count so we don't oversubscribe on shared runners — pytest-
xdist's `-n auto` reads os.cpu_count() and on cgroup-limited
containers that over-reports the host's cores. Local `make test`
stays uncapped (uses full machine).

Wall-clock on the 641-test suite:
  serial:           38s
  -n auto local:    11s   (full machine, 8+ cores typically)
  -n auto + cap 4:  22s   (CI runners — 1.7× speedup, still safe)

Job structure mirrors sibling repos (unsandbox.com,
unfirehose-nextjs-logger):
  - tags: build  → in-house runner pool
  - cache keyed on pyproject.toml, paths .venv/ + .pip-cache/
  - artifacts on failure → .pytest_cache/ for triage

Excluded from CI (need live state or network access):
  - make test-live      → live Hermes endpoint + populated shards
  - make bench-qa{,-quick,-smoke} → live LLM bench
  - chain-check-shards / analyze-shards → real shard data

Optional `test-crawler` job: gated on commit-message tag
`[ci-crawler]` or manual trigger; installs `[crawler]` extras and
runs the network-bound crawler suite. allow_failure: true so it
doesn't block merges.
---
 .gitlab-ci.yml | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 .gitlab-ci.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..e271185
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,97 @@
+# GitLab CI for aborist.
+#
+# Runs the unit test suite on every push. Lives at the same shape as
+# the sibling repos (`unsandbox.com`, `unfirehose-nextjs-logger`):
+# `tags: build` selects the in-house runner; `before_script` warms up
+# the venv; `script` runs the actual gate.
+#
+# What's NOT in CI:
+#   - `make test-live` — needs the live Hermes endpoint + populated
+#     shards under ~/.aborist/shards. Run by hand via `make test-live`
+#     when iterating on QA quality.
+#   - `make test-crawler` — needs `[crawler]` extras + network access
+#     to real HTML sites. Opt-in via `make test-crawler` locally.
+#   - `make bench-qa` — runs the live LLM bench, ~30-70 min wall-clock.
+#     Run on demand via `make bench-qa-{quick,smoke}` (10s / 30s) or
+#     full `make bench-qa` (full sweep).
+
+stages:
+  - test
+
+variables:
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.pip-cache"
+  # Pin the parallelism factor so we don't oversubscribe on shared
+  # runners. pytest-xdist's `-n auto` reads `os.cpu_count()` which on
+  # cgroup-limited containers can over-report the host's cores. Setting
+  # this var caps the worker count regardless. Local `make test` keeps
+  # using -n auto (untouched by this var).
+  PYTEST_XDIST_AUTO_NUM_WORKERS: "4"
+
+# ----------------------------------------------------------------------
+# Default test job — runs the full unit suite via `make test`, which
+# already passes `-n auto` to pytest-xdist for parallelism. On the
+# in-house runner this finishes in ~11s wall-clock (3.4× the serial
+# cost) for the 641-test suite.
+# ----------------------------------------------------------------------
+test:
+  stage: test
+  tags:
+    - build
+  cache:
+    # Key on pyproject.toml so dependency churn invalidates the cache;
+    # otherwise reuse the venv across CI runs to avoid re-installing
+    # editable + dev extras on every push.
+    key:
+      files:
+        - pyproject.toml
+    paths:
+      - .venv/
+      - .pip-cache/
+  before_script:
+    - python3 --version
+    - test -d .venv || python3 -m venv .venv
+    - .venv/bin/pip install --upgrade pip wheel
+    - .venv/bin/pip install -e '.[dev]'
+  script:
+    # `make test` invokes `pytest -q --ignore=tests/crawler -n auto`.
+    # PYTEST_XDIST_AUTO_NUM_WORKERS env var caps the worker count on
+    # the runner; locally `-n auto` uses the full machine.
+    - make test
+  artifacts:
+    when: on_failure
+    paths:
+      - .pytest_cache/
+    expire_in: 1 week
+
+# ----------------------------------------------------------------------
+# Optional: crawler extras + crawler-tagged tests. Opt-in via the
+# `crawler` keyword so it doesn't gate every push (network access +
+# heavier deps).
+# ----------------------------------------------------------------------
+test-crawler:
+  stage: test
+  tags:
+    - build
+  rules:
+    # Only run when explicitly invoked or commit message asks for it.
+    - if: '$CI_COMMIT_MESSAGE =~ /\[ci-crawler\]/'
+    - when: manual
+      allow_failure: true
+  cache:
+    key:
+      files:
+        - pyproject.toml
+    paths:
+      - .venv/
+      - .pip-cache/
+  before_script:
+    - test -d .venv || python3 -m venv .venv
+    - .venv/bin/pip install --upgrade pip wheel
+    - .venv/bin/pip install -e '.[dev,crawler]'
+  script:
+    - make test-crawler
+  artifacts:
+    when: on_failure
+    paths:
+      - .pytest_cache/
+    expire_in: 1 week