From 1035d256fef6ac2bd76bc723d65f71bc3e2781d8 Mon Sep 17 00:00:00 2001
From: "russell@unturf.com" <russell@unturf.com>
Date: Thu, 14 May 2026 11:16:20 -0400
Subject: [PATCH] =?UTF-8?q?ci:=20clean=20stale=20editable-install=20metada?=
 =?UTF-8?q?ta=20before=20pip=20install=20=E2=80=94=20fixes=20OSError=20whe?=
 =?UTF-8?q?n=20cached=20.venv/=20is=20restored=20into=20a=20different=20gi?=
 =?UTF-8?q?tlab-runner=20build=20dir?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

gitlab-runner rotates between build dirs (builds/RUNNER_ID/0, /1, /3, ...). The cached .venv/ embeds the absolute build-dir path into __editable__.arborist-0.0.1.pth + the dist-info RECORD via pip's editable install. When the cache is restored into a different build dir, pip's implicit uninstall-then-reinstall step fails with `OSError: No such file or directory` looking for files at the old build dir.

Fix: shared .warm-venv-setup hidden job referenced from every real job's before_script. It wipes arborist-*.dist-info, the __editable__*.pth marker, the finder, and the bin/arborist entry script before pip install — so pip does a fresh install at the current build dir. Cache stays warm (deps don't re-download), only the editable-install bookkeeping is rebuilt (~1s per job).

DRY'd via `!reference [.warm-venv-setup, before_script]` so test / bench-suite / substrate-score / test-crawler all share the same setup; each job adds only its own install line ('.[dev]' vs '.[dev,crawler]').
---
 .gitlab-ci.yml | 48 ++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 38 insertions(+), 10 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6c9a415..35ed6fd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,6 +35,40 @@ variables:
   # using -n auto (untouched by this var).
   PYTEST_XDIST_AUTO_NUM_WORKERS: "4"
 
+# ----------------------------------------------------------------------
+# Shared venv-warmup steps.
+#
+# gitlab-runner rotates between build directories
+# (builds/RUNNER_ID/0, /1, /3, ...). We cache `.venv/` to skip
+# reinstalling deps on every push — but an editable install
+# (`pip install -e .`) bakes the absolute build-dir path into
+# `__editable__.arborist-0.0.1.pth` and the dist-info RECORD. When
+# the cached venv is restored into a different build dir, the
+# embedded path is stale; the next `pip install -e` triggers an
+# implicit uninstall-then-reinstall that fails with `OSError: No such
+# file or directory` looking for files at the old build dir.
+#
+# Fix: before installing, wipe the editable-install metadata so pip
+# does a fresh install at the current build dir's path. Cache stays
+# warm (deps don't re-download), only the editable-install
+# bookkeeping is rebuilt — costs ~1s per job.
+#
+# Each job's before_script references this via
+# `!reference [.warm-venv-setup, before_script]` and then adds its
+# own `pip install -e '...'` line (so jobs can pick different extras
+# — `.[dev]` vs `.[dev,crawler]`).
+# ----------------------------------------------------------------------
+.warm-venv-setup:
+  before_script:
+    - python3 --version
+    - test -d .venv || python3 -m venv .venv
+    # Wipe stale editable-install bookkeeping from a prior build dir.
+    - rm -rf .venv/lib/python*/site-packages/arborist-*.dist-info
+    - rm -f  .venv/lib/python*/site-packages/__editable__.arborist*.pth
+    - rm -f  .venv/lib/python*/site-packages/__editable___arborist*_finder.py
+    - rm -f  .venv/bin/arborist
+    - .venv/bin/pip install --upgrade pip wheel
+
 # ----------------------------------------------------------------------
 # Default test job — runs the CI-scoped suite via `make test-ci`,
 # which is `make test` minus tests/crawler (network) and the
@@ -61,9 +95,7 @@ test:
       - .venv/
       - .pip-cache/
   before_script:
-    - python3 --version
-    - test -d .venv || python3 -m venv .venv
-    - .venv/bin/pip install --upgrade pip wheel
+    - !reference [.warm-venv-setup, before_script]
     - .venv/bin/pip install -e '.[dev]'
   script:
     # `make test-ci` invokes pytest with --ignore=tests/crawler
@@ -100,9 +132,7 @@ bench-suite:
       - .venv/
       - .pip-cache/
   before_script:
-    - python3 --version
-    - test -d .venv || python3 -m venv .venv
-    - .venv/bin/pip install --upgrade pip wheel
+    - !reference [.warm-venv-setup, before_script]
     - .venv/bin/pip install -e '.[dev]'
   script:
     - mkdir -p bench/results
@@ -146,8 +176,7 @@ substrate-score:
       - .venv/
       - .pip-cache/
   before_script:
-    - test -d .venv || python3 -m venv .venv
-    - .venv/bin/pip install --upgrade pip wheel
+    - !reference [.warm-venv-setup, before_script]
     - .venv/bin/pip install -e '.[dev]'
   script:
     # Find this branch's bench result.
@@ -188,8 +217,7 @@ test-crawler:
       - .venv/
       - .pip-cache/
   before_script:
-    - test -d .venv || python3 -m venv .venv
-    - .venv/bin/pip install --upgrade pip wheel
+    - !reference [.warm-venv-setup, before_script]
     - .venv/bin/pip install -e '.[dev,crawler]'
   script:
     - make test-crawler