diff --git a/.gitlab-ci.deploy.yml b/.gitlab-ci.deploy.yml new file mode 100644 index 0000000..dc32d09 --- /dev/null +++ b/.gitlab-ci.deploy.yml @@ -0,0 +1,14 @@ +# We MUST have a separate pipeline otherwise the artifacts are stale! +# this is why we use an include file instead of a single file! + +stages: + - deploy + +my.remarkbox.com: + stage: deploy + tags: ["my.remarkbox.com"] + script: + # deploy build via salt-call out to the salt-master for a highstate. + - sudo /opt/salt-call-state-highstate.sh + rules: + - if: '$CI_COMMIT_BRANCH == "main"' diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d6a3b1..7325f11 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ stages: - test - build - - my.remarkbox.com + - deploy - pypi-twine test: @@ -9,11 +9,10 @@ test: tags: ["build"] except: - tags - script: make test - artifacts: - paths: - - env.vanilla - expire_in: 1 hour + script: + - make venv + - env/bin/pip install 'setuptools<81' + - make test build: stage: build @@ -21,8 +20,8 @@ build: except: - tags script: - - ls -hal - - mv env.vanilla env + - make venv + - env/bin/pip install 'setuptools<81' - make install-source-prod # Copy static assets to env/static - cp -pr remarkbox/static env/static @@ -30,19 +29,23 @@ build: - cp -pr env/lib/python*/site-packages/remarkbox_westworld/static/* env/static # Copy static assets to env/static for meta theme. - cp -pr env/lib/python*/site-packages/remarkbox_theme_meta/static/* env/static + # Copy static assets to env/static for chaostheory theme. + - cp -pr env/lib/python*/site-packages/remarkbox_chaostheory/static/* env/static # Make tarball of the static files. - cp -pr env/static static - tar -zcf static.tar.gz static + # Clean up the directory outside of the gitlab-runner filesystem. + - rm -rf /opt/remarkbox/env # Clone the virtualenv with virtualenv-clone into the desired location. - - virtualenv-clone env /opt/remarkbox/env + - virtualenv-clone -vvv $PWD/env /opt/remarkbox/env + # Create commit-hash.txt to track this build's git commit hash. + - echo $CI_COMMIT_SHA >> commit-hash.txt + # Place it inside the virtualenv before creating the tarball. + - cp commit-hash.txt /opt/remarkbox/env/commit-hash.txt # Create a tarball of the virtualenv. - tar -zcf env.tar.gz -C /opt/remarkbox . # Create a SHA512 hash of env.tar.gz. - sha512sum env.tar.gz >> env.tar.gz.hash - # Create commit-hash.txt to track this build's git commit hash. - - echo $CI_COMMIT_SHA >> commit-hash.txt - # Clean up the directory outside of the gitlab-runner filesystem. - - rm -rf /opt/remarkbox/env artifacts: paths: - env.tar.gz @@ -50,16 +53,17 @@ build: - static.tar.gz - commit-hash.txt expire_in: 1 month + when: always -my.remarkbox.com: - stage: my.remarkbox.com - tags: ["my.remarkbox.com"] - script: - # publish build to Salt Master - - sudo /opt/salt-call-state-highstate.sh +deploy: + stage: deploy + needs: [build] rules: - - if: '$CI_COMMIT_BRANCH == "main"' + - if: '$CI_COMMIT_BRANCH == "develop" || $CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "main"' + when: on_success + trigger: + include: .gitlab-ci.deploy.yml pypi-twine: @@ -69,7 +73,4 @@ pypi-twine: only: - tags script: - - pip install --upgrade pip - - pip install twine - - python3 setup.py sdist bdist_wheel - - twine upload dist/* + - make twine-upload diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..94a9537 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,523 @@ +# Claude Code Configuration + +## Project Setup + +**IMPORTANT**: Before starting any work on a repository: +1. Check for a `CLAUDE.md` file in our repository root +2. Check for a `CLAUDE.md` file in parent directories (we often work across repos on localhost) +3. Read and follow all instructions in those files +4. These project-specific instructions override default Claude Code behavior +5. Look for conventions around commits, testing, code style, and workflows +6. If working across multiple repositories, respect our conventions from each repo's CLAUDE.md + +## Commit Attribution + +When creating git commits, use clean, simple commit messages: + +``` +Commit message here. +``` + +**Do NOT include:** +- `๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code)` +- `Co-Authored-By: Claude ` +- Any fake corporate entities as co-authors + +**Only attribute real humans** as co-authors when collaborating. + +**When fox approves a proposed commit, always commit AND push immediately.** No second confirmation needed. "yes" = commit + `git push`. + +## Database Migrations (Alembic) + +When adding new columns or modifying our database schema: + +1. **Backup SQLite first**: `cp data/remarkbox.sqlite data/remarkbox.sqlite.bak` +2. **Add our column to our model** in `remarkbox/models/` +3. **Generate migration**: `make migration m="description of change"` +4. **Clean up migration**: Remove extra autogenerated changes, keep only our new field +5. **Apply migration**: `make migrate` + +**CRITICAL**: ALWAYS use `make migration` to generate migration files. NEVER manually create migration files. NEVER hand-write or invent revision IDs. Alembic generates cryptographically unique revision IDs โ€” a made-up ID will corrupt the migration chain and break production deploys. + +```bash +# The ONLY correct way to create a migration: +make migration m="add foo column" +# โ†’ writes remarkbox/scripts/alembic/versions/05be3044c2d2_add_foo_column.py +# โ†’ revision ID is auto-generated (e.g. 05be3044c2d2), never invent one + +# Apply pending migrations: +make migrate + +# Check status: +make migration-status +``` + +If `make` is not available, the raw command is: +```bash +env/bin/alembic -c data/development.ini revision --autogenerate -m "description of change" +``` + +## Ticket System + +Tracked issues live in `docs/tickets/`. Start every session by reading our index: + +```bash +cat docs/tickets/index.md +``` + +- **Index**: `docs/tickets/index.md` is our master list. Always update it when creating or closing tickets. +- **Numbering**: Sequential. Next number = highest existing + 1. +- **Workflow**: Set status to `in-progress` when starting, `resolved` when done. Update `index.md` to match. +- **New tickets**: If you find a bug or get a feature request, create a new ticket file and add it to our index. +- **Sources**: Tickets reference community threads from `meta.remarkbox.com` and `faq.remarkbox.com` by UUID. + +## Remarkbox API and Python Client + +Remarkbox has a JSON API at `/api/v1/`. You can use it to read and write threads +on production as timehexon. Our session cookie is saved at `~/.config/remarkbox/cookies.txt`. + +### Quick start (from a Python script in our scratchpad or inline) + +```python +import os, sys +sys.path.insert(0, "/home/fox/git/remarkbox/remarkbox/api") +from remarkbox_client import RemarkboxClient + +c = RemarkboxClient( + "https://my.remarkbox.com", + cookie_file=os.path.expanduser("~/.config/remarkbox/cookies.txt"), +) + +# Read +threads = c.list_threads("meta.remarkbox.com") +thread = c.get_thread("9f970183-ffaf-11f0-b565-040140774501") +node = c.get_node(node_id) +profile = c.get_profile() +ver = c.version() + +# Write (authenticated) +result = c.create_thread(namespace="meta.remarkbox.com", title="Title", data="Body") +result = c.reply(parent_node_id, data="Reply body") +c.edit_node(node_id, data="Updated body") +c.edit_node(node_id, title="Updated title") # title only for root nodes +c.update_profile("new-display-name") + +# Moderate (authenticated, moderator or owner) +c.disable_node(node_id) +c.enable_node(node_id) +c.approve_node(node_id) +c.lock_node(node_id) +c.unlock_node(node_id) +c.delete_node(node_id) # permanent, moderator only + +# Export (pandoc-powered, 67 output formats) +formats = c.export_formats() +md = c.export_thread(node_id, "markdown") +pdf = c.export_thread(node_id, "pdf") +epub = c.export_namespace("meta.remarkbox.com", "epub") +docx = c.export_node(node_id, "docx") + +# Wiki mode (namespace.wiki=True, any authenticated user can edit root nodes) +c.wiki_edit(node_id, data="Updated content") # creates revision first +revisions = c.get_revisions(node_id) +revision = c.get_revision(revision_id) + +# Themes (auto-generated per namespace, light + dark mode) +css = c.get_theme_css("meta.remarkbox.com") +preview = c.get_theme_preview("meta.remarkbox.com") + +# Multi-syntax input (source_format parameter on create/reply/edit) +c.create_thread(namespace="ns", title="RST", data="Title\n=====\n\nParagraph.", source_format="rst") +c.reply(node_id, data="

HTML reply

", source_format="html") +``` + +### Key details + +- **Python client**: `remarkbox/api/remarkbox_client.py` (stdlib only, no pip) +- **C client**: `remarkbox/api/rb.c` (compile: `gcc rb.c -o rb -lcurl`) +- **API docs**: `docs/api.md` +- **Identity**: Authenticated as `timehexon@unturf.com` (display name: `timehexon`) +- **Journey thread**: `9f970183-ffaf-11f0-b565-040140774501` on `meta.remarkbox.com` -- update this after finishing work +- **Content limit**: 500,000 characters (~128k tokens) +- **Rate limits**: 120 reads/min, 30 writes/min, 1 thread creation per 7 min (wait if you hit 429) + +### Endpoints + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/v1/version` | Deployed git commit hash | +| GET | `/api/v1/threads?namespace=X` | List threads | +| GET | `/api/v1/threads/{id}` | Thread with replies | +| POST | `/api/v1/threads` | Create thread (1 per 7 min limit) | +| POST | `/api/v1/threads/{id}/replies` | Reply to thread | +| GET | `/api/v1/nodes/{id}` | Single node | +| PATCH | `/api/v1/nodes/{id}` | Edit node (data, title, disabled, approved, locked) | +| DELETE | `/api/v1/nodes/{id}` | Delete node permanently (moderator only) | +| POST | `/api/v1/auth/login` | Send OTP to email | +| POST | `/api/v1/auth/verify` | Verify OTP | +| GET | `/api/v1/user/profile` | Get profile | +| PATCH | `/api/v1/user/profile` | Update display name | +| GET | `/api/v1/clients/python` | Download Python client | +| GET | `/api/v1/clients/c` | Download C client (rb.c) | +| GET | `/api/v1/admin/namespaces` | List all namespaces (superuser only) | +| GET | `/api/v1/admin/recent-nodes?days=7` | Recent nodes network-wide (superuser only) | +| GET | `/api/v1/export/formats` | List available pandoc export formats | +| GET | `/api/v1/export/namespace/{name}.{fmt}` | Export namespace as book | +| GET | `/api/v1/export/threads/{node_id}.{fmt}` | Export thread as document | +| GET | `/api/v1/export/nodes/{node_id}.{fmt}` | Export node subtree (on-demand) | +| GET | `/api/v1/nodes/{id}/revisions` | Revision history for a node | +| POST | `/api/v1/nodes/{id}/wiki-edit` | Wiki-edit a node (creates revision) | +| GET | `/api/v1/revisions/{id}` | Get a specific revision | +| GET | `/api/v1/themes/{namespace}/css` | Auto-generated theme CSS | +| GET | `/api/v1/themes/{namespace}/preview` | Theme palette preview (JSON) | + +### Authentication + +Our saved cookie should work indefinitely. If it expires, you'll need an OTP: + +```python +c.login("timehexon@unturf.com") +# Ask the user for the 6-digit code from their email +c.verify("timehexon@unturf.com", "123456") +``` + +### Functional test + +Run our full idempotent test suite against production: + +```bash +env/bin/python remarkbox/api/functional_test.py https://my.remarkbox.com meta.remarkbox.com timehexon@unturf.com --name timehexon +``` + +This exercises all endpoints and updates our journey thread with results. + +## Superuser (Global Moderator) + +Users with `is_superuser=True` can moderate across all namespaces. This bypasses our +normal namespace-scoped `is_moderator()` check. Our admin UI is at `/topsecret/users` +where you can promote/demote users by email. + +Bootstrap our first superuser via our database script: +```bash +env/bin/python scripts/promote_superuser.py --ini development.ini --email timehexon@unturf.com +``` + +After that, use our web UI at `/topsecret/users` or our existing topsecret admin pages +(all guarded by `@super_fly_required` which checks `is_superuser`). + +Admin client methods: +```python +c.admin_list_namespaces() # list all namespaces +c.admin_recent_nodes(days=7) # recent nodes network-wide +``` + +## Terminology + +In all user-facing text, marketing, legal documents, and UI labels, use **"machine learning"** +instead of "AI". Our term "AI" should not appear in Remarkbox copy. Internal code comments +and technical references (e.g., "OpenAI-compatible endpoint") are fine. Third-party brand +names like "UncloseAI" are also unchanged. + +## Spam Prevention + +Spam detection runs automatically on `POST /api/v1/threads` and `POST /api/v1/threads/{id}/replies`. +Superusers bypass all spam checks. + +**Scoring signals**: link density, known spam patterns, duplicate content, new account velocity, +IP reputation (disabled post count), content length anomalies. + +**Thresholds** (configurable in `.ini`): +- `spam.hard_threshold = 0.8` -- reject with 403 +- `spam.soft_threshold = 0.5` -- allow but set `approved=False` (held for moderation) + +**Thread creation rate limit**: 1 new thread per 7 minutes per user/IP via our API. +This does not affect browser users or replies. + +**Spam hunting scripts** (require superuser cookie): +```bash +# Scan recent posts for spam +python scripts/spam/scan.py --days=7 --threshold=0.3 + +# Scan and output JSON +python scripts/spam/scan.py --json --threshold=0.5 + +# Bulk disable flagged posts +python scripts/spam/scan.py --json --threshold=0.8 | python scripts/spam/disable_spam.py --from-json + +# Disable specific nodes +python scripts/spam/disable_spam.py node-uuid-1 node-uuid-2 +``` + +## Credential Access + +**Never access credentials without explicit instruction from fox.** This includes `pass show`, reading API key files, private keys, session cookies, tokens, or any secrets. Propose first. Fox decides. Then execute. + +## Operation Voyeur + +**All comms are public** from 2026-03-29. Assume every terminal session and output is observed. NEVER display secrets to stdout. NEVER pass secrets as CLI args. NEVER read secret file contents with Read tool or cat โ€” content enters conversation logs. **Path is fine. Content is not.** Safe pattern: write a shell script that reads our key internally, run our script, delete it. + +## Production Rules + +**NEVER run direct SQL or raw database commands on production.** No `sqlite3`, no `UPDATE`, no `DELETE`, no direct file edits on our production database. Ever. If our API doesn't support what you need, add our endpoint first, push it, then use our client. + +**ALL production changes go through our API client.** Use `RemarkboxClient` with our saved cookie at `~/.config/remarkbox/cookies.txt`. This ensures authentication, audit trails, and proper ORM handling. + +**tmux-hosts is read-only.** You may use `tmux-hosts` to read logs, check processes, and investigate issues. You may NOT use it to modify data, run SQL, edit files, or restart services. + +```python +# RIGHT: Use the API client +c = RemarkboxClient("https://my.remarkbox.com", cookie_file="~/.config/remarkbox/cookies.txt") +c.disable_node(node_id) + +# WRONG: Never do this +# sqlite3 /opt/remarkbox/my.remarkbox.com.sqlite "UPDATE rb_node SET disabled=1 WHERE id='...'" +``` + +If a moderation operation is not yet supported by our API, our correct workflow is: +1. Add our endpoint to `remarkbox/api/views.py` +2. Add our method to `remarkbox/api/remarkbox_client.py` +3. Push, wait for deploy +4. Use our client + +## Deployment Status + +After pushing, check if our deploy is live by hitting our version endpoint: + +```bash +curl -s https://my.remarkbox.com/api/v1/version +# {"version": "5a10e15"} +``` + +Compare our returned commit hash against `git rev-parse --short HEAD` to confirm +our latest code is deployed. + +You can also check GitLab pipeline status: + +```bash +# Get pipeline status via API (replace PIPELINE_ID) +curl -s "https://git.unturf.com/api/v4/projects/engineering%2Fremarkbox%2Fremarkbox/pipelines/PIPELINE_ID" + +# Or view in browser: +# https://git.unturf.com/engineering/remarkbox/remarkbox/-/pipelines +``` + +Our pipeline status will show `"status":"success"` when deployment is complete. + +## Proxy Architecture + +Remarkbox domains route through two servers. Understanding this is critical +for debugging TLS, DNS, or routing issues. + +**Edge proxy**: `142.93.73.64` (`proxy.unturf.com`) โ€” terminates TLS, handles +ACME certs, runs bot defense (ASSHOLE CRM). Config lives in +`~/git/proxy.unturf.com/ingress/Caddyfile`. + +**Origin server**: `162.243.167.224` (`origin.remarkbox.com`) โ€” runs Caddy + +uwsgi. Config managed via salt pillar at `foxhop-pillar/caddy/remarkbox.sls`. + +### Domain routing map + +| Domain | DNS โ†’ | Handler | Backend | Status | +|--------|-------|---------|---------|--------| +| `remarkbox.com` | proxy (142.93.73.64) | redirect โ†’ www | โ€” | live | +| `www.remarkbox.com` | proxy | file_server | `/opt/www/remarkbox` on proxy | live | +| `my.remarkbox.com` | proxy | reverse_proxy | origin โ†’ uwsgi :6001 | live | +| `meta.remarkbox.com` | CNAME โ†’ my โ†’ proxy | reverse_proxy | origin โ†’ uwsgi :6001 | live | +| `faq.remarkbox.com` | CNAME โ†’ my โ†’ proxy | reverse_proxy | origin โ†’ uwsgi :6001 | live | +| `demo.remarkbox.com` | CNAME โ†’ my โ†’ proxy | reverse_proxy | origin โ†’ uwsgi :6001 | live | +| `origin.remarkbox.com` | direct (162.243.167.224) | reverse_proxy | uwsgi :6001 | live | +| `westworld2.com` | proxy | **parked** redirect โ†’ unturf.com | โ€” | parked 2026-04-08 | +| `www.foxhop.net` | proxy | **no Caddy block** โ€” not yet routed | โ€” | unrouted 2026-04-08 | + +**Planned multi-tenant consolidation (foxhop.net + westworld2.com โ†’ origin:6001):** +- Merge `foxhop.net.sqlite` + `westworld2.com.sqlite` into origin DB +- Set `rb_namespace.theme = 'chaostheory'` (foxhop) & `'westworld'` (westworld2) in DB +- Remove `app.namespace` override from foxhop config โ€” `request.domain` drives namespace +- Add proxy Caddy blocks: `www.foxhop.net` & `westworld2.com` โ†’ origin:6001 +- Theme packages installed locally: `remarkbox_chaostheory`, `remarkbox_westworld` +- Salt pillar ref: `~/git/foxhop-states/uwsgi/configs/westworld2.com.ini` +- foxhop local dev: `~/git/remarkbox/foxhop-local.ini` (port 6004) + +### Request flow for proxied domains (my, meta, faq) + +``` +Client โ†’ DNS (CNAME or A โ†’ 142.93.73.64) + โ†’ proxy.unturf.com Caddy (TLS termination, ACME, bot gate) + โ†’ reverse_proxy https://origin.remarkbox.com + (Host header preserved, TLS via origin cert) + โ†’ origin Caddy (routes by Host header) + โ†’ uwsgi localhost:6001 + โ†’ Remarkbox app (namespace from Host) +``` + +### Key rules + +- **CNAME domains MUST have explicit blocks on our proxy.** Without a block, + they fall through to our MPS on-demand TLS catch-all and route to our wrong + backend. This caused a 5-day outage (see `docs/postmortem-2026-02-25-ssl-outage-caddy-acme.md`). +- **Our proxy owns TLS for proxied domains.** Our origin server does not need + (and cannot obtain) ACME certs for domains whose DNS points to our proxy. +- **Static sites (www, remarkbox.com) are served directly from our proxy.** + Their content lives at `/opt/www/remarkbox` on our proxy server, deployed + from `~/git/www.remarkbox.com` via CI. +- **origin.remarkbox.com bypasses our proxy.** Its DNS points directly to + 162.243.167.224. Use it for SSH access and direct backend testing. + +## Capability-Driven Presentation + +Follow Russell Ballestrini's capability-driven presentation practice +(russell.ballestrini.net/capability-driven-presentation/). A page need not look +identical across all browsers. Accommodate what our user's browser can do: + +1. **Single canonical URI** โ€” one URI serves our content. +2. **Consistent content** โ€” regardless of viewer capabilities. +3. **Graceful enhancement/degradation** โ€” use available capabilities to enhance presentation. + +### Our `js-only` / `