Author SHA1 Message Date
joshuaandClaude Opus 5.5 0312767d75 Release agent@1.0.1
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 10:01:23 +02:00
joshua 46e8c15602 Merge branch 'fix/central-status' 2026-11-18 10:01:23 +02:00
joshuaandClaude Opus 5.5 f1d6234800 Agent: answer status on the central server itself
Central servers only get the agent (install.sh --central), not the crosscode script, but a
central agent accepts status jobs and ran `crosscode status` for them, which failed with
`spawn /usr/local/bin/crosscode ENOENT`. A central agent now reports its own stats (frps and
the agent version) for status; church servers still run crosscode status.

Also ignores temp/ (local screenshots).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 10:01:22 +02:00
joshuaandClaude Opus 5.5 a286f8abce docs: handover after the 2.3.0 / agent 1.0 / Hub 2.1 release
Root HANDOVER: phases 1b–1d released and VM tested, the nine bugs the VM
test found, rollout next. Events HANDOVER: 2.3.0, where two-factor and the
Hub-install pieces live. AGENT-1.0-PLAN and EVENTS-2.2-PLAN: tested and
released, the VM changes; ROADMAP §9 marked.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ls1vMdni7EfytGYqQTprME
2026-11-17 22:41:10 +02:00
joshuaandClaude Opus 5.5 d985a4c853 docs: rollout checklist for Hub 2.1, the agent and Events 2.3
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ls1vMdni7EfytGYqQTprME
2026-11-17 22:29:06 +02:00
13 changed files with 249 additions and 43 deletions
+1
View File
@@ -25,3 +25,4 @@ npm-debug.log*
# OS # OS
.DS_Store .DS_Store
Thumbs.db Thumbs.db
temp/
+6
View File
@@ -2,6 +2,12 @@
Each package and app has its own section and its own tags (`contracts@1.0.0`, `accounts@1.0.0`, …). Each package and app has its own section and its own tags (`contracts@1.0.0`, `accounts@1.0.0`, …).
## agent@1.0.1 - 2026-11-18
- A `status` job on the central server is answered by the agent itself (its stats, frps and
the agent version). It used to run `crosscode status`, which central servers don't have, and
failed with `spawn /usr/local/bin/crosscode ENOENT`.
## events@2.3.0, contracts@1.3.0, agent@1.0.0 - 2026-11-17 ## events@2.3.0, contracts@1.3.0, agent@1.0.0 - 2026-11-17
Phase 1c (authenticator-app sign-in) and phase 1d (docs/AGENT-1.0-PLAN.md): installs, updates and Phase 1c (authenticator-app sign-in) and phase 1d (docs/AGENT-1.0-PLAN.md): installs, updates and
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@crosscode/agent", "name": "@crosscode/agent",
"version": "1.0.0", "version": "1.0.1",
"private": true, "private": true,
"description": "The server agent: runs the crosscode script for signed jobs from Cross Code Hub, reports stats (docs/AGENT-1.0-PLAN.md)", "description": "The server agent: runs the crosscode script for signed jobs from Cross Code Hub, reports stats (docs/AGENT-1.0-PLAN.md)",
"license": "UNLICENSED", "license": "UNLICENSED",
+9
View File
@@ -159,6 +159,15 @@ export class Agent {
} }
case "export": case "export":
return await this.runExport(job, limit); return await this.runExport(job, limit);
case "status":
// The central server has no crosscode script (install.sh --central): its status is the
// agent's own stats, with frps.
if (this.deps.config.mode === "central") {
const stats = await (this.deps.stats ?? (() => collectStats("central")))();
const warning = stats.central && !stats.central.frpsRunning ? "frps isn't running." : undefined;
return { status: "succeeded", steps: [], result: { stats } as unknown as Record<string, unknown>, message: warning, log: [], startedAt, finishedAt: new Date().toISOString() };
}
return await this.run(crosscodeArgs(job)!, job, limit);
default: { default: {
const args = crosscodeArgs(job); const args = crosscodeArgs(job);
if (!args) throw new Error(`No command for ${job.type}`); if (!args) throw new Error(`No command for ${job.type}`);
+1 -1
View File
@@ -12,7 +12,7 @@ import { paths } from "./paths";
* the tools that are there; anything missing is null rather than an error. * the tools that are there; anything missing is null rather than an error.
*/ */
export const AGENT_VERSION = "1.0.0"; export const AGENT_VERSION = "1.0.1";
function run(cmd: string, args: string[], timeoutMs = 10_000): Promise<string | null> { function run(cmd: string, args: string[], timeoutMs = 10_000): Promise<string | null> {
return new Promise((resolve) => { return new Promise((resolve) => {
+18
View File
@@ -104,6 +104,24 @@ describe("the agent loop", () => {
expect(fs.readdirSync(path.join(state, "exports"))).toEqual([]); expect(fs.readdirSync(path.join(state, "exports"))).toEqual([]);
}); });
it("status on the central server: its own stats, without crosscode (which isn't installed there)", async () => {
const h = fakeHub([job({ type: "status", args: {} })]);
const run = jest.fn<ReturnType<Run>, Parameters<Run>>(async () => ok());
const stats = { schema: "crosscode.agent-stats/1", central: { proxyBindAddr: "127.0.0.1", allowPorts: null, frpsRunning: true } } as unknown as agent.AgentStats;
const a = new Agent({ config: { ...config, mode: "central" }, keys: { ed25519Pem: serverKeys.ed25519.privateKeyPem, x25519Pem: serverKeys.x25519.privateKeyPem }, hub: h.hub, seen: new SeenJobs(path.join(state, `seen-${Math.random()}.json`)), run, stats: async () => stats, now: () => NOW });
await a.pollOnce();
expect(run).not.toHaveBeenCalled();
const r = reports(h.posts)[0]!;
expect(r).toMatchObject({ type: "status", status: "succeeded", result: { stats } });
expect(agent.agentReportSchema.safeParse(r).success).toBe(true);
});
it("status on a church server runs crosscode status", async () => {
const { a, run } = make([job({ type: "status", args: {} })]);
await a.pollOnce();
expect(run.mock.calls[0]![0]).toEqual(["status", "--no-fetch"]);
});
it("a job that throws is reported as failed", async () => { it("a job that throws is reported as failed", async () => {
const { a, posts } = make([job()], jest.fn<ReturnType<Run>, Parameters<Run>>(async () => { throw new Error("boom"); })); const { a, posts } = make([job()], jest.fn<ReturnType<Run>, Parameters<Run>>(async () => { throw new Error("boom"); }));
await a.pollOnce(); await a.pollOnce();
+30 -7
View File
@@ -1,6 +1,8 @@
# Handover — where things stand and how to carry on # Handover — where things stand and how to carry on
Last updated: 2026-11-05, after **2.1.1** (deployed). **This file covers Cross Events only** Last updated: 2026-11-17, after **2.3.0** (released, tested on a VM, not deployed yet; servers
still run 2.1.1 until they're adopted, see the root [ROLLOUT-1.0.md](../../../docs/ROLLOUT-1.0.md)).
**This file covers Cross Events only**
(what's been done, where the code lives, known gaps). **The plan for every product and where each (what's been done, where the code lives, known gaps). **The plan for every product and where each
phase stands are in the monorepo's root `docs/`** → phase stands are in the monorepo's root `docs/`** →
[ROADMAP-v3.md](../../../docs/ROADMAP-v3.md), [ROADMAP-v3.md](../../../docs/ROADMAP-v3.md),
@@ -95,13 +97,13 @@ encrypted settings unreadable or break existing installs.
## Next up ## Next up
1. **Roadmap v3**, across every product: see `crosscode-platform/docs/HANDOVER.md` ("Where each 1. **Roadmap v3**, across every product: see `crosscode-platform/docs/HANDOVER.md` ("Where each
phase stands", "Next up"). For Cross Events that means **2.2.0** (the move into the monorepo as phase stands", "Next up"). **2.2.0** (the move into the monorepo) and **2.3.0** (authenticator
`apps/events`, with the `crosscode` script; `crosscode-platform/docs/EVENTS-2.2-PLAN.md`), then app 2FA and recovery codes, plus what installs from the Hub need) are released (2026-11-17).
**2.3.0** (authenticator app 2FA, `TwoFactorMethod` in `prisma/schema.prisma`) and **2.4.0** Next for Cross Events is **rolling them out** (each install adopted from the Hub), then **2.4.0**
(protocol v2, Linked products, the event feed). (protocol v2, Linked products, the event feed).
- **Deploying until 2.2.0:** `bash scripts/deploy.sh --check`, then - **Deploying:** installs still on 2.1.1 move with `adopt` (Hub → Servers → Installs → Link
`bash scripts/deploy.sh v<version>` on each server (installs ask through Request update as existing → Move onto the monorepo, or `crosscode adopt` by hand), never `deploy`. After that,
usual). From 2.2.0 on: `crosscode deploy <church> events`. updates are planned on the Hub (Updates), or `crosscode deploy <church> events` by hand.
- **AI for a church:** set the licence's monthly AI cap above R0 on the Hub (Hub 1.6.0+); with - **AI for a church:** set the licence's monthly AI cap above R0 on the Hub (Hub 1.6.0+); with
no cap the buttons don't show. no cap the buttons don't show.
- Testing AI locally: run the Hub (`../events-hub`, port 5200), set Settings → AI there, give a - Testing AI locally: run the Hub (`../events-hub`, port 5200), set Settings → AI there, give a
@@ -119,6 +121,27 @@ encrypted settings unreadable or break existing installs.
Languages (Afrikaans, §10) and Wallet passes (§7) are removed from scope (2026-10-28). Languages (Afrikaans, §10) and Wallet passes (§7) are removed from scope (2026-10-28).
## Where two-factor lives (2.3.0)
- `backend/src/utils/twoFactor.js`: every method (email, WhatsApp, `authenticator`), challenges,
recovery codes (10 per set, stored as HMACs, single use). `backend/src/utils/totp.js`: RFC 6238
with a last-step check (`User.totpLastStep`), so a code can't be used twice.
- **`totpSecret` never goes to the browser:** `LOGIN_USER_SELECT` doesn't include it (that selection
is sent back on sign-in), so `createChallenge` reads it from the database itself.
- Frontend: the profile page draws the QR code in the browser (a dynamic `import("qrcode")`);
`LoginForm.tsx` has "Use a recovery code". Tests: `tests/twoFactorAuthenticator.test.js`.
## Where the Hub-install pieces live (2.3.0)
- `env.template.json` (copied to `scripts/products/events.env.template.json` at the monorepo root;
keep them the same): every `.env` variable with its layer (`server`, `defaults`, `install`,
`generated`). The Hub has its own copy (`events-hub/backend/src/utils/envTemplates/events.json`).
A new variable goes in all three.
- `backend/scripts/export-church.ts` (`src/services/churchExport.ts`): the church's data as
`crosscode.church-export/1`, for `crosscode export` and uninstalls. No internal ids.
- `backend/src/hub/client.js` `connectFromEnv()`: a site the Hub installed registers by itself on
first start from `HUB_LICENCE_KEY`.
## Working on this repo ## Working on this repo
- **Release routine:** branch `feat/<name>` → commit → bump with - **Release routine:** branch `feat/<name>` → commit → bump with
+16 -4
View File
@@ -8,9 +8,9 @@ This replaces the old rule "the Hub never deploys" (decisions 10 and 23). The pa
stays: **no site depends on the Hub to keep running**, and everything still goes through the stays: **no site depends on the Hub to keep running**, and everything still goes through the
`crosscode` script (EVENTS-2.2-PLAN §4), which keeps working by hand over SSH. `crosscode` script (EVENTS-2.2-PLAN §4), which keeps working by hand over SSH.
**Built 2026-11-18** (crosscode-platform branch `feat/agent-1.0`, events-hub branch **Built, VM tested and released 2026-11-17** (`agent@1.0.0`, `contracts@1.3.0`, Hub `v2.1.0`; with
`feat/servers-agent`; not merged or released). Waiting for the VM tests (§13). Where the build `events@2.3.0`). Not rolled out yet: the steps are in [ROLLOUT-1.0.md](./ROLLOUT-1.0.md). Where the
differs from this plan, see [Changes while building](#changes-while-building). build differs from this plan, see [Changes while building](#changes-while-building).
## Contents ## Contents
@@ -459,7 +459,7 @@ Central check: both ports are open in frps and only on 127.0.0.1.
## Changes while building ## Changes while building
Made on 2026-11-18, while building; none changes what Joshua or a church sees. Made on 2026-11-17, while building and in the VM test; none changes what Joshua or a church sees.
1. **Config encryption** is X25519 + HKDF-SHA256 + AES-256-GCM (`x25519-hkdf-sha256-aes-256-gcm`) 1. **Config encryption** is X25519 + HKDF-SHA256 + AES-256-GCM (`x25519-hkdf-sha256-aes-256-gcm`)
instead of `x25519-xsalsa20-poly1305`: Node's built-in crypto has no XSalsa20, and this needs no instead of `x25519-xsalsa20-poly1305`: Node's built-in crypto has no XSalsa20, and this needs no
@@ -480,6 +480,18 @@ Made on 2026-11-18, while building; none changes what Joshua or a church sees.
`local_port` and `remote_port`. `local_port` and `remote_port`.
7. **`crosscode install`** refuses a database or role that already exists, so it can never take 7. **`crosscode install`** refuses a database or role that already exists, so it can never take
over (or, on a failed install, drop) another site's database. over (or, on a failed install, drop) another site's database.
8. **PM2 runs under its own systemd unit** (`pm2-<user>`) for every site user, set up by the
`crosscode` script before any PM2 command. Found in the VM test: a PM2 daemon started from the
agent's job belonged to the agent's service, so stopping or updating the agent stopped the
sites. The agent's stats never start a PM2 daemon.
9. **frpc is restarted once**, the first time its admin lines are added (a running frpc only reads
them when it starts); later changes reload it. Every tunnel on that server drops for a few
seconds that first time.
10. **The VM test** (§13) also fixed: `adopt` applies a release's new migrations; a failed
migration is marked rolled back so later deploys aren't blocked; the agent install builds with
the root's tools; tags are fetched with `--force`; the Hub saves the servers settings, lets a
failed or removed install's short name be used again, and links a Hub-installed site to its
registration so banners reach it. The full list is in [HANDOVER.md](./HANDOVER.md).
## Decisions ## Decisions
+4
View File
@@ -252,6 +252,10 @@ database or its name.
## 6. Testing ## 6. Testing
**Done 2026-11-17** on a WSL Ubuntu 24.04 VM (steps 1–7 below; step 8 in the agent test). All
pass after two fixes: `adopt` refused a release with new migrations (it now applies them in the
switch-over), and a failed migration blocked later deploys (now marked rolled back).
- **Events, unchanged behaviour:** backend Jest, `tsc --noEmit`, frontend build and the - **Events, unchanged behaviour:** backend Jest, `tsc --noEmit`, frontend build and the
Playwright e2e run, all from the monorepo. They should give the same counts as the 2.1.1 Playwright e2e run, all from the monorepo. They should give the same counts as the 2.1.1
baseline. baseline.
+54 -26
View File
@@ -1,21 +1,21 @@
# Handover — Cross Code, all products # Handover — Cross Code, all products
Last updated: 2026-11-12. **The plan is [ROADMAP-v3.md](./ROADMAP-v3.md)** (decided 2026-11-03, Last updated: 2026-11-17. **The plan is [ROADMAP-v3.md](./ROADMAP-v3.md)** (decided 2026-11-03,
decisions table near the end). This file says where each phase stands and what comes next. decisions table near the end). This file says where each phase stands and what comes next.
Product-specific notes (where the code lives, known gaps) stay with each product: Cross Events' Product-specific notes (where the code lives, known gaps) stay with each product: Cross Events'
are in `hope-events/docs/HANDOVER.md` (`apps/events/docs/HANDOVER.md` from Events 2.2.0), and are in [`apps/events/docs/HANDOVER.md`](../apps/events/docs/HANDOVER.md), and the Hub's are in
the Hub's are in `events-hub/docs/`. `events-hub/docs/` and `events-hub/deployment.md`.
The docs for every product are here in `docs/`: the roadmap, the Hub protocol The docs for every product are here in `docs/`: the roadmap, the Hub protocol
([hub-protocol.md](./hub-protocol.md), with a copy in `events-hub`), this handover and the plans ([hub-protocol.md](./hub-protocol.md), with a copy in `events-hub`), this handover, the plans
for each phase. They moved here from `hope-events/docs/` on 2026-11-05. for each phase and the rollout checklist ([ROLLOUT-1.0.md](./ROLLOUT-1.0.md)).
## Repos ## Repos
| Repo | What | | Repo | What |
|------|------| |------|------|
| `joshua/crosscode-platform` (this one) | Contracts, the apps (Accounts, Website, App, and Events from 2.2.0), shared packages, the `crosscode` script (from Events 2.2.0), the server agent (`apps/agent`, phase 1d), and these docs | | `joshua/crosscode-platform` (this one) | Contracts, the apps (Events from 2.2.0; Accounts, Website and App to come), shared packages, the `crosscode` script (`scripts/`), the server agent (`apps/agent`), and these docs |
| `joshua/hope-events` | Cross Events up to 2.1.x. Archived after the 2.2.0 move | | `joshua/hope-events` | Cross Events up to 2.1.x. Frozen; archived once every install is adopted (EVENTS-2.2-PLAN §7) |
| `joshua/events-hub` | Cross Code Hub. Its own server, its own `scripts/deploy.sh` | | `joshua/events-hub` | Cross Code Hub. Its own server, its own `scripts/deploy.sh` |
| `joshua/crosscode-site` | crosscode.co.za. Updated at the end of every phase that changes what it says (ROADMAP §9) | | `joshua/crosscode-site` | crosscode.co.za. Updated at the end of every phase that changes what it says (ROADMAP §9) |
@@ -23,26 +23,44 @@ for each phase. They moved here from `hope-events/docs/` on 2026-11-05.
| Phase | What | State | | Phase | What | State |
|-------|------|-------| |-------|------|-------|
| 0 | contracts 1.0.0 / 1.1.0 | **Released** 2026-11-04 (`contracts@1.0.0`, `contracts@1.1.0`). Not published to a registry (decision 22): the Hub gets a packed release committed to its repo | | 0 | contracts 1.0.0 / 1.1.0 | **Released** 2026-11-04. Not published to a registry (decision 22): the Hub gets a packed release committed to its repo (`backend/vendor/`) |
| 1 | Hub 2.0.0 | **Released** 2026-11-05 (`events-hub` `v2.0.0`; `docs/HUB-2.0-PLAN.md` there). **Not deployed yet:** `events-hub/deployment.md` → "Upgrading to 2.0" (run `check-hub2` first). crosscode-site's address prices are merged, **not deployed yet** | | 1 | Hub 2.0.x | **Released** (`v2.0.0` 2026-11-05, `v2.0.1` AI tokens and `v2.0.2` refunds / voiding refunded invoices, 2026-11-17). **Not deployed yet:** `events-hub/deployment.md` → "Upgrading to 2.0" (run `check-hub2` first) |
| 1b | Cross Events 2.2.0: moves here as `apps/events`, `crosscode` script | **In progress.** Planned in [EVENTS-2.2-PLAN.md](./EVENTS-2.2-PLAN.md). **Steps 1–3 done 2026-11-12** (merged into `main`): history merged in under `apps/events` (from `hope-events` `e6c9670`), workspaces and one root lockfile with versions pinned to 2.1.1, same test results as 2.1.1 (711 backend tests, typecheck, both builds). **The `crosscode` script and `adopt` (§4–5) are written** (branch `feat/crosscode-script`, 2026-11-12; shellcheck and 47 bats tests pass, not yet run on a server). **Tagged `events@2.2.0` on 2026-11-17, before the VM testing (Joshua's call), with two additions: AI use in tokens and the backend under `/api` (Hub `v2.0.1` and crosscode-site `v1.2.0` released with them).** **Still to do:** VM testing (§6) of the tagged release, adopting each install, archiving `hope-events` (§7). Until then servers keep deploying from `hope-events` `v2.1.1`, and **Events work happens here** (`hope-events` is frozen; a hotfix there as 2.1.2 is cherry-picked into `apps/events`) | | 1b | Cross Events 2.2.0: moved here as `apps/events`, the `crosscode` script | **Released** 2026-11-17 (`events@2.2.0`, with AI use in tokens and the backend under `/api`). **VM tested 2026-11-17** (EVENTS-2.2-PLAN §6, all passing after fixes). **Not rolled out:** servers still run `hope-events` `v2.1.1` until each install is adopted |
| 1c | Cross Events 2.3.0: authenticator app 2FA (TOTP + recovery codes) | TOTP not started (`TwoFactorMethod` enum in Events' `prisma/schema.prisma`). | | 1c | Cross Events 2.3.0: authenticator app 2FA (TOTP + recovery codes) | **Released** 2026-11-17 (`events@2.3.0`). crosscode-site `v1.3.0` lists it (merged and tagged, **not deployed**) |
| 1d | Agent 1.0.0 + Hub 2.1.0 (+ contracts 1.3.0): server agent, installs / scheduled updates / uninstalls from the Hub, server stats | **Built 2026-11-18, not released.** crosscode-platform branch `feat/agent-1.0` (contracts 1.3.0, the crosscode script with `--json`, env templates, frpc, restart / export / uninstall / purge, Events env template + export + `HUB_LICENCE_KEY`, `apps/agent`), events-hub branch `feat/servers-agent` (Hub 2.1.0). All unit and script tests pass. **Still to do:** the VM tests (AGENT-1.0-PLAN §13, together with EVENTS-2.2-PLAN §6), then release and roll out (§14). | | 1d | Agent 1.0.0 + Hub 2.1.0 (+ contracts 1.3.0) | **Released** 2026-11-17 (`agent@1.0.0`, `contracts@1.3.0`, Hub `v2.1.0`). **VM tested** end to end (AGENT-1.0-PLAN §13: joins, installs behind frp, scheduled / batch / missed updates, uninstall with export and purge, forged and replayed jobs). **Not rolled out** |
| 2 | Cross Events 2.4.0: protocol v2, Linked products, event feed, `packages/seo` | Not started. Events keeps speaking protocol 1 until then | | 2 | Cross Events 2.4.0: protocol v2, Linked products, event feed, `packages/seo` | Not started. Events keeps speaking protocol 1 until then |
| 3–7 | Accounts 1.0.0 → Website 1.0.0 → Events 3.0.0 → App 1.0.0 → App 1.x | Not started | | 3–7 | Accounts 1.0.0 → Website 1.0.0 → Events 3.0.0 → App 1.0.0 → App 1.x | Not started |
## Next up ## Next up
1. **Deploy Hub 2.0.0** and **crosscode-site** (the address prices). This finishes phase 1 and is 1. **Roll out**, in the order in [ROLLOUT-1.0.md](./ROLLOUT-1.0.md): Hub 2.1.0 (make and back up
needed before 2.2.0, because Hub 1.7 can't see `events@…` tags. the job-signing key first) → the central server's agent (check `proxy_bind_addr = 127.0.0.1`)
2. **Cross Events 2.2.0**: [EVENTS-2.2-PLAN.md](./EVENTS-2.2-PLAN.md). Decided 2026-11-05: → each church server's agent → **adopt** the demo site, then each church (Link existing → Move
`adopt` builds next to the running install and switches over, and `adopt --undo` is built onto the monorepo) → a throwaway New install to prove FASTPANEL, DNS and Let's Encrypt.
(ROADMAP decisions 25–26). 2. **Deploy crosscode-site `v1.3.0`** (secure sign-in in the features and the brochure).
3. **Agent 1.0.0 + Hub 2.1.0** (phase 1d): [AGENT-1.0-PLAN.md](./AGENT-1.0-PLAN.md). Before the 3. **Archive `hope-events`** once every install reports 2.2.0 or later (EVENTS-2.2-PLAN §7).
first install through it: check `proxy_bind_addr = 127.0.0.1` in `frps.ini` on the central 4. **Cross Events 2.4.0** (phase 2), as in ROADMAP §9.
server (worth doing now: without it, tunnel ports such as `:5000` may be reachable straight from
the internet), and set each server's database settings and `TRUST_PROXY` in the Hub. ## The VM test (2026-11-17)
4. **Cross Events 2.3.0** (authenticator app 2FA), then 2.4.0, as in ROADMAP §9.
Everything above was run on a WSL Ubuntu 24.04 VM before release: a church server, a "central"
network namespace running frps with `proxy_bind_addr`, the Hub (`feat/servers-agent`) and local
git mirrors standing in for c-git. It found nine bugs, all fixed with tests before tagging. None
lost data, because each failed safely:
1. `adopt` refused a release with new migrations (it now applies them in the switch-over).
2. A failed migration blocked every later deploy (Prisma P3009; now marked rolled back).
3. The agent install skipped the root's build tools (`npm ci --include-workspace-root`).
4. A failed install's short name couldn't be used again.
5. The Hub didn't save the servers settings (install defaults, port range, central IP).
6. frpc only reads its admin lines when it starts (restarted once the first time).
7. **A site's PM2 belonged to the agent's service**, so stopping or updating the agent took down
every site it had installed. PM2 now always runs under `pm2-<user>`.
8. Update and closing banners never reached Hub-installed sites (registration now links them).
9. A moved tag blocked a church's deploys (tags are fetched with `--force`).
Not covered by the VM: FASTPANEL, real DNS, Let's Encrypt. Hence the throwaway install in the
rollout.
## Things to remember across phases ## Things to remember across phases
@@ -51,10 +69,20 @@ for each phase. They moved here from `hope-events/docs/` on 2026-11-05.
`backend/src/utils/billing/tiers.js`) in the same phase, so the site and the Hub never disagree. `backend/src/utils/billing/tiers.js`) in the same phase, so the site and the Hub never disagree.
- **A phase isn't done until crosscode-site matches it** (ROADMAP §9, "Updating crosscode-site"). - **A phase isn't done until crosscode-site matches it** (ROADMAP §9, "Updating crosscode-site").
- Servers stay **PM2 + nginx** (no Docker), with one database per product per church, under - Servers stay **PM2 + nginx** (no Docker), with one database per product per church, under
`/var/www/sites/<church>/<product>`. `/var/www/sites/<church>/<product>`. Each site user's PM2 runs as a systemd unit,
`pm2-<user>`; the `crosscode` script makes sure of it before any PM2 command.
- **The agent pins the Hub's job key** when it joins. Back up `events-hub/backend/keys/` (licence
and job keys); replacing the job key means every server joins again.
- **A new `.env` variable** goes in three env templates: `apps/<product>/env.template.json`,
`scripts/products/<product>.env.template.json` and the Hub's `envTemplates/<product>.json`.
- **A new Events install never takes over an existing database**: moving an existing site is
`adopt`, not a New install pointed at the old database.
- **Prisma in the monorepo:** each app other than Events sets its own generator `output`, so - **Prisma in the monorepo:** each app other than Events sets its own generator `output`, so
hoisted clients don't overwrite each other (EVENTS-2.2-PLAN §2). hoisted clients don't overwrite each other (EVENTS-2.2-PLAN §2).
- **AI for a church:** set the licence's monthly AI cap above R0 on the Hub; with no cap the - **AI for a church:** set the licence's monthly AI cap above R0 on the Hub; with no cap the
buttons don't show (rules in Events' handover, "AI rules"). buttons don't show (rules in Events' handover, "AI rules"). Churches see tokens; only the Hub
- **Releasing:** branch → merge → tag (`<product>@<version>`) → push, with a CHANGELOG entry, and shows rand (markup 15% by default).
`npm run check` first. Pushes to c-git often fail authentication the first time; retry once. - **Releasing:** branch → merge → tag (`<product>@<version>` here, `v<version>` in events-hub and
crosscode-site) → push, with a CHANGELOG entry, and `npm run check` first. Pushes to c-git often
fail authentication the first time; retry once. A push can also hang waiting on a login prompt
after it has already gone through: check with `git ls-remote` before pushing again.
+3 -3
View File
@@ -613,9 +613,9 @@ Planned in [AGENT-1.0-PLAN.md](./AGENT-1.0-PLAN.md). In short:
|-------|---------|------|----------------| |-------|---------|------|----------------|
| 0 | contracts 1.0.0 | `crosscode.event/1`, `crosscode.church/1`, `crosscode.identity/1`, Hub protocol v2: schemas, generated JSON Schema, docs, sample payloads, the monorepo set up. **Released 2026-11-04** | Everything else depends on them | | 0 | contracts 1.0.0 | `crosscode.event/1`, `crosscode.church/1`, `crosscode.identity/1`, Hub protocol v2: schemas, generated JSON Schema, docs, sample payloads, the monorepo set up. **Released 2026-11-04** | Everything else depends on them |
| 1 | **Hub 2.0.0** | Cross Code Hub: products, licence per product, per-customer billing with bundles, protocol v2 (v1 still accepted), sibling discovery, AI usage per product, **releases per product with requirements** (§5), contracts as a committed file (§2). **Released 2026-11-05** (crosscode-site address prices merged) | Must exist before a second product is sold, and before Events releases come from the monorepo | | 1 | **Hub 2.0.0** | Cross Code Hub: products, licence per product, per-customer billing with bundles, protocol v2 (v1 still accepted), sibling discovery, AI usage per product, **releases per product with requirements** (§5), contracts as a committed file (§2). **Released 2026-11-05** (crosscode-site address prices merged) | Must exist before a second product is sold, and before Events releases come from the monorepo |
| 1b | **Cross Events 2.2.0** | **Moves into the monorepo** as `apps/events`, with its git history (merged in, not copied); its packages become workspaces; the **`crosscode` script** (`deploy`, `status`, `install`, `adopt`; §8) replaces `scripts/deploy.sh` and `scripts/install.sh`; every install moved with `crosscode adopt`; `hope-events` archived. Planned with no feature changes; two small ones were added on 2026-11-17 (decided by Joshua): **AI use shown in tokens**, never rand, outside the Hub (needs Hub 2.0.1), and **everything the backend serves under `/api`**, so a site and its API share one address and one proxy rule. **Tagged 2026-11-17, before the VM testing** | Before more products, so every product is released and deployed the same way from the start | | 1b | **Cross Events 2.2.0** | **Moves into the monorepo** as `apps/events`, with its git history (merged in, not copied); its packages become workspaces; the **`crosscode` script** (`deploy`, `status`, `install`, `adopt`; §8) replaces `scripts/deploy.sh` and `scripts/install.sh`; every install moved with `crosscode adopt`; `hope-events` archived. Planned with no feature changes; two small ones were added on 2026-11-17 (decided by Joshua): **AI use shown in tokens**, never rand, outside the Hub (needs Hub 2.0.1), and **everything the backend serves under `/api`**, so a site and its API share one address and one proxy rule. **Tagged 2026-11-17, before the VM testing; VM tested the same day** | Before more products, so every product is released and deployed the same way from the start |
| 1c | **Cross Events 2.3.0** | **Authenticator app 2FA** (TOTP) next to email and WhatsApp, with recovery codes; admins can see which method each staff account uses. | Independent and useful now; the Accounts migration later carries it over | | 1c | **Cross Events 2.3.0** | **Authenticator app 2FA** (TOTP) next to email and WhatsApp, with recovery codes; admins can see which method each staff account uses. **Released 2026-11-17** (`events@2.3.0`) | Independent and useful now; the Accounts migration later carries it over |
| 1d | **Agent 1.0.0 + Hub 2.1.0** (contracts 1.3.0) | **Server agent** on every server (polls only, signed jobs, runs the `crosscode` script); Hub: Servers page with stats and alerts, server configuration, frp port list, New install with a choice of server, scheduled / emergency / batch updates with notice to the church, uninstall with an encrypted JSON export, emailed reports. [AGENT-1.0-PLAN.md](./AGENT-1.0-PLAN.md) | Right after 1b (it runs the `crosscode` script), so every new church from then on is installed from the Hub. Can run alongside 1c | | 1d | **Agent 1.0.0 + Hub 2.1.0** (contracts 1.3.0) | **Server agent** on every server (polls only, signed jobs, runs the `crosscode` script); Hub: Servers page with stats and alerts, server configuration, frp port list, New install with a choice of server, scheduled / emergency / batch updates with notice to the church, uninstall with an encrypted JSON export, emailed reports. [AGENT-1.0-PLAN.md](./AGENT-1.0-PLAN.md). **Released 2026-11-17** after the VM test; rollout in [ROLLOUT-1.0.md](./ROLLOUT-1.0.md) | Right after 1b (it runs the `crosscode` script), so every new church from then on is installed from the Hub. Can run alongside 1c |
| 2 | **Cross Events 2.4.0** | Reports `product: "events"` (protocol v2), **Linked products**, the event feed (publish + show events from siblings), `packages/seo` (§11) with canonical links for imported events | Small; ready for when the Website exists | | 2 | **Cross Events 2.4.0** | Reports `product: "events"` (protocol v2), **Linked products**, the event feed (publish + show events from siblings), `packages/seo` (§11) with canonical links for imported events | Small; ready for when the Website exists |
| 3 | **Accounts 1.0.0** | OIDC provider, sign-in, 2FA (email, WhatsApp, authenticator app), password reset, consent and notice, export / delete fan-out, admin, install script | The Website is built on it from day one | | 3 | **Accounts 1.0.0** | OIDC provider, sign-in, 2FA (email, WhatsApp, authenticator app), password reset, consent and notice, export / delete fan-out, admin, install script | The Website is built on it from day one |
| 4 | **Church Website 1.0.0** | Public site with the **four templates** (§6), admin, simple events + feed, SEO (§11), Hope Family Church as the first church, then a second test church on a different template to prove it repeats | More churches need a website than an app; it also covers the "website only" case | | 4 | **Church Website 1.0.0** | Public site with the **four templates** (§6), admin, simple events + feed, SEO (§11), Hope Family Church as the first church, then a second test church on a different template to prove it repeats | More churches need a website than an app; it also covers the "website only" case |
+105
View File
@@ -0,0 +1,105 @@
# Rollout: Hub 2.1, the agent and Events 2.3 on the real servers
The order to move the live servers onto the agent, so that installs, updates and uninstalls are run
from the Hub. Each step can be stopped after without leaving anything half done. Tested end to end
on a VM (WSL: a church server, a central server with frps, the Hub) on 2026-11-17; the real servers
add FASTPANEL, real DNS and Let's Encrypt, which the VM didn't have, so step 6 starts with a
throwaway church.
Releases: `events@2.3.0`, `contracts@1.3.0`, `agent@1.0.0` (crosscode-platform), Hub `v2.1.0`,
crosscode-site `v1.3.0`.
## Before you start
- [ ] A backup of the Hub's database and `backend/keys/` (the licence key).
- [ ] A backup of each church's database (`pg_dump`) and `uploads/`. `adopt` and `deploy` take
their own backups too, but keep one that's yours.
- [ ] Know your frp port range (`allow_ports` in `frps.ini`) and which ports your hand-written
tunnels use.
## 1. The Hub (2.1.0)
Follow `events-hub/deployment.md` → "Upgrading to 2.1":
- [ ] `npm run generate-keys -- --job` in `backend/`; back up `keys/hub-job-key.pem` with the
licence key. **Agents pin this key when they join**: losing or replacing it means every server
joins again.
- [ ] `.env`: `HUB_JOB_SIGNING_KEY_FILE=./keys/hub-job-key.pem` and
`HUB_PUBLIC_URL=https://hub.crosscode.co.za`.
- [ ] `bash scripts/deploy.sh v2.1.0`.
- [ ] Servers → Ports & defaults: the port range, the central server's IP, the install defaults.
Mark every port a hand-written tunnel uses as taken.
## 2. The central server (frps)
- [ ] `frps.ini` has `proxy_bind_addr = 127.0.0.1` (tunnelled ports are then only reachable by
nginx on that server). Restart frps and check every FASTPANEL site still answers.
- [ ] Hub → Servers → Add server → **Central**; run the command it shows as root. It's read-only:
it checks a new install's ports in frps and watches frps.
- [ ] Within a minute the server shows on Servers with its stats.
## 3. Each church server
- [ ] Hub → Servers → Add server → **Church**; run the command as root. It installs Node tools,
the agent (`/opt/crosscode/agent`) and the crosscode script (`/opt/crosscode/tools`, from
main), and starts `crosscode-agent`. The agent listens on nothing.
- [ ] On the server's page: its configuration. `TRUST_PROXY=loopback` behind frp; the database:
`DB_HOST`/`DB_PORT`/`DB_ADMIN_URL` for a separate PostgreSQL, or nothing for the local one.
- [ ] `systemctl status crosscode-agent` is active; the server's stats show on the Hub.
- [ ] The first time the agent changes `frpc.ini` it adds the admin lines and **restarts frpc once**
(every tunnel on that server drops for a few seconds). Do the first install on a quiet
evening. Later changes only reload frpc.
## 4. Existing sites (the demo site and the churches): adopt, don't reinstall
Moving a site you already run onto the monorepo is `adopt`. It keeps the same database, uploads,
`.env`, ports and nginx site; it only swaps the code and PM2 underneath, with a backup and an undo.
A New install can't take over an existing database: it refuses a database or role that already
exists, so that it can never overwrite a church's data. "Install new and point it at the old
database" would therefore stop at that check.
For each site:
- [ ] Servers → Installs → **Link existing**: the server, its folder name and its two ports.
- [ ] **Move onto the monorepo** (an `adopt` job): planned at least 4 hours ahead; the church gets
the email and banner. Start with the demo site.
- [ ] After it: the site answers, `crosscode status` on the server shows it on `events@2.3.0`, its
PM2 runs as `pm2-<user>` under systemd.
- [ ] If something's wrong: the job's page has the log. `crosscode adopt <church> --undo` on the
server puts the old install back (for 14 days).
## 5. Updates from then on
- [ ] Updates → pick installs and a version → a time at least 4 hours ahead (or an emergency
update: 15 minutes, a reason and your 2FA code). A batch runs one church at a time and stops at
the first failure; a failed update rolls back by itself.
- [ ] An agent that's offline when a job is due doesn't run it late: the job is **missed** and
you plan it again (the church gets a new notice).
## 6. New churches
- [ ] First, a throwaway church on a real server (a test domain): New install → the report email
with the FASTPANEL steps → add the nginx site with its two ports → **Proxy done** → live.
Uninstall it after (it exports first; check the download link works).
- [ ] Then real churches the same way.
## 7. Decommissioning the old setup
Only after every site is adopted and has run on 2.3.0 for a while:
- [ ] Each adopted site keeps its old folder 14 days for `adopt --undo`. After that, `crosscode
status` warns about it and shows the `rm -rf` to remove it; once removed, undo isn't possible.
- [ ] The archived `hope-events` repo stays read-only (its `v…` tags are the history).
- [ ] Old PM2 processes or cron jobs from the hand-run deploys: check `pm2 ls` for each site user
shows only `<church>-events-backend` and `<church>-events-frontend`.
## If something goes wrong
| Symptom | Look at |
|---|---|
| A server doesn't show on the Hub | `journalctl -u crosscode-agent`; the server can reach `HUB_PUBLIC_URL` |
| A job is refused by the agent | The job key changed (the server must join again) or the server's clock is off |
| A new install's tunnels don't open | `journalctl -u frpc`; Hub → Jobs → the central `check-ports` job |
| A site is down after an agent restart | `systemctl status pm2-<user>` (sites run under their own unit, not the agent's) |
| An update failed | Hub → Jobs → its log; it rolled back. A failed migration is marked rolled back, so the next update isn't blocked |
+1 -1
View File
@@ -40,7 +40,7 @@
}, },
"apps/agent": { "apps/agent": {
"name": "@crosscode/agent", "name": "@crosscode/agent",
"version": "1.0.0", "version": "1.0.1",
"license": "UNLICENSED", "license": "UNLICENSED",
"dependencies": { "dependencies": {
"@crosscode/contracts": "*", "@crosscode/contracts": "*",