diff --git a/apps/agent/README.md b/apps/agent/README.md new file mode 100644 index 0000000..62663e2 --- /dev/null +++ b/apps/agent/README.md @@ -0,0 +1,55 @@ +# Cross Code server agent + +Runs on every server (docs/AGENT-1.0-PLAN.md). It polls Cross Code Hub every 30 seconds for +signed jobs, runs the `crosscode` script for them, sends the results, and sends the server's +stats every 60 seconds. **It listens on nothing**: no port, no URL; every request goes out to the Hub. + +## Joining a server + +Hub → Servers → **Add server** shows a command with a one-time token (valid an hour): + +```bash +curl -fsSL https://c-git.crosscode.co.za/joshua/crosscode-platform/raw/branch/main/apps/agent/deploy/install.sh \ + | sudo bash -s -- --hub https://hub.crosscode.co.za --token +``` + +Add `--central` on the central (FastPanel) server: there the agent is read-only and only runs +`check-ports` and `status`. The script installs Node 24 if needed, the agent in +`/opt/crosscode/agent/releases/` (with `current` pointing at it), the `crosscode` script in +`/opt/crosscode/tools` (church servers), makes the server's two key pairs, joins, and starts +`crosscode-agent.service`. + +## On the server + +| What | Where | +|------|-------| +| Config (Hub address, server id, the Hub's pinned job key) | `/etc/crosscode/agent.json` | +| Private keys (never leave the server) | `/etc/crosscode/agent-ed25519.pem`, `agent-x25519.pem` | +| Seen job ids (90 days), unsent reports, exports in progress | `/var/lib/crosscode-agent/` | +| A job's opened config, only while it runs | `/run/crosscode-agent/` | +| Logs | `journalctl -u crosscode-agent` | + +## Security + +- **Jobs are signed** by the Hub's job key (Ed25519, pinned when joining). The agent refuses a + job that's unsigned, for another server, outside its time window, already seen, of an unknown + type, or with an argument outside its schema. Refusals are reported to the Hub. +- **Only the `crosscode` script runs**, with arguments built from the checked job (an array, + never a shell). There's no "run this command" job. +- **Config** (database admin, server settings) arrives sealed to this server's X25519 key. It's + opened into a root-only file for the job and removed straight after. +- **Log lines** sent with a report are filtered for passwords, tokens, keys and licence keys. +- **Exports** are encrypted on the server with a new key; the key travels in the job's report + so the Hub can email the download link, and the Hub doesn't store it. + +## Updates + +An `update-agent` job builds the new release next to the running one, switches `current`, and +restarts. If the new agent hasn't polled the Hub within 150 seconds, `revert.sh` switches back. + +## Development + +```bash +npm run build -w apps/agent # dist/cli.js: one file, everything bundled +npm test -w apps/agent # the no-listening-socket test runs on Linux, after a build +``` diff --git a/apps/agent/deploy/crosscode-agent.service b/apps/agent/deploy/crosscode-agent.service new file mode 100644 index 0000000..2e61218 --- /dev/null +++ b/apps/agent/deploy/crosscode-agent.service @@ -0,0 +1,23 @@ +# The Cross Code server agent (docs/AGENT-1.0-PLAN.md §3). Root, because the crosscode script it +# runs needs root (PostgreSQL, PM2 for each app user, frpc). It listens on nothing. +[Unit] +Description=Cross Code server agent +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/node /opt/crosscode/agent/current/apps/agent/dist/cli.js run +Restart=always +RestartSec=10 +RuntimeDirectory=crosscode-agent +RuntimeDirectoryMode=0700 +StateDirectory=crosscode-agent +StateDirectoryMode=0700 +Environment=NODE_ENV=production +# Logs: journalctl -u crosscode-agent +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/apps/agent/deploy/install.sh b/apps/agent/deploy/install.sh new file mode 100644 index 0000000..f5d626b --- /dev/null +++ b/apps/agent/deploy/install.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Installs the Cross Code server agent and joins this server to the Hub +# (docs/AGENT-1.0-PLAN.md §3). Hub → Servers → Add server shows the command, with a one-time token: +# +# curl -fsSL https://c-git.crosscode.co.za/joshua/crosscode-platform/raw/branch/main/apps/agent/deploy/install.sh \ +# | sudo bash -s -- --hub https://hub.crosscode.co.za --token [--central] +# +# Church servers also get the crosscode script (/opt/crosscode/tools, linked as +# /usr/local/bin/crosscode). The central server (--central) only gets the agent, which is +# read-only there. Running it again updates nothing that's there already; use --force to join again. +set -Eeuo pipefail + +REPO="${CROSSCODE_REPO:-https://c-git.crosscode.co.za/joshua/crosscode-platform.git}" +ROOT=/opt/crosscode/agent +TOOLS=/opt/crosscode/tools +HUB="" TOKEN="" MODE=church VERSION="" FORCE="" + +die() { printf '!!> %s\n' "$*" >&2; exit 1; } +step() { printf '==> %s\n' "$*"; } + +while [ $# -gt 0 ]; do + case "$1" in + --hub) HUB="$2"; shift 2 ;; + --token) TOKEN="$2"; shift 2 ;; + --central) MODE=central; shift ;; + --version) VERSION="$2"; shift 2 ;; + --force) FORCE=--force; shift ;; + *) die "Unknown option $1" ;; + esac +done +[ "$(id -u)" = 0 ] || die "Run as root (sudo bash -s -- …)." +[[ "$HUB" =~ ^https?:// ]] || die "--hub is required." +[[ "$TOKEN" =~ ^[A-Za-z0-9_-]{32,128}$ ]] || die "--token is required (Hub → Servers → Add server)." +[ -z "$VERSION" ] || [[ "$VERSION" =~ ^agent@[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "--version: expected agent@x.y.z." + +step "System packages" +export DEBIAN_FRONTEND=noninteractive +apt-get update -q +apt-get install -y -q git curl ca-certificates +if ! command -v node >/dev/null || [ "$(node -p 'process.versions.node.split(".")[0]')" -lt 24 ]; then + step "Node.js 24" + curl -fsSL https://deb.nodesource.com/setup_24.x | bash - + apt-get install -y -q nodejs +fi + +if [ -z "$VERSION" ]; then + VERSION="$(git ls-remote --tags --refs "$REPO" 'agent@*' | sed 's#.*refs/tags/##' | sort -t@ -k2 -V | tail -n 1)" + [ -n "$VERSION" ] || die "No agent@x.y.z release yet." +fi +DIR="$ROOT/releases/${VERSION#agent@}" + +step "Agent $VERSION in $DIR" +mkdir -p "$ROOT/releases" +chmod 755 /opt/crosscode "$ROOT" 2>/dev/null || true +if [ ! -d "$DIR/.git" ]; then + git clone --quiet --depth 1 --branch "$VERSION" "$REPO" "$DIR" +fi +( + cd "$DIR" + NODE_ENV=development npm ci --no-audit --no-fund --include=dev -w packages/contracts -w apps/agent + npm run build -w packages/contracts + npm run build -w apps/agent +) +[ "$(node "$DIR/apps/agent/dist/cli.js" version)" = "${VERSION#agent@}" ] || die "The build doesn't report $VERSION." +ln -sfn "$DIR" "$ROOT/current" +install -m 755 "$DIR/apps/agent/deploy/revert.sh" "$ROOT/revert.sh" + +if [ "$MODE" = church ]; then + step "The crosscode script ($TOOLS)" + [ -d "$TOOLS/.git" ] || git clone --quiet "$REPO" "$TOOLS" + ln -sfn "$TOOLS/scripts/crosscode" /usr/local/bin/crosscode +fi + +step "Joining the Hub ($HUB)" +mkdir -p /etc/crosscode /var/lib/crosscode-agent +chmod 700 /etc/crosscode /var/lib/crosscode-agent +args=(join --hub "$HUB" --token "$TOKEN") +[ "$MODE" = central ] && args+=(--central) +[ -n "$FORCE" ] && args+=("$FORCE") +if [ -f /etc/crosscode/agent.json ] && [ -z "$FORCE" ]; then + echo " Already joined (/etc/crosscode/agent.json); keeping it." +else + node "$ROOT/current/apps/agent/dist/cli.js" "${args[@]}" +fi + +step "systemd" +install -m 644 "$DIR/apps/agent/deploy/crosscode-agent.service" /etc/systemd/system/crosscode-agent.service +systemctl daemon-reload +systemctl enable --now crosscode-agent +systemctl restart crosscode-agent +step "Done: the server shows on Hub → Servers within a minute (journalctl -u crosscode-agent)." diff --git a/apps/agent/deploy/revert.sh b/apps/agent/deploy/revert.sh new file mode 100644 index 0000000..a4ff804 --- /dev/null +++ b/apps/agent/deploy/revert.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Started by a one-shot systemd timer after an agent update (src/update.ts). If the new agent +# hasn't polled the Hub by now, update-pending.json is still there: switch back to the previous +# release and restart. Copied to /opt/crosscode/agent/revert.sh by install.sh. +set -euo pipefail +ROOT=/opt/crosscode/agent +PENDING=/var/lib/crosscode-agent/update-pending.json +[ -f "$PENDING" ] || exit 0 +[ -L "$ROOT/previous" ] || { echo "crosscode-agent revert: no previous release" >&2; exit 1; } +ln -sfn "$(readlink "$ROOT/previous")" "$ROOT/current.tmp" +mv -T "$ROOT/current.tmp" "$ROOT/current" +rm -f "$PENDING" +systemctl restart crosscode-agent +echo "crosscode-agent revert: the new agent didn't poll in time; back on $(readlink "$ROOT/current")" >&2 diff --git a/apps/agent/jest.config.js b/apps/agent/jest.config.js new file mode 100644 index 0000000..8b0f9cf --- /dev/null +++ b/apps/agent/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + testEnvironment: "node", + testMatch: ["/tests/**/*.test.ts"], + transform: { "^.+\\.ts$": ["ts-jest", { tsconfig: { module: "commonjs", moduleResolution: "node", isolatedModules: true } }] }, + moduleFileExtensions: ["ts", "js", "json"], + modulePathIgnorePatterns: ["/dist/"], +}; diff --git a/apps/agent/package.json b/apps/agent/package.json new file mode 100644 index 0000000..e3845a6 --- /dev/null +++ b/apps/agent/package.json @@ -0,0 +1,18 @@ +{ + "name": "@crosscode/agent", + "version": "1.0.0", + "private": true, + "description": "The server agent: runs the crosscode script for signed jobs from Cross Code Hub, reports stats (docs/AGENT-1.0-PLAN.md)", + "license": "UNLICENSED", + "bin": { "crosscode-agent": "./dist/cli.js" }, + "scripts": { + "build": "tsup", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "jest" + }, + "dependencies": { + "@crosscode/contracts": "*", + "zod": "^4.6.7" + }, + "engines": { "node": ">=24" } +} diff --git a/apps/agent/src/central.ts b/apps/agent/src/central.ts new file mode 100644 index 0000000..09097f5 --- /dev/null +++ b/apps/agent/src/central.ts @@ -0,0 +1,98 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import { paths } from "./paths"; + +/** + * The central server (AGENT-1.0-PLAN §6): read-only checks of frps for `check-ports`. Nothing on + * the central server is ever changed: no frps edits, no firewall, no FastPanel. + */ + +export interface FrpsSettings { + proxyBindAddr: string | null; + allowPorts: string | null; +} + +/** frps.ini's [common] proxy_bind_addr and allow_ports (null when not set or no file). */ +export function parseFrpsIni(text: string): FrpsSettings { + let inCommon = false; + const out: FrpsSettings = { proxyBindAddr: null, allowPorts: null }; + for (const raw of text.split(/\r?\n/)) { + const line = raw.trim(); + if (/^\[.*\]$/.test(line)) { inCommon = line === "[common]"; continue; } + if (!inCommon || line.startsWith("#") || line.startsWith(";")) continue; + const m = /^([a-z_]+)\s*=\s*(.*)$/.exec(line); + if (!m) continue; + if (m[1] === "proxy_bind_addr") out.proxyBindAddr = m[2]!.trim() || null; + if (m[1] === "allow_ports") out.allowPorts = m[2]!.trim() || null; + } + return out; +} + +export function readFrps(file = paths.frpsIni): FrpsSettings | null { + try { + return parseFrpsIni(fs.readFileSync(file, "utf8")); + } catch { + return null; + } +} + +/** Whether a port is inside allow_ports ("2000-3000,3001,4000-5000"); no allow_ports allows all. */ +export function portAllowed(port: number, allowPorts: string | null): boolean { + if (!allowPorts) return true; + return allowPorts.split(",").map((s) => s.trim()).filter(Boolean).some((range) => { + const [a, b] = range.split("-").map((n) => Number(n.trim())); + if (!Number.isFinite(a)) return false; + return b === undefined ? port === a : Number.isFinite(b) && port >= a! && port <= b; + }); +} + +/** Listening TCP sockets from `ss -ltnH`: port → local addresses. */ +export function parseSs(text: string): Map { + const out = new Map(); + for (const line of text.split("\n")) { + const cols = line.trim().split(/\s+/); + const local = cols[3]; + if (!local) continue; + const i = local.lastIndexOf(":"); + const port = Number(local.slice(i + 1)); + const addr = local.slice(0, i).replace(/^\[|\]$/g, "").replace(/%.*$/, ""); + if (!Number.isFinite(port)) continue; + out.set(port, [...(out.get(port) ?? []), addr]); + } + return out; +} + +const LOOPBACK = new Set(["127.0.0.1", "::1", "::ffff:127.0.0.1"]); + +export interface PortCheck { + port: number; + allowed: boolean; + listening: boolean; + loopbackOnly: boolean; + ok: boolean; +} + +export function checkPorts(ports: number[], frps: FrpsSettings | null, listening: Map): { ports: PortCheck[]; proxyBindAddr: string | null; warnings: string[] } { + const warnings: string[] = []; + if (!frps) warnings.push(`No frps.ini at ${paths.frpsIni}.`); + const bind = frps?.proxyBindAddr ?? null; + if (frps && (!bind || !LOOPBACK.has(bind))) warnings.push(`proxy_bind_addr is ${bind ? `"${bind}"` : "not set"}: tunnels can be reached straight from the internet. Set proxy_bind_addr = 127.0.0.1 in frps.ini.`); + const results = ports.map((port) => { + const addrs = listening.get(port) ?? []; + const allowed = portAllowed(port, frps?.allowPorts ?? null); + const loopbackOnly = addrs.length > 0 && addrs.every((a) => LOOPBACK.has(a)); + return { port, allowed, listening: addrs.length > 0, loopbackOnly, ok: allowed && addrs.length > 0 && loopbackOnly }; + }); + for (const r of results) { + if (!r.allowed) warnings.push(`Port ${r.port} isn't in allow_ports.`); + else if (!r.listening) warnings.push(`Port ${r.port} isn't open in frps (is the church server's frpc connected?).`); + else if (!r.loopbackOnly) warnings.push(`Port ${r.port} is open on more than 127.0.0.1.`); + } + return { ports: results, proxyBindAddr: bind, warnings }; +} + +export function listeningNow(): Promise> { + return new Promise((resolve) => { + execFile("ss", ["-ltnH"], { timeout: 10_000 }, (err, stdout) => resolve(err ? new Map() : parseSs(String(stdout)))); + }); +} diff --git a/apps/agent/src/cli.ts b/apps/agent/src/cli.ts new file mode 100644 index 0000000..b6a7bbe --- /dev/null +++ b/apps/agent/src/cli.ts @@ -0,0 +1,87 @@ +import fs from "node:fs"; +import os from "node:os"; +import { agent } from "@crosscode/contracts"; +import { generateEd25519, generateX25519 } from "@crosscode/contracts/signing"; +import { configFile, ed25519File, loadConfig, saveConfig, writePrivate, x25519File } from "./config"; +import { hubClient, postUnsigned } from "./hub"; +import { Agent } from "./loop"; +import { AGENT_VERSION } from "./stats"; + +/** + * crosscode-agent (AGENT-1.0-PLAN §3) + * crosscode-agent join --hub https://hub.crosscode.co.za --token [--central] + * crosscode-agent run what systemd runs (crosscode-agent.service) + * crosscode-agent version + */ + +const USAGE = `crosscode-agent ${AGENT_VERSION} + + crosscode-agent join --hub --token [--central] [--force] + Join this server to the Hub with a one-time token (Hub → Servers → Add server). + Makes the server's key pairs; the private keys never leave it. + crosscode-agent run poll the Hub for jobs and send stats (run by systemd) + crosscode-agent version +`; + +function flag(args: string[], name: string): string | undefined { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : undefined; +} + +export async function join(args: string[]): Promise { + const hub = flag(args, "--hub"); + const token = flag(args, "--token"); + const mode: agent.AgentMode = args.includes("--central") ? "central" : "church"; + if (!hub || !/^https?:\/\//.test(hub)) throw new Error("--hub is required (https://hub.crosscode.co.za)."); + if (!token) throw new Error("--token is required (Hub → Servers → Add server)."); + if (fs.existsSync(configFile()) && !args.includes("--force")) { + throw new Error(`This server has already joined (${configFile()}). Use --force to join again (the Hub then sees a new server).`); + } + const ed = generateEd25519(); + const x = generateX25519(); + const reply = agent.joinResponseSchema.parse(await postUnsigned(hub, "/api/agent/v1/join", { + agentApiVersion: agent.AGENT_API_VERSION, + token, + mode, + hostname: os.hostname().slice(0, 253), + agentVersion: AGENT_VERSION, + publicKeys: { ed25519: ed.publicKey, x25519: x.publicKey }, + })); + writePrivate(ed25519File(), ed.privateKeyPem); + writePrivate(x25519File(), x.privateKeyPem); + saveConfig({ hubUrl: hub, serverId: reply.serverId, mode, hubJobKey: reply.hubJobKey, pollSeconds: reply.pollSeconds, statsSeconds: reply.statsSeconds, joinedAt: new Date().toISOString() }); + console.log(`Joined the Hub as ${reply.serverId} (${mode}). Start the agent: systemctl enable --now crosscode-agent`); +} + +export async function main(argv = process.argv.slice(2)): Promise { + const [command, ...args] = argv; + switch (command) { + case "version": + case "--version": + console.log(AGENT_VERSION); + return; + case "join": + await join(args); + return; + case "run": { + if (process.getuid && process.getuid() !== 0 && !process.env.CROSSCODE_AGENT_ALLOW_NON_ROOT) { + throw new Error("Run the agent as root (it runs crosscode, which needs root)."); + } + const { config, keys } = loadConfig(); + const a = new Agent({ config, keys, hub: hubClient(config, keys) }); + for (const sig of ["SIGTERM", "SIGINT"] as const) process.on(sig, () => { a.stop(); setTimeout(() => process.exit(0), 1000).unref(); }); + await a.start(); + return; + } + default: + process.stdout.write(USAGE); + if (command && command !== "help" && command !== "--help") process.exitCode = 1; + } +} + +if (require.main === module) { + main().catch((e) => { + console.error(`crosscode-agent: ${(e as Error).message}`); + process.exit(1); + }); +} diff --git a/apps/agent/src/command.ts b/apps/agent/src/command.ts new file mode 100644 index 0000000..c428f01 --- /dev/null +++ b/apps/agent/src/command.ts @@ -0,0 +1,58 @@ +import type { agent } from "@crosscode/contracts"; + +/** + * The `crosscode` arguments for a job (AGENT-1.0-PLAN §4): built from the checked arguments only, + * as an array passed straight to the process (never through a shell). Jobs the agent handles + * itself (update-agent, check-ports) have none. + * + * `--config ` is added by the runner when the job brings config; `--json` always. + */ +export function crosscodeArgs(job: agent.AgentJob, opts: { exportFile?: string } = {}): string[] | null { + switch (job.type) { + case "install": { + const a = job.args; + if (a.products.length !== 1 || a.products[0] !== "events") throw new Error("This agent can only install Events so far."); + const domain = a.domains.events; + const ports = a.ports.events; + if (!domain || !ports) throw new Error("An Events install needs its domain and ports."); + // The church's address: the Events domain without "events." (crosscode uses it for the + // church.env ADDRESS; the domain itself goes in --events-domain). + const address = domain.startsWith("events.") ? domain.slice("events.".length) : domain; + const out = ["install", a.church, "--products", "events", "--domain", address, "--events-domain", domain, + "--api-port", String(ports.api), "--web-port", String(ports.web), "--frp"]; + const v = a.versions?.events; + if (v) out.push("--version", v); + return out; + } + case "add-product": + return ["add", job.args.church, job.args.product]; + case "deploy": { + const out = ["deploy", job.args.church, ...job.args.products]; + for (const v of job.args.versions ?? []) out.push("--version", v); + if (job.args.check) out.push("--check"); + return out; + } + case "adopt": { + const out = ["adopt", job.args.church, "--events-dir", job.args.eventsDir]; + if (job.args.version) out.push("--version", job.args.version); + if (job.args.address) out.push("--address", job.args.address); + return out; + } + case "adopt-undo": + return ["adopt", job.args.church, "--undo"]; + case "restart": + return ["restart", job.args.church, ...(job.args.product ? [job.args.product] : [])]; + case "export": + if (!opts.exportFile) throw new Error("export needs a file to write"); + return ["export", job.args.church, "--out", opts.exportFile]; + case "uninstall": + return ["uninstall", job.args.church, "--confirm", job.args.church]; + case "purge": + return ["purge", job.args.church, "--confirm", job.args.church]; + case "status": + return ["status", ...(job.args.church ? [job.args.church] : []), "--no-fetch"]; + case "update-agent": + case "check-ports": + return null; + } +} diff --git a/apps/agent/src/config.ts b/apps/agent/src/config.ts new file mode 100644 index 0000000..3114d9d --- /dev/null +++ b/apps/agent/src/config.ts @@ -0,0 +1,49 @@ +import fs from "node:fs"; +import path from "node:path"; +import { z } from "zod"; +import { agent } from "@crosscode/contracts"; +import { publicKeyOf } from "@crosscode/contracts/signing"; +import { paths } from "./paths"; + +/** /etc/crosscode/agent.json, written by `crosscode-agent join`. No secrets: the keys are files next to it. */ +export const agentConfigSchema = z.object({ + hubUrl: z.url({ protocol: /^https?$/ }), + serverId: z.string().regex(/^srv_[A-Za-z0-9_-]{8,64}$/), + mode: agent.agentModeSchema, + /** The Hub's job-signing key, pinned when joining. */ + hubJobKey: z.string().regex(/^[A-Za-z0-9_-]{43}$/), + pollSeconds: z.int().min(5).max(600).default(30), + statsSeconds: z.int().min(15).max(3600).default(60), + joinedAt: z.string(), +}); +export type AgentConfig = z.infer; + +export interface Keys { + ed25519Pem: string; + x25519Pem: string; +} + +export const configFile = () => path.join(paths.configDir, "agent.json"); +export const ed25519File = () => path.join(paths.configDir, "agent-ed25519.pem"); +export const x25519File = () => path.join(paths.configDir, "agent-x25519.pem"); + +/** Writes a file only root can read, atomically. */ +export function writePrivate(file: string, text: string): void { + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + const tmp = `${file}.tmp-${process.pid}`; + fs.writeFileSync(tmp, text, { mode: 0o600 }); + fs.renameSync(tmp, file); +} + +export function loadConfig(): { config: AgentConfig; keys: Keys } { + const config = agentConfigSchema.parse(JSON.parse(fs.readFileSync(configFile(), "utf8"))); + const keys = { ed25519Pem: fs.readFileSync(ed25519File(), "utf8"), x25519Pem: fs.readFileSync(x25519File(), "utf8") }; + // A broken pair would sign requests the Hub can't check; better to stop at start. + publicKeyOf(keys.ed25519Pem); + publicKeyOf(keys.x25519Pem); + return { config, keys }; +} + +export function saveConfig(config: AgentConfig): void { + writePrivate(configFile(), `${JSON.stringify(config, null, 2)}\n`); +} diff --git a/apps/agent/src/hub.ts b/apps/agent/src/hub.ts new file mode 100644 index 0000000..dc87b7d --- /dev/null +++ b/apps/agent/src/hub.ts @@ -0,0 +1,75 @@ +import { signAgentRequest } from "@crosscode/contracts/signing"; +import type { AgentConfig, Keys } from "./config"; + +/** + * Requests to the Hub. The agent only ever makes outgoing requests (it listens on nothing, + * AGENT-1.0-PLAN §2 rule 1); every one after joining is signed with the server's Ed25519 key. + */ + +export class HubRequestError extends Error { + constructor(message: string, readonly status: number, readonly code: string | null) { + super(message); + } +} + +export type Fetch = typeof fetch; + +export interface HubClient { + post(path: string, body: unknown): Promise; + put(path: string, body: Uint8Array, contentType: string): Promise; +} + +const TIMEOUT_MS = 30_000; +const UPLOAD_TIMEOUT_MS = 10 * 60_000; + +async function send(fetcher: Fetch, url: string, init: RequestInit, timeoutMs: number): Promise { + let res: Response; + try { + res = await fetcher(url, { ...init, signal: AbortSignal.timeout(timeoutMs) }); + } catch (e) { + throw new HubRequestError(`The Hub can't be reached: ${(e as Error).message}`, 0, "HUB_UNREACHABLE"); + } + if (res.status === 204) return null; + let body: unknown; + try { + body = await res.json(); + } catch { + body = null; + } + if (!res.ok) { + const b = (body ?? {}) as { message?: string; code?: string }; + throw new HubRequestError(b.message || `The Hub answered ${res.status}`, res.status, b.code ?? null); + } + return body; +} + +/** An unsigned request (only `join`, where the one-time token is the proof). */ +export function postUnsigned(hubUrl: string, path: string, body: unknown, fetcher: Fetch = fetch): Promise { + return send(fetcher, `${hubUrl.replace(/\/$/, "")}${path}`, { + method: "POST", + headers: { "content-type": "application/json", "user-agent": "crosscode-agent" }, + body: JSON.stringify(body), + }, TIMEOUT_MS); +} + +export function hubClient(config: AgentConfig, keys: Keys, fetcher: Fetch = fetch): HubClient { + const base = config.hubUrl.replace(/\/$/, ""); + const signed = (raw: string | Uint8Array) => signAgentRequest({ serverId: config.serverId, privateKeyPem: keys.ed25519Pem, rawBody: raw }); + return { + post(path, body) { + const raw = JSON.stringify(body); + return send(fetcher, `${base}${path}`, { + method: "POST", + headers: { "content-type": "application/json", "user-agent": "crosscode-agent", ...signed(raw) }, + body: raw, + }, TIMEOUT_MS); + }, + put(path, body, contentType) { + return send(fetcher, `${base}${path}`, { + method: "PUT", + headers: { "content-type": contentType, "user-agent": "crosscode-agent", ...signed(body) }, + body, + }, UPLOAD_TIMEOUT_MS); + }, + }; +} diff --git a/apps/agent/src/loop.ts b/apps/agent/src/loop.ts new file mode 100644 index 0000000..491afd1 --- /dev/null +++ b/apps/agent/src/loop.ts @@ -0,0 +1,226 @@ +import fs from "node:fs"; +import path from "node:path"; +import { agent } from "@crosscode/contracts"; +import { encryptExport } from "@crosscode/contracts/signing"; +import { checkPorts, listeningNow, readFrps } from "./central"; +import { crosscodeArgs } from "./command"; +import type { AgentConfig, Keys } from "./config"; +import { type HubClient, HubRequestError } from "./hub"; +import { paths } from "./paths"; +import { runCrosscode, type RunResult } from "./runner"; +import { AGENT_VERSION, collectStats } from "./stats"; +import { confirmUpdate, updateAgent } from "./update"; +import { checkJob, SeenJobs } from "./verify"; + +/** + * The agent's life: poll the Hub every 30 s for jobs due, send stats every 60 s, run one job at a + * time and send each result. Reports that can't be sent are kept in an outbox and sent later, so + * a Hub outage never loses a job's result. Nothing here listens: every request goes out. + */ + +const log = (...a: unknown[]) => console.log(new Date().toISOString(), ...a); + +export interface AgentDeps { + config: AgentConfig; + keys: Keys; + hub: HubClient; + seen?: SeenJobs; + /** For tests: replaces running crosscode and the stats. */ + run?: (args: string[], job: agent.AgentJob) => Promise; + stats?: () => Promise; + now?: () => Date; +} + +const outboxDir = () => path.join(paths.stateDir, "outbox"); + +function report(job: { id: string; type: agent.JobType }, serverId: string, r: Omit & { log?: string[] }): agent.AgentReport { + return { + schema: "crosscode.agent-report/1", + jobId: job.id, + serverId, + type: job.type, + status: r.status, + startedAt: r.startedAt, + finishedAt: r.finishedAt, + steps: r.steps.slice(0, 200), + result: r.result, + ...(r.message ? { message: r.message.slice(0, 2000) } : {}), + log: (r.log ?? []).slice(-200), + }; +} + +export class Agent { + private seen: SeenJobs; + private stopped = false; + private confirmed = false; + private failures = 0; + + constructor(private deps: AgentDeps) { + this.seen = deps.seen ?? new SeenJobs(); + } + + private now(): Date { + return this.deps.now?.() ?? new Date(); + } + + /** Sends a report, or keeps it in the outbox to send later. */ + async send(r: agent.AgentReport): Promise { + try { + await this.deps.hub.post("/api/agent/v1/report", r); + } catch (e) { + log(`[report] ${r.jobId}: couldn't send (${(e as Error).message}); kept to send later.`); + fs.mkdirSync(outboxDir(), { recursive: true, mode: 0o700 }); + fs.writeFileSync(path.join(outboxDir(), `${r.jobId}.json`), JSON.stringify(r), { mode: 0o600 }); + } + } + + async flushOutbox(): Promise { + let files: string[]; + try { files = fs.readdirSync(outboxDir()).filter((f) => f.endsWith(".json")); } catch { return; } + for (const f of files) { + const file = path.join(outboxDir(), f); + try { + await this.deps.hub.post("/api/agent/v1/report", JSON.parse(fs.readFileSync(file, "utf8"))); + fs.rmSync(file, { force: true }); + } catch (e) { + // A report the Hub refuses outright (4xx) would never go: drop it rather than retry forever. + if (e instanceof HubRequestError && e.status >= 400 && e.status < 500) fs.rmSync(file, { force: true }); + return; + } + } + } + + /** One poll: fetch the jobs due and run them, one after the other. Returns the next delay (s). */ + async pollOnce(): Promise { + const { config } = this.deps; + let reply: { jobs: unknown[]; pollSeconds: number }; + try { + reply = agent.pollResponseSchema.parse(await this.deps.hub.post("/api/agent/v1/poll", { + agentApiVersion: agent.AGENT_API_VERSION, agentVersion: AGENT_VERSION, mode: config.mode, runningJobId: null, + })); + } catch (e) { + this.failures++; + const delay = Math.min(300, config.pollSeconds * 2 ** Math.min(this.failures, 4)); + log(`[poll] ${(e as Error).message}; next try in ${delay}s.`); + return delay; + } + this.failures = 0; + if (!this.confirmed) { + confirmUpdate(); + this.confirmed = true; + } + await this.flushOutbox(); + for (const raw of reply.jobs) { + if (this.stopped) break; + await this.handle(raw); + } + return reply.pollSeconds || config.pollSeconds; + } + + async handle(raw: unknown): Promise { + const { config } = this.deps; + const verdict = checkJob(raw, config, this.seen, this.now()); + const at = this.now().toISOString(); + if (!verdict.ok) { + log(`[job] ${verdict.id ?? "(no id)"} ${verdict.status}: ${verdict.message}`); + if (verdict.id && verdict.type && !this.seen.has(verdict.id)) { + this.seen.add(verdict.id); + await this.send(report({ id: verdict.id, type: verdict.type }, config.serverId, { status: verdict.status, steps: [], result: {}, message: verdict.message, startedAt: at, finishedAt: at })); + } + return; + } + const job = verdict.job; + // Seen before it runs, so a crash part-way can never run it twice. + this.seen.add(job.id); + log(`[job] ${job.id} ${job.type} ${JSON.stringify(job.args)}`); + const result = await this.execute(job); + log(`[job] ${job.id} ${result.status}${result.message ? `: ${result.message}` : ""}`); + await this.send(report(job, config.serverId, result)); + if ((result as RunResult & { restart?: boolean }).restart) { + log("[agent] Exiting so systemd starts the new version."); + this.stopped = true; + setTimeout(() => process.exit(0), 500); + } + } + + private async execute(job: agent.AgentJob): Promise { + const startedAt = this.now().toISOString(); + const limit = agent.JOB_TIME_LIMIT_MINUTES[job.type]; + try { + switch (job.type) { + case "update-agent": { + const u = await updateAgent(job.args.version, AGENT_VERSION); + return { status: u.status, steps: u.steps, result: u.result, message: u.message, log: [], startedAt, finishedAt: new Date().toISOString(), restart: u.restart }; + } + case "check-ports": { + const res = checkPorts(job.args.ports, readFrps(), await listeningNow()); + const ok = res.ports.every((p) => p.ok) && res.warnings.length === 0; + return { status: ok ? "succeeded" : "failed", steps: [], result: res as unknown as Record, message: res.warnings.join(" ") || undefined, log: [], startedAt, finishedAt: new Date().toISOString() }; + } + case "export": + return await this.runExport(job, limit); + default: { + const args = crosscodeArgs(job); + if (!args) throw new Error(`No command for ${job.type}`); + return await this.run(args, job, limit); + } + } + } catch (e) { + return { status: "failed", steps: [], result: {}, message: (e as Error).message, log: [], startedAt, finishedAt: new Date().toISOString() }; + } + } + + private run(args: string[], job: agent.AgentJob, limit: number): Promise { + if (this.deps.run) return this.deps.run(args, job); + return runCrosscode({ args, sealedConfig: job.config ?? null, x25519Pem: this.deps.keys.x25519Pem, timeLimitMinutes: limit }); + } + + /** + * export: crosscode writes the ZIP, the agent encrypts it with a fresh key and uploads it. The + * key goes to the Hub only in this job's report, so it can email the download link; the Hub + * doesn't keep it (AGENT-1.0-PLAN §9). + */ + private async runExport(job: Extract, limit: number): Promise { + const dir = path.join(paths.stateDir, "exports"); + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + const zip = path.join(dir, `${job.args.exportId}.zip`); + fs.rmSync(zip, { force: true }); + try { + const r = await this.run(crosscodeArgs(job, { exportFile: zip })!, job, limit); + if (r.status !== "succeeded") return r; + const { key, file } = encryptExport(fs.readFileSync(zip)); + await this.deps.hub.put(`/api/agent/v1/exports/${job.args.exportId}`, file, "application/octet-stream"); + return { ...r, result: { ...r.result, file: undefined, encryptedBytes: file.length, key } }; + } finally { + fs.rmSync(zip, { force: true }); + } + } + + async statsOnce(): Promise { + try { + const stats = await (this.deps.stats ?? (() => collectStats(this.deps.config.mode)))(); + await this.deps.hub.post("/api/agent/v1/stats", stats); + } catch (e) { + log(`[stats] ${(e as Error).message}`); + } + } + + /** Runs until stopped: polls and stats on their own timers, never at the same time as a job. */ + async start(): Promise { + log(`[agent] ${AGENT_VERSION} (${this.deps.config.mode}) for ${this.deps.config.serverId}, Hub ${this.deps.config.hubUrl}`); + let nextStats = 0; + while (!this.stopped) { + if (Date.now() >= nextStats) { + await this.statsOnce(); + nextStats = Date.now() + this.deps.config.statsSeconds * 1000; + } + const delay = await this.pollOnce(); + await new Promise((r) => setTimeout(r, Math.max(5, Math.min(delay, this.deps.config.statsSeconds)) * 1000)); + } + } + + stop(): void { + this.stopped = true; + } +} + diff --git a/apps/agent/src/paths.ts b/apps/agent/src/paths.ts new file mode 100644 index 0000000..53b0718 --- /dev/null +++ b/apps/agent/src/paths.ts @@ -0,0 +1,22 @@ +/** + * Where the agent keeps things on a server (docs/AGENT-1.0-PLAN.md §3). Each can be moved with an + * environment variable, which the tests use. + */ +const env = (name: string, fallback: string): string => process.env[name] || fallback; + +export const paths = { + /** agent.json and the two private keys, root only (mode 600). */ + get configDir() { return env("CROSSCODE_AGENT_CONFIG_DIR", "/etc/crosscode"); }, + /** Seen job ids (replay protection) and exports waiting to upload. */ + get stateDir() { return env("CROSSCODE_AGENT_STATE_DIR", "/var/lib/crosscode-agent"); }, + /** A job's opened config, only while the job runs (tmpfs). */ + get runDir() { return env("CROSSCODE_AGENT_RUN_DIR", "/run/crosscode-agent"); }, + /** The crosscode script (a checkout of the monorepo in /opt/crosscode/tools). */ + get crosscode() { return env("CROSSCODE_BIN", "/usr/local/bin/crosscode"); }, + /** Agent releases: releases// (checkouts), current → the running one, previous. */ + get agentRoot() { return env("CROSSCODE_AGENT_ROOT", "/opt/crosscode/agent"); }, + /** frps.ini on the central server. */ + get frpsIni() { return env("CROSSCODE_FRPS_INI", "/etc/frp/frps.ini"); }, + /** The churches on a church server. */ + get sites() { return env("CROSSCODE_SITES", "/var/www/sites"); }, +}; diff --git a/apps/agent/src/redact.ts b/apps/agent/src/redact.ts new file mode 100644 index 0000000..4c3bc09 --- /dev/null +++ b/apps/agent/src/redact.ts @@ -0,0 +1,51 @@ +/** + * Log lines are filtered for anything that looks like a secret before they leave the server + * (AGENT-1.0-PLAN §4, rule 7). The crosscode script never prints .env values, but tools it runs + * (npm, Prisma, git) might print a URL with a password or a token in an error. + */ + +const RULES: [RegExp, string][] = [ + // user:password@ in any URL (postgresql://, https://). + [/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s@/]+@/gi, "$1***@"], + // KEY=value and KEY: value for keys that hold secrets. + [/\b([A-Z0-9_]*(SECRET|PASSWORD|PASSWD|TOKEN|API_KEY|PRIVATE_KEY|DSN|LICENCE_KEY|LICENSE_KEY)[A-Z0-9_]*)(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/g, "$1$3***"], + // Authorization headers and bearer tokens. + [/\b(authorization\s*:\s*)(bearer|basic)\s+\S+/gi, "$1$2 ***"], + [/\bbearer\s+[A-Za-z0-9._~+/=-]{16,}/gi, "Bearer ***"], + // PEM blocks. + [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, "[private key]"], + // Cross Code licence keys (CEH-7K2M-…). + [/\b(CEH|CCA|CCW|CCP)-[A-Z0-9]{4}(-[A-Z0-9]{4}){3}\b/g, "$1-****"], + // Long random-looking strings (hex or base64 of 32+ characters): likely keys or tokens. + [/\b[A-Fa-f0-9]{40,}\b/g, "***"], + [/\b[A-Za-z0-9+/_-]{48,}={0,2}/g, "***"], +]; + +export function redact(line: string): string { + let out = line; + for (const [re, to] of RULES) out = out.replace(re, to); + return out.length > 1000 ? `${out.slice(0, 997)}...` : out; +} + +/** Keeps the last `max` lines of output, redacted, without holding all of it in memory. */ +export class LogTail { + private lines: string[] = []; + private partial = ""; + constructor(private max = 200) {} + push(chunk: string): void { + const parts = (this.partial + chunk).split(/\r?\n/); + this.partial = parts.pop() ?? ""; + for (const p of parts) this.add(p); + } + private add(line: string): void { + this.lines.push(redact(line)); + if (this.lines.length > this.max) this.lines.shift(); + } + end(): string[] { + if (this.partial) { + this.add(this.partial); + this.partial = ""; + } + return [...this.lines]; + } +} diff --git a/apps/agent/src/runner.ts b/apps/agent/src/runner.ts new file mode 100644 index 0000000..7346165 --- /dev/null +++ b/apps/agent/src/runner.ts @@ -0,0 +1,116 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { agent } from "@crosscode/contracts"; +import { openConfig } from "@crosscode/contracts/signing"; +import { paths } from "./paths"; +import { LogTail, redact } from "./redact"; + +/** + * Runs `crosscode … --json` for a job: argument array (no shell), the job's config opened to a + * file only root can read and removed as soon as the job ends, a time limit per type (the whole + * process group is stopped past it), and the last 200 log lines, redacted. + */ + +export interface RunResult { + status: agent.JobStatus; + steps: agent.ReportStep[]; + result: Record; + message?: string; + log: string[]; + startedAt: string; + finishedAt: string; +} + +export interface RunOptions { + args: string[]; + sealedConfig?: agent.Sealed | null; + x25519Pem: string; + timeLimitMinutes: number; + /** For tests. */ + bin?: string; +} + +const iso = (d = new Date()) => d.toISOString(); + +function writeConfig(sealed: agent.Sealed, x25519Pem: string): string { + const opened = agent.jobConfigSchema.parse(openConfig(sealed, x25519Pem)); + fs.mkdirSync(paths.runDir, { recursive: true, mode: 0o700 }); + const file = path.join(paths.runDir, `config-${process.pid}-${Date.now()}.json`); + fs.writeFileSync(file, JSON.stringify(opened), { mode: 0o600 }); + return file; +} + +export async function runCrosscode(opts: RunOptions): Promise { + const startedAt = iso(); + let configFile: string | null = null; + try { + if (opts.sealedConfig) { + try { + configFile = writeConfig(opts.sealedConfig, opts.x25519Pem); + } catch (e) { + return { status: "refused", steps: [], result: {}, message: `The job's config couldn't be opened with this server's key: ${(e as Error).message}`, log: [], startedAt, finishedAt: iso() }; + } + } + const args = [...opts.args, "--json", ...(configFile ? ["--config", configFile] : [])]; + return await new Promise((resolve) => { + const tail = new LogTail(200); + let stdout = ""; + const child = spawn(opts.bin ?? paths.crosscode, args, { + detached: true, + stdio: ["ignore", "pipe", "pipe"], + env: { ...process.env, CROSSCODE_NO_UPDATE: process.env.CROSSCODE_NO_UPDATE ?? "", LC_ALL: "C.UTF-8" }, + }); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + try { process.kill(-child.pid!, "SIGTERM"); } catch { /* already gone */ } + setTimeout(() => { try { process.kill(-child.pid!, "SIGKILL"); } catch { /* gone */ } }, 10_000).unref(); + }, opts.timeLimitMinutes * 60_000); + child.stdout.setEncoding("utf8").on("data", (d: string) => { stdout += d; if (stdout.length > 5_000_000) stdout = stdout.slice(-1_000_000); }); + child.stderr.setEncoding("utf8").on("data", (d: string) => tail.push(d)); + child.on("error", (e) => { + clearTimeout(timer); + resolve({ status: "failed", steps: [], result: {}, message: `crosscode couldn't be started: ${e.message}`, log: tail.end(), startedAt, finishedAt: iso() }); + }); + child.on("close", (code) => { + clearTimeout(timer); + const log = tail.end(); + const report = parseReport(stdout); + if (timedOut) { + resolve({ status: "failed", steps: report?.steps ?? [], result: report?.result ?? {}, message: `Stopped after the time limit (${opts.timeLimitMinutes} minutes).`, log, startedAt, finishedAt: iso() }); + return; + } + if (!report) { + resolve({ status: code === 0 ? "succeeded" : "failed", steps: [], result: {}, message: code === 0 ? undefined : `crosscode ended with code ${code} without a report.`, log, startedAt, finishedAt: iso() }); + return; + } + resolve({ + status: report.status, + steps: report.steps, + result: report.result, + message: report.message ? redact(report.message) : undefined, + log, + startedAt: report.startedAt, + finishedAt: report.finishedAt, + }); + }); + }); + } finally { + if (configFile) fs.rmSync(configFile, { force: true }); + } +} + +/** The last line of stdout that's a valid script report. */ +export function parseReport(stdout: string): agent.ScriptReport | null { + const lines = stdout.trim().split("\n").reverse(); + for (const line of lines) { + try { + const parsed = agent.scriptReportSchema.safeParse(JSON.parse(line)); + if (parsed.success) return parsed.data; + } catch { + // not JSON + } + } + return null; +} diff --git a/apps/agent/src/stats.ts b/apps/agent/src/stats.ts new file mode 100644 index 0000000..b1af290 --- /dev/null +++ b/apps/agent/src/stats.ts @@ -0,0 +1,152 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { agent } from "@crosscode/contracts"; +import { readFrps } from "./central"; +import { paths } from "./paths"; + +/** + * The server's stats every 60 s (AGENT-1.0-PLAN §10, crosscode.agent-stats/1): RAM, CPU, disks, + * uptime, versions, frpc, and each PM2 process with its church and product. Read from /proc and + * the tools that are there; anything missing is null rather than an error. + */ + +export const AGENT_VERSION = "1.0.0"; + +function run(cmd: string, args: string[], timeoutMs = 10_000): Promise { + return new Promise((resolve) => { + execFile(cmd, args, { timeout: timeoutMs, maxBuffer: 10_000_000 }, (err, stdout) => resolve(err ? null : String(stdout))); + }); +} + +export function parseMeminfo(text: string): agent.AgentStats["ram"] { + const kb = (key: string) => Number(new RegExp(`^${key}:\\s+(\\d+)`, "m").exec(text)?.[1] ?? 0) * 1024; + const total = kb("MemTotal"); + const available = kb("MemAvailable"); + return { totalBytes: total, usedBytes: Math.max(0, total - available), availableBytes: available, swapTotalBytes: kb("SwapTotal"), swapUsedBytes: Math.max(0, kb("SwapTotal") - kb("SwapFree")) }; +} + +/** CPU busy % between two /proc/stat readings. */ +export function cpuPercent(a: string, b: string): number { + const parse = (t: string) => { + const f = (/^cpu\s+(.+)$/m.exec(t)?.[1] ?? "").trim().split(/\s+/).map(Number); + const idle = (f[3] ?? 0) + (f[4] ?? 0); + return { idle, total: f.reduce((s, n) => s + (Number.isFinite(n) ? n : 0), 0) }; + }; + const x = parse(a), y = parse(b); + const total = y.total - x.total; + if (total <= 0) return 0; + return Math.min(100, Math.max(0, Math.round(((total - (y.idle - x.idle)) / total) * 1000) / 10)); +} + +async function disks(): Promise { + const out = await run("df", ["-P", "-B1", "-x", "tmpfs", "-x", "devtmpfs", "-x", "squashfs", "-x", "overlay"]); + if (!out) return []; + return out.trim().split("\n").slice(1).map((l) => l.trim().split(/\s+/)).filter((c) => c.length >= 6 && c[5]!.startsWith("/")) + .slice(0, 30).map((c) => ({ mount: c[5]!, totalBytes: Number(c[1]), usedBytes: Number(c[2]), freeBytes: Number(c[3]) })); +} + +function osName(): string { + try { + const t = fs.readFileSync("/etc/os-release", "utf8"); + return /^PRETTY_NAME="?([^"\n]+)"?/m.exec(t)?.[1] ?? `${os.type()} ${os.release()}`; + } catch { + return `${os.type()} ${os.release()}`; + } +} + +const versionIn = (text: string | null) => (text ? /(\d+\.\d+(\.\d+)?)/.exec(text)?.[1] ?? null : null); + +/** church.env → { app user → [{ church, product, names }] }, read as data like the script does. */ +export function churchProcesses(sitesDir = paths.sites): Map { + const byUser = new Map(); + let entries: string[]; + try { entries = fs.readdirSync(sitesDir); } catch { return byUser; } + for (const church of entries) { + if (!/^[a-z][a-z0-9-]{0,39}$/.test(church)) continue; + let text: string; + try { text = fs.readFileSync(path.join(sitesDir, church, "church.env"), "utf8"); } catch { continue; } + const vars = new Map(); + for (const line of text.split(/\r?\n/)) { + const m = /^([A-Z][A-Z0-9_]*)=("([^"]*)"|'([^']*)'|([^#\s]*))/.exec(line); + if (m) vars.set(m[1]!, m[3] ?? m[4] ?? m[5] ?? ""); + } + for (const product of ["accounts", "events", "website", "app"] as const) { + const p = product.toUpperCase(); + const user = vars.get(`${p}_USER`); + const names = (vars.get(`${p}_PM2`) ?? "").split(/\s+/).filter(Boolean); + if (!user || !names.length || !/^[a-z_][a-z0-9_-]{0,31}$/.test(user)) continue; + const list = byUser.get(user) ?? []; + list.push({ church, product, names }); + byUser.set(user, list); + } + } + return byUser; +} + +interface Pm2Proc { name?: string; monit?: { memory?: number; cpu?: number }; pm2_env?: { status?: string; restart_time?: number; pm_uptime?: number } } + +async function pm2Processes(): Promise { + const out: agent.AgentStats["processes"] = []; + for (const [user, installs] of churchProcesses()) { + const json = await run("sudo", ["-n", "-u", user, "-H", "--", "pm2", "jlist"]); + let list: Pm2Proc[]; + try { list = JSON.parse(json ?? "[]") as Pm2Proc[]; } catch { list = []; } + for (const p of list) { + const owner = installs.find((i) => i.names.includes(p.name ?? "")); + out.push({ + name: String(p.name ?? "?").slice(0, 100), + church: owner?.church ?? null, + product: owner?.product ?? null, + status: String(p.pm2_env?.status ?? "unknown").slice(0, 40), + memoryBytes: Number(p.monit?.memory ?? 0), + cpuPercent: Number(p.monit?.cpu ?? 0), + restarts: Math.max(0, Math.trunc(Number(p.pm2_env?.restart_time ?? 0))), + uptimeSeconds: p.pm2_env?.pm_uptime ? Math.max(0, (Date.now() - p.pm2_env.pm_uptime) / 1000) : 0, + }); + } + } + return out.slice(0, 200); +} + +/** A systemd unit is active; null when there's no systemctl. (is-active exits non-zero when inactive.) */ +function active(unit: string): Promise { + return new Promise((resolve) => { + execFile("systemctl", ["is-active", unit], { timeout: 10_000 }, (err, stdout) => { + if (err && (err as NodeJS.ErrnoException).code === "ENOENT") resolve(null); + else resolve(String(stdout).trim() === "active"); + }); + }); +} + +export async function collectStats(mode: agent.AgentMode): Promise { + const read = (f: string) => { try { return fs.readFileSync(f, "utf8"); } catch { return ""; } }; + const stat1 = read("/proc/stat"); + await new Promise((r) => setTimeout(r, 1000)); + const stat2 = read("/proc/stat"); + const [load1, load5, load15] = os.loadavg(); + const [diskList, postgres, frpc, processes, frpcRunning] = await Promise.all([ + disks(), + run("psql", ["--version"]), + run("frpc", ["-v"]), + mode === "church" ? pm2Processes() : Promise.resolve([]), + mode === "church" ? active("frpc") : Promise.resolve(null), + ]); + const stats: agent.AgentStats = { + schema: "crosscode.agent-stats/1", + at: new Date().toISOString(), + ram: parseMeminfo(read("/proc/meminfo")), + cpu: { percent: cpuPercent(stat1, stat2), load1: load1 ?? 0, load5: load5 ?? 0, load15: load15 ?? 0, cores: Math.max(1, os.cpus().length) }, + disks: diskList, + uptimeSeconds: os.uptime(), + versions: { os: osName().slice(0, 200), node: process.versions.node, postgres: versionIn(postgres), frpc: versionIn(frpc), agent: AGENT_VERSION }, + frpcRunning, + processes, + }; + if (mode === "central") { + const frps = readFrps(); + stats.central = { proxyBindAddr: frps?.proxyBindAddr ?? null, allowPorts: frps?.allowPorts ?? null, frpsRunning: await active("frps") }; + } + return stats; +} diff --git a/apps/agent/src/update.ts b/apps/agent/src/update.ts new file mode 100644 index 0000000..3c6dd53 --- /dev/null +++ b/apps/agent/src/update.ts @@ -0,0 +1,103 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import type { agent } from "@crosscode/contracts"; +import { paths } from "./paths"; + +/** + * update-agent (AGENT-1.0-PLAN §3): the newest agent@x.y.z is checked out and built next to the + * running copy, `current` is switched to it and the agent exits, so systemd starts the new one. + * A one-shot systemd timer switches back to `previous` unless the new agent clears + * update-pending.json by polling the Hub within 150 seconds. + * + * /opt/crosscode/agent/releases// checkouts of the monorepo at agent@ + * /opt/crosscode/agent/current, previous symlinks to releases + * /opt/crosscode/agent/revert.sh written by the install script + */ + +const REPO = process.env.CROSSCODE_REPO || "https://c-git.crosscode.co.za/joshua/crosscode-platform.git"; +const CONFIRM_SECONDS = 150; + +const pendingFile = () => path.join(paths.stateDir, "update-pending.json"); + +function exec(cmd: string, args: string[], cwd?: string, timeoutMs = 15 * 60_000): Promise { + return new Promise((resolve, reject) => { + execFile(cmd, args, { cwd, timeout: timeoutMs, maxBuffer: 20_000_000, env: { ...process.env, NODE_ENV: "development" } }, (err, stdout, stderr) => { + if (err) reject(new Error(`${cmd} ${args.join(" ")}: ${String(stderr).trim().split("\n").slice(-3).join(" ") || err.message}`)); + else resolve(String(stdout)); + }); + }); +} + +export interface UpdateOutcome { + status: agent.JobStatus; + message: string; + steps: agent.ReportStep[]; + result: Record; + /** Exit after reporting, so systemd starts the new version. */ + restart: boolean; +} + +export async function updateAgent(tag: string, currentVersion: string): Promise { + const version = tag.replace(/^agent@/, ""); + const steps: agent.ReportStep[] = []; + const step = async (name: string, fn: () => Promise) => { + const s: agent.ReportStep = { name, startedAt: new Date().toISOString(), finishedAt: null, outcome: "ok" }; + steps.push(s); + try { + await fn(); + } catch (e) { + s.outcome = "failed"; + s.detail = (e as Error).message.slice(0, 2000); + throw e; + } finally { + s.finishedAt = new Date().toISOString(); + } + }; + if (version === currentVersion) return { status: "succeeded", message: `Already on ${tag}.`, steps, result: { version }, restart: false }; + + const root = paths.agentRoot; + const dir = path.join(root, "releases", version); + try { + await step(`Checking out ${tag}`, async () => { + if (!fs.existsSync(path.join(dir, ".git"))) { + fs.mkdirSync(path.dirname(dir), { recursive: true }); + await exec("git", ["clone", "--quiet", "--depth", "1", "--branch", tag, REPO, dir]); + } + }); + await step("Installing and building", async () => { + await exec("npm", ["ci", "--no-audit", "--no-fund", "--include=dev", "-w", "packages/contracts", "-w", "apps/agent"], dir); + await exec("npm", ["run", "build", "-w", "packages/contracts"], dir); + await exec("npm", ["run", "build", "-w", "apps/agent"], dir); + }); + await step("Checking the new build", async () => { + const out = await exec(process.execPath, [path.join(dir, "apps/agent/dist/cli.js"), "version"], dir, 30_000); + if (out.trim() !== version) throw new Error(`The new build says it's ${out.trim()}, not ${version}.`); + }); + await step("Switching over", async () => { + const current = path.join(root, "current"); + const previous = path.join(root, "previous"); + const old = fs.readlinkSync(current); + fs.writeFileSync(pendingFile(), JSON.stringify({ from: currentVersion, to: version, at: new Date().toISOString() }), { mode: 0o600 }); + replaceLink(previous, old); + replaceLink(current, dir); + // If the new agent doesn't poll within CONFIRM_SECONDS, revert.sh puts `previous` back. + await exec("systemd-run", [`--on-active=${CONFIRM_SECONDS}`, `--unit=crosscode-agent-revert-${Date.now()}`, path.join(root, "revert.sh")]); + }); + } catch (e) { + return { status: "failed", message: `The agent wasn't updated: ${(e as Error).message}`, steps, result: { from: currentVersion, to: version }, restart: false }; + } + return { status: "succeeded", message: `Switching to ${tag}; it confirms by polling within ${CONFIRM_SECONDS} seconds.`, steps, result: { from: currentVersion, to: version }, restart: true }; +} + +function replaceLink(link: string, target: string): void { + const tmp = `${link}.tmp-${process.pid}`; + fs.rmSync(tmp, { force: true }); + fs.symlinkSync(target, tmp); + fs.renameSync(tmp, link); +} + +/** Called after the first successful poll: this version works, so it stays. */ +export function confirmUpdate(): void { + fs.rmSync(pendingFile(), { force: true }); +} diff --git a/apps/agent/src/verify.ts b/apps/agent/src/verify.ts new file mode 100644 index 0000000..2562208 --- /dev/null +++ b/apps/agent/src/verify.ts @@ -0,0 +1,78 @@ +import fs from "node:fs"; +import path from "node:path"; +import { agent } from "@crosscode/contracts"; +import { verifyJob } from "@crosscode/contracts/signing"; +import type { AgentConfig } from "./config"; +import { paths } from "./paths"; + +/** + * Whether to run a job (AGENT-1.0-PLAN §4). Checked in this order, and a job is refused at the + * first check it fails: + * 1. it's an object with an id and type the agent can report on, + * 2. the signature verifies with the Hub key pinned when the server joined, + * 3. it's for this server, + * 4. it's within notBefore … expiresAt (an expired job is reported as `expired`, never run late), + * 5. its id hasn't been seen in the last 90 days (replays), + * 6. its type is one this agent runs (a central agent only runs check-ports, status, update-agent), + * 7. every argument passes that type's schema (unknown arguments refused). + */ + +export type Verdict = + | { ok: true; job: agent.AgentJob } + | { ok: false; status: "refused" | "expired"; message: string; id: string | null; type: agent.JobType | null }; + +const SEEN_DAYS = 90; + +/** Job ids already run or refused, with when (replay protection). */ +export class SeenJobs { + private seen = new Map(); + constructor(private file = path.join(paths.stateDir, "seen-jobs.json")) { + try { + const data = JSON.parse(fs.readFileSync(this.file, "utf8")) as Record; + for (const [id, at] of Object.entries(data)) this.seen.set(id, at); + } catch { + // First run, or an unreadable file: start empty (signatures and expiry still apply). + } + } + has(id: string): boolean { + return this.seen.has(id); + } + add(id: string, now = Date.now()): void { + this.seen.set(id, now); + const cutoff = now - SEEN_DAYS * 86_400_000; + for (const [k, at] of this.seen) if (at < cutoff) this.seen.delete(k); + fs.mkdirSync(path.dirname(this.file), { recursive: true, mode: 0o700 }); + const tmp = `${this.file}.tmp-${process.pid}`; + fs.writeFileSync(tmp, JSON.stringify(Object.fromEntries(this.seen)), { mode: 0o600 }); + fs.renameSync(tmp, this.file); + } +} + +const typeOf = (v: unknown): agent.JobType | null => (agent.jobTypeSchema.safeParse(v).success ? (v as agent.JobType) : null); + +export function checkJob(raw: unknown, config: AgentConfig, seen: SeenJobs, now = new Date()): Verdict { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return { ok: false, status: "refused", message: "Not a job.", id: null, type: null }; + const j = raw as Record; + const id = typeof j.id === "string" && /^job_[A-Za-z0-9_-]{8,64}$/.test(j.id) ? j.id : null; + const type = typeOf(j.type); + const refuse = (message: string): Verdict => ({ ok: false, status: "refused", message, id, type }); + + if (!id) return refuse("The job has no valid id."); + if (!verifyJob(j, config.hubJobKey)) return refuse("The signature doesn't match the Hub's key."); + if (j.serverId !== config.serverId) return refuse(`The job is for ${String(j.serverId)}, not this server.`); + const notBefore = Date.parse(String(j.notBefore)); + const expiresAt = Date.parse(String(j.expiresAt)); + if (!Number.isFinite(notBefore) || !Number.isFinite(expiresAt) || expiresAt <= notBefore) return refuse("The job's times aren't valid."); + if (now.getTime() > expiresAt) return { ok: false, status: "expired", message: `The job expired at ${new Date(expiresAt).toISOString()}; it wasn't run.`, id, type }; + // A minute's leeway for clocks that differ a little. + if (now.getTime() < notBefore - 60_000) return refuse(`The job isn't due until ${new Date(notBefore).toISOString()}.`); + if (seen.has(id)) return refuse("This job was already received (a replay)."); + if (!type) return refuse(`Unknown job type ${JSON.stringify(j.type)}.`); + if (config.mode === "central" && !agent.CENTRAL_JOB_TYPES.includes(type)) return refuse(`A central agent doesn't run ${type} jobs.`); + const parsed = agent.agentJobSchema.safeParse(raw); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + return refuse(`The job's ${issue?.path.join(".") || "content"} isn't valid: ${issue?.message ?? "unknown problem"}.`); + } + return { ok: true, job: parsed.data }; +} diff --git a/apps/agent/tests/command.test.ts b/apps/agent/tests/command.test.ts new file mode 100644 index 0000000..e6ea869 --- /dev/null +++ b/apps/agent/tests/command.test.ts @@ -0,0 +1,36 @@ +import { agent } from "@crosscode/contracts"; +import { crosscodeArgs } from "../src/command"; +import { job } from "./helpers"; + +const parsed = (over: Record) => agent.agentJobSchema.parse(job(over)); + +describe("crosscode arguments for each job", () => { + it("install: behind frp with the Hub's ports and the release", () => { + expect(crosscodeArgs(parsed({ + type: "install", + args: { church: "hope", products: ["events"], versions: { events: "events@2.4.1" }, domains: { events: "events.hope.co.za" }, ports: { events: { web: 20010, api: 20011 } } }, + }))).toEqual(["install", "hope", "--products", "events", "--domain", "hope.co.za", "--events-domain", "events.hope.co.za", "--api-port", "20011", "--web-port", "20010", "--frp", "--version", "events@2.4.1"]); + }); + + it("deploy, adopt, restart, uninstall, purge, status", () => { + expect(crosscodeArgs(parsed({}))).toEqual(["deploy", "hope", "events", "--version", "events@2.4.1"]); + expect(crosscodeArgs(parsed({ args: { church: "hope", products: ["events"], check: true } }))).toEqual(["deploy", "hope", "events", "--check"]); + expect(crosscodeArgs(parsed({ type: "adopt", args: { church: "hope", eventsDir: "/var/www/hope-events", version: "events@2.2.0" } }))) + .toEqual(["adopt", "hope", "--events-dir", "/var/www/hope-events", "--version", "events@2.2.0"]); + expect(crosscodeArgs(parsed({ type: "adopt-undo", args: { church: "hope" } }))).toEqual(["adopt", "hope", "--undo"]); + expect(crosscodeArgs(parsed({ type: "restart", args: { church: "hope" } }))).toEqual(["restart", "hope"]); + expect(crosscodeArgs(parsed({ type: "uninstall", args: { church: "hope" } }))).toEqual(["uninstall", "hope", "--confirm", "hope"]); + expect(crosscodeArgs(parsed({ type: "purge", args: { church: "hope" } }))).toEqual(["purge", "hope", "--confirm", "hope"]); + expect(crosscodeArgs(parsed({ type: "status", args: {} }))).toEqual(["status", "--no-fetch"]); + }); + + it("export writes to the file the agent gives", () => { + const j = parsed({ type: "export", args: { church: "hope", exportId: "exp_abcdefgh" } }); + expect(crosscodeArgs(j, { exportFile: "/var/lib/crosscode-agent/exports/exp_abcdefgh.zip" })).toEqual(["export", "hope", "--out", "/var/lib/crosscode-agent/exports/exp_abcdefgh.zip"]); + }); + + it("the agent's own jobs run no crosscode command", () => { + expect(crosscodeArgs(parsed({ type: "check-ports", args: { ports: [20010] } }))).toBeNull(); + expect(crosscodeArgs(parsed({ type: "update-agent", args: { version: "agent@1.0.1" } }))).toBeNull(); + }); +}); diff --git a/apps/agent/tests/helpers.ts b/apps/agent/tests/helpers.ts new file mode 100644 index 0000000..af97c30 --- /dev/null +++ b/apps/agent/tests/helpers.ts @@ -0,0 +1,44 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { generateEd25519, generateX25519, signJob } from "@crosscode/contracts/signing"; +import type { AgentConfig } from "../src/config"; + +export const hubKeys = generateEd25519(); +export const serverKeys = { ed25519: generateEd25519(), x25519: generateX25519() }; + +export const config: AgentConfig = { + hubUrl: "https://hub.test", + serverId: "srv_test00001", + mode: "church", + hubJobKey: hubKeys.publicKey, + pollSeconds: 30, + statsSeconds: 60, + joinedAt: "2026-11-20T00:00:00.000Z", +}; + +export const NOW = new Date("2026-11-20T12:05:00.000Z"); + +let n = 0; +export function job(over: Record = {}, key = hubKeys.privateKeyPem) { + n++; + return signJob({ + schema: "crosscode.agent-job/1", + id: `job_test${String(n).padStart(6, "0")}`, + serverId: config.serverId, + type: "deploy", + args: { church: "hope", products: ["events"], versions: ["events@2.4.1"] }, + config: null, + notBefore: "2026-11-20T12:00:00.000Z", + expiresAt: "2026-11-20T12:15:00.000Z", + ...over, + }, key); +} + +/** A temporary state folder per test file. */ +export function tempState(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-test-")); + process.env.CROSSCODE_AGENT_STATE_DIR = dir; + process.env.CROSSCODE_AGENT_RUN_DIR = path.join(dir, "run"); + return dir; +} diff --git a/apps/agent/tests/loop.test.ts b/apps/agent/tests/loop.test.ts new file mode 100644 index 0000000..92a8aef --- /dev/null +++ b/apps/agent/tests/loop.test.ts @@ -0,0 +1,112 @@ +import fs from "node:fs"; +import path from "node:path"; +import { agent } from "@crosscode/contracts"; +import { decryptExport } from "@crosscode/contracts/signing"; +import { HubRequestError, type HubClient } from "../src/hub"; +import { Agent } from "../src/loop"; +import type { RunResult } from "../src/runner"; +import { SeenJobs } from "../src/verify"; +import { config, job, NOW, serverKeys, tempState } from "./helpers"; + +const state = tempState(); + +function fakeHub(jobs: unknown[]) { + const posts: { path: string; body: unknown }[] = []; + const puts: { path: string; body: Uint8Array }[] = []; + let down = false; + const hub: HubClient = { + async post(p, body) { + if (down) throw new HubRequestError("down", 0, "HUB_UNREACHABLE"); + posts.push({ path: p, body }); + if (p === "/api/agent/v1/poll") return { jobs: jobs.splice(0), pollSeconds: 30 }; + return null; + }, + async put(p, body) { + puts.push({ path: p, body }); + return null; + }, + }; + return { hub, posts, puts, setDown: (v: boolean) => { down = v; } }; +} + +const ok = (over: Partial = {}): RunResult => ({ status: "succeeded", steps: [{ name: "x", startedAt: NOW.toISOString(), finishedAt: NOW.toISOString(), outcome: "ok" }], result: {}, log: ["==> done"], startedAt: NOW.toISOString(), finishedAt: NOW.toISOString(), ...over }); + +type Run = (args: string[], job: agent.AgentJob) => Promise; + +function make(jobs: unknown[], run: jest.Mock, Parameters> = jest.fn, Parameters>(async () => ok())) { + const h = fakeHub(jobs); + const a = new Agent({ config, keys: { ed25519Pem: serverKeys.ed25519.privateKeyPem, x25519Pem: serverKeys.x25519.privateKeyPem }, hub: h.hub, seen: new SeenJobs(path.join(state, `seen-${Math.random()}.json`)), run, now: () => NOW }); + return { a, run, ...h }; +} + +const reports = (posts: { path: string; body: unknown }[]) => posts.filter((p) => p.path === "/api/agent/v1/report").map((p) => p.body as agent.AgentReport); + +describe("the agent loop", () => { + it("runs a valid job once and reports it", async () => { + const j = job(); + const { a, run, posts } = make([j, j]); + await a.pollOnce(); + expect(run).toHaveBeenCalledTimes(1); + expect(run.mock.calls[0]).toEqual([["deploy", "hope", "events", "--version", "events@2.4.1"], expect.objectContaining({ id: j.id })]); + // The second copy is a replay: refused quietly (the first one's report already went). + const rs = reports(posts); + expect(rs.map((r) => [r.jobId, r.status])).toEqual([[j.id, "succeeded"]]); + expect(agent.agentReportSchema.safeParse(rs[0]).success).toBe(true); + }); + + it("reports refused and expired jobs without running them", async () => { + const forged = { ...job(), signature: "A".repeat(86) }; + const late = job({ expiresAt: "2026-11-20T12:01:00.000Z", notBefore: "2026-11-20T11:00:00.000Z" }); + const { a, run, posts } = make([forged, late]); + await a.pollOnce(); + expect(run).not.toHaveBeenCalled(); + expect(reports(posts).map((r) => r.status)).toEqual(["refused", "expired"]); + }); + + it("keeps a report it can't send and sends it on the next poll", async () => { + const j = job(); + let setDownLater: (v: boolean) => void = () => {}; + const { a, posts, setDown } = make([j], jest.fn, Parameters>(async () => { setDownLater(true); return ok(); })); + setDownLater = setDown; + await a.pollOnce(); + expect(reports(posts)).toEqual([]); + expect(fs.readdirSync(path.join(state, "outbox"))).toContain(`${j.id}.json`); + setDown(false); + await a.pollOnce(); + expect(reports(posts).map((r) => r.jobId)).toEqual([j.id]); + expect(fs.readdirSync(path.join(state, "outbox"))).toEqual([]); + }); + + it("backs off while the Hub is unreachable", async () => { + const { a, setDown } = make([]); + setDown(true); + expect(await a.pollOnce()).toBe(60); + expect(await a.pollOnce()).toBe(120); + setDown(false); + expect(await a.pollOnce()).toBe(30); + }); + + it("export: encrypts the ZIP, uploads it, and only the report carries the key", async () => { + const j = job({ type: "export", args: { church: "hope", exportId: "exp_abcdefgh" } }); + const run = jest.fn, Parameters>(async (args: string[]) => { + fs.writeFileSync(args[3]!, "PK zip bytes"); + return ok({ result: { file: args[3], bytes: 12 } }); + }); + const { a, puts, posts } = make([j], run); + await a.pollOnce(); + expect(puts).toHaveLength(1); + expect(puts[0]!.path).toBe("/api/agent/v1/exports/exp_abcdefgh"); + const r = reports(posts)[0]!; + expect(r.status).toBe("succeeded"); + const key = r.result.key as string; + expect(decryptExport(puts[0]!.body, key).toString()).toBe("PK zip bytes"); + expect(Buffer.from(puts[0]!.body).includes(Buffer.from("PK zip"))).toBe(false); + expect(fs.readdirSync(path.join(state, "exports"))).toEqual([]); + }); + + it("a job that throws is reported as failed", async () => { + const { a, posts } = make([job()], jest.fn, Parameters>(async () => { throw new Error("boom"); })); + await a.pollOnce(); + expect(reports(posts)[0]).toMatchObject({ status: "failed", message: "boom" }); + }); +}); diff --git a/apps/agent/tests/no-listening-socket.test.ts b/apps/agent/tests/no-listening-socket.test.ts new file mode 100644 index 0000000..e2e9712 --- /dev/null +++ b/apps/agent/tests/no-listening-socket.test.ts @@ -0,0 +1,72 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import { config, serverKeys } from "./helpers"; + +/** + * AGENT-1.0-PLAN §2 rule 1: the agent listens on nothing. Starts the built agent (dist/cli.js) + * against a fake Hub served by this test, lets it poll, then compares the agent's open sockets + * with every listening TCP socket the kernel knows (/proc/net/tcp, tcp6). Linux only (the + * servers); run `npm run build -w apps/agent` first. + */ +const cli = path.join(__dirname, "..", "dist", "cli.js"); +const linux = process.platform === "linux" && fs.existsSync(cli); +const maybe = linux ? it : it.skip; + +function listeningInodes(): Set { + const out = new Set(); + for (const f of ["/proc/net/tcp", "/proc/net/tcp6"]) { + let text: string; + try { text = fs.readFileSync(f, "utf8"); } catch { continue; } + for (const line of text.split("\n").slice(1)) { + const cols = line.trim().split(/\s+/); + if (cols[3] === "0A") out.add(cols[9]!); // 0A = LISTEN + } + } + return out; +} + +function socketInodes(pid: number): Set { + const out = new Set(); + for (const fd of fs.readdirSync(`/proc/${pid}/fd`)) { + try { + const m = /^socket:\[(\d+)\]$/.exec(fs.readlinkSync(`/proc/${pid}/fd/${fd}`)); + if (m) out.add(m[1]!); + } catch { /* closed meanwhile */ } + } + return out; +} + +describe("the running agent", () => { + maybe("has no listening socket", async () => { + let polls = 0; + const hub = http.createServer((req, res) => { + if (req.url === "/api/agent/v1/poll") polls++; + res.setHeader("content-type", "application/json"); + res.end(req.url === "/api/agent/v1/poll" ? JSON.stringify({ jobs: [], pollSeconds: 5 }) : "{}"); + }); + await new Promise((r) => hub.listen(0, "127.0.0.1", r)); + const port = (hub.address() as { port: number }).port; + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-nolisten-")); + fs.writeFileSync(path.join(dir, "agent.json"), JSON.stringify({ ...config, hubUrl: `http://127.0.0.1:${port}`, pollSeconds: 5, statsSeconds: 15 })); + fs.writeFileSync(path.join(dir, "agent-ed25519.pem"), serverKeys.ed25519.privateKeyPem); + fs.writeFileSync(path.join(dir, "agent-x25519.pem"), serverKeys.x25519.privateKeyPem); + const child = spawn(process.execPath, [cli, "run"], { + env: { ...process.env, CROSSCODE_AGENT_CONFIG_DIR: dir, CROSSCODE_AGENT_STATE_DIR: path.join(dir, "state"), CROSSCODE_AGENT_ALLOW_NON_ROOT: "1", CROSSCODE_SITES: path.join(dir, "sites") }, + stdio: "ignore", + }); + try { + for (let i = 0; i < 40 && polls === 0; i++) await new Promise((r) => setTimeout(r, 250)); + expect(polls).toBeGreaterThan(0); + const listening = listeningInodes(); + const mine = [...socketInodes(child.pid!)].filter((inode) => listening.has(inode)); + expect(mine).toEqual([]); + } finally { + child.kill("SIGTERM"); + hub.close(); + } + }, 30_000); +}); diff --git a/apps/agent/tests/redact-central.test.ts b/apps/agent/tests/redact-central.test.ts new file mode 100644 index 0000000..a601f27 --- /dev/null +++ b/apps/agent/tests/redact-central.test.ts @@ -0,0 +1,79 @@ +import { checkPorts, parseFrpsIni, parseSs, portAllowed } from "../src/central"; +import { LogTail, redact } from "../src/redact"; +import { churchProcesses, cpuPercent, parseMeminfo } from "../src/stats"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +describe("log lines are redacted before they leave the server", () => { + it.each([ + ["Error: connect postgresql://hope_events:Zx81kLmQ@localhost:5432/hope_events", "postgresql://hope_events:***@localhost"], + ["JWT_SECRET=abcdefghijklmnop", "JWT_SECRET=***"], + ['DB_PASSWORD: "hunter2 x"', "DB_PASSWORD: ***"], + ["Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "Authorization: Bearer ***"], + ["licence CEH-7K2M-9QXA-P4RT-WD8C refused", "CEH-****"], + ["token a3f9c1d27b8e4f60a3f9c1d27b8e4f60a3f9c1d2", "token ***"], + ])("%s", (line, expected) => { + const out = redact(line); + expect(out).toContain(expected); + }); + + it("keeps ordinary lines", () => { + expect(redact("==> events: migrations")).toBe("==> events: migrations"); + }); + + it("keeps only the last lines", () => { + const t = new LogTail(3); + t.push("1\n2\n3\n4"); + t.push("\n5\n"); + expect(t.end()).toEqual(["3", "4", "5"]); + }); +}); + +describe("central server checks", () => { + const ini = "[common]\nbind_port = 7000\nproxy_bind_addr = 127.0.0.1\nallow_ports = 20000-29999,3000\n\n[web]\nproxy_bind_addr = 0.0.0.0\n"; + + it("reads [common] only", () => { + expect(parseFrpsIni(ini)).toEqual({ proxyBindAddr: "127.0.0.1", allowPorts: "20000-29999,3000" }); + }); + + it("allow_ports ranges", () => { + expect(portAllowed(20010, "20000-29999,3000")).toBe(true); + expect(portAllowed(3000, "20000-29999,3000")).toBe(true); + expect(portAllowed(5000, "20000-29999,3000")).toBe(false); + expect(portAllowed(5000, null)).toBe(true); + }); + + it("ports must be allowed, open and bound to 127.0.0.1 only", () => { + const ss = "LISTEN 0 4096 127.0.0.1:20010 0.0.0.0:*\nLISTEN 0 4096 0.0.0.0:20011 0.0.0.0:*\nLISTEN 0 4096 [::1]:20012 [::]:*\n"; + const res = checkPorts([20010, 20011, 20012, 20013], parseFrpsIni(ini), parseSs(ss)); + expect(res.ports.map((p) => [p.port, p.ok])).toEqual([[20010, true], [20011, false], [20012, true], [20013, false]]); + expect(res.warnings.join(" ")).toMatch(/20011 is open on more than 127.0.0.1/); + expect(res.warnings.join(" ")).toMatch(/20013 isn't open/); + }); + + it("warns when proxy_bind_addr is missing", () => { + const res = checkPorts([20010], parseFrpsIni("[common]\nbind_port = 7000\n"), parseSs("LISTEN 0 1 127.0.0.1:20010 0.0.0.0:*")); + expect(res.warnings.join(" ")).toMatch(/proxy_bind_addr is not set/); + }); +}); + +describe("stats", () => { + it("RAM from /proc/meminfo", () => { + const r = parseMeminfo("MemTotal: 4000000 kB\nMemFree: 100 kB\nMemAvailable: 1000000 kB\nSwapTotal: 2000 kB\nSwapFree: 500 kB\n"); + expect(r).toEqual({ totalBytes: 4096000000, usedBytes: 3072000000, availableBytes: 1024000000, swapTotalBytes: 2048000, swapUsedBytes: 1536000 }); + }); + + it("CPU % between two readings", () => { + expect(cpuPercent("cpu 100 0 100 700 100 0 0 0", "cpu 150 0 150 750 150 0 0 0")).toBe(50); + }); + + it("PM2 processes are matched to churches through church.env", () => { + const sites = fs.mkdtempSync(path.join(os.tmpdir(), "sites-")); + fs.mkdirSync(path.join(sites, "hope")); + fs.writeFileSync(path.join(sites, "hope", "church.env"), 'CHURCH=hope\nEVENTS_USER=cc-hope\nEVENTS_PM2="hope-events-backend hope-events-frontend"\n'); + fs.mkdirSync(path.join(sites, "Bad Name")); + const map = churchProcesses(sites); + expect([...map.entries()]).toEqual([["cc-hope", [{ church: "hope", product: "events", names: ["hope-events-backend", "hope-events-frontend"] }]]]); + }); +}); diff --git a/apps/agent/tests/verify.test.ts b/apps/agent/tests/verify.test.ts new file mode 100644 index 0000000..5575b05 --- /dev/null +++ b/apps/agent/tests/verify.test.ts @@ -0,0 +1,61 @@ +import path from "node:path"; +import { generateEd25519 } from "@crosscode/contracts/signing"; +import { checkJob, SeenJobs } from "../src/verify"; +import { config, job, NOW, tempState } from "./helpers"; + +const dir = tempState(); +const fresh = () => new SeenJobs(path.join(dir, `seen-${Math.random()}.json`)); + +describe("checking a job (AGENT-1.0-PLAN §4)", () => { + it("accepts a signed job for this server, in its window", () => { + expect(checkJob(job(), config, fresh(), NOW)).toMatchObject({ ok: true }); + }); + + it("refuses a job signed with another key", () => { + expect(checkJob(job({}, generateEd25519().privateKeyPem), config, fresh(), NOW)).toMatchObject({ ok: false, status: "refused", message: expect.stringContaining("signature") }); + }); + + it("refuses a job whose content was changed after signing", () => { + const j = job(); + expect(checkJob({ ...j, args: { ...j.args, church: "other" } }, config, fresh(), NOW)).toMatchObject({ ok: false, status: "refused" }); + }); + + it("refuses a job for another server", () => { + expect(checkJob(job({ serverId: "srv_other0001" }), config, fresh(), NOW)).toMatchObject({ ok: false, message: expect.stringContaining("not this server") }); + }); + + it("reports an expired job as expired, and never runs it", () => { + expect(checkJob(job(), config, fresh(), new Date("2026-11-20T12:16:00.000Z"))).toMatchObject({ ok: false, status: "expired" }); + }); + + it("refuses a job that isn't due yet (beyond a minute of clock difference)", () => { + expect(checkJob(job(), config, fresh(), new Date("2026-11-20T11:58:00.000Z"))).toMatchObject({ ok: false, status: "refused" }); + expect(checkJob(job(), config, fresh(), new Date("2026-11-20T11:59:30.000Z"))).toMatchObject({ ok: true }); + }); + + it("refuses a replayed job, also after a restart", () => { + const file = path.join(dir, "seen-replay.json"); + const seen = new SeenJobs(file); + const j = job(); + expect(checkJob(j, config, seen, NOW)).toMatchObject({ ok: true }); + seen.add(j.id); + expect(checkJob(j, config, seen, NOW)).toMatchObject({ ok: false, message: expect.stringContaining("replay") }); + expect(checkJob(j, config, new SeenJobs(file), NOW)).toMatchObject({ ok: false, message: expect.stringContaining("replay") }); + }); + + it("refuses an unknown type, and arguments with shell characters", () => { + expect(checkJob(job({ type: "shell", args: { command: "id" } }), config, fresh(), NOW)).toMatchObject({ ok: false, message: expect.stringContaining("Unknown job type") }); + expect(checkJob(job({ args: { church: "hope;reboot", products: ["events"] } }), config, fresh(), NOW)).toMatchObject({ ok: false, message: expect.stringContaining("isn't valid") }); + expect(checkJob(job({ type: "adopt", args: { church: "hope", eventsDir: "/var/www/$(id)" } }), config, fresh(), NOW)).toMatchObject({ ok: false }); + }); + + it("a central agent only runs check-ports, status and update-agent", () => { + const central = { ...config, mode: "central" as const }; + expect(checkJob(job(), central, fresh(), NOW)).toMatchObject({ ok: false, message: expect.stringContaining("central") }); + expect(checkJob(job({ type: "check-ports", args: { ports: [20010, 20011] } }), central, fresh(), NOW)).toMatchObject({ ok: true }); + }); + + it("refuses things that aren't jobs", () => { + for (const raw of [null, "job", [], { id: "nope" }]) expect(checkJob(raw, config, fresh(), NOW)).toMatchObject({ ok: false, status: "refused" }); + }); +}); diff --git a/apps/agent/tsconfig.json b/apps/agent/tsconfig.json new file mode 100644 index 0000000..6d7bf4f --- /dev/null +++ b/apps/agent/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "types": ["node", "jest"], + "declaration": false + }, + "include": ["src", "tests", "tsup.config.ts"] +} diff --git a/apps/agent/tsup.config.ts b/apps/agent/tsup.config.ts new file mode 100644 index 0000000..2d627a1 --- /dev/null +++ b/apps/agent/tsup.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "tsup"; + +// One self-contained file (contracts and zod bundled in), so a server needs no node_modules to run +// the agent: /opt/crosscode/agent/current/apps/agent/dist/cli.js. +export default defineConfig({ + entry: ["src/cli.ts"], + format: ["cjs"], + platform: "node", + target: "node24", + noExternal: [/.*/], + clean: true, + sourcemap: true, + banner: { js: "#!/usr/bin/env node" }, +}); diff --git a/package-lock.json b/package-lock.json index bbd6ba7..3252bbb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,21 @@ "name": "@crosscode/accounts", "version": "0.0.0" }, + "apps/agent": { + "name": "@crosscode/agent", + "version": "1.0.0", + "license": "UNLICENSED", + "dependencies": { + "@crosscode/contracts": "*", + "zod": "^4.6.7" + }, + "bin": { + "crosscode-agent": "dist/cli.js" + }, + "engines": { + "node": ">=24" + } + }, "apps/app": { "name": "@crosscode/app", "version": "0.0.0" @@ -876,6 +891,10 @@ "resolved": "apps/accounts", "link": true }, + "node_modules/@crosscode/agent": { + "resolved": "apps/agent", + "link": true + }, "node_modules/@crosscode/app": { "resolved": "apps/app", "link": true @@ -19633,7 +19652,7 @@ }, "packages/contracts": { "name": "@crosscode/contracts", - "version": "1.2.0", + "version": "1.3.0", "license": "UNLICENSED", "dependencies": { "zod": "^4.6.7" diff --git a/package.json b/package.json index 58e38a6..8061ce8 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "test:scripts": "bats scripts/tests", "typecheck": "npm run typecheck --workspaces --if-present", "lint": "eslint .", - "lint:scripts": "shellcheck -x scripts/crosscode scripts/products/events.sh scripts/lib/report.sh scripts/lib/env.sh scripts/lib/frpc.sh scripts/lib/manage.sh", + "lint:scripts": "shellcheck -x scripts/crosscode scripts/products/events.sh scripts/lib/report.sh scripts/lib/env.sh scripts/lib/frpc.sh scripts/lib/manage.sh apps/agent/deploy/install.sh apps/agent/deploy/revert.sh", "check": "npm run lint && npm run lint:scripts && npm run typecheck && npm test && npm run test:scripts && npm run build" }, "engines": {