Agent 1.0.0: polls the Hub for signed jobs, runs crosscode, sends reports and stats
apps/agent (docs/AGENT-1.0-PLAN.md §3-4, §10): listens on nothing; checks every job (pinned Hub key, server, time window, replays for 90 days, type per mode, strict arguments) before running crosscode with an argument array; opens the sealed config into a root-only file only for the job; redacts log lines; keeps unsent reports in an outbox; encrypts exports before upload; check-ports on the central server (read-only); self-update with automatic revert. One bundled dist/cli.js, a systemd unit and the install script that joins a server with a one-time token. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,98 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import { paths } from "./paths";
|
||||
|
||||
/**
|
||||
* The central server (AGENT-1.0-PLAN §6): read-only checks of frps for `check-ports`. Nothing on
|
||||
* the central server is ever changed: no frps edits, no firewall, no FastPanel.
|
||||
*/
|
||||
|
||||
export interface FrpsSettings {
|
||||
proxyBindAddr: string | null;
|
||||
allowPorts: string | null;
|
||||
}
|
||||
|
||||
/** frps.ini's [common] proxy_bind_addr and allow_ports (null when not set or no file). */
|
||||
export function parseFrpsIni(text: string): FrpsSettings {
|
||||
let inCommon = false;
|
||||
const out: FrpsSettings = { proxyBindAddr: null, allowPorts: null };
|
||||
for (const raw of text.split(/\r?\n/)) {
|
||||
const line = raw.trim();
|
||||
if (/^\[.*\]$/.test(line)) { inCommon = line === "[common]"; continue; }
|
||||
if (!inCommon || line.startsWith("#") || line.startsWith(";")) continue;
|
||||
const m = /^([a-z_]+)\s*=\s*(.*)$/.exec(line);
|
||||
if (!m) continue;
|
||||
if (m[1] === "proxy_bind_addr") out.proxyBindAddr = m[2]!.trim() || null;
|
||||
if (m[1] === "allow_ports") out.allowPorts = m[2]!.trim() || null;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function readFrps(file = paths.frpsIni): FrpsSettings | null {
|
||||
try {
|
||||
return parseFrpsIni(fs.readFileSync(file, "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a port is inside allow_ports ("2000-3000,3001,4000-5000"); no allow_ports allows all. */
|
||||
export function portAllowed(port: number, allowPorts: string | null): boolean {
|
||||
if (!allowPorts) return true;
|
||||
return allowPorts.split(",").map((s) => s.trim()).filter(Boolean).some((range) => {
|
||||
const [a, b] = range.split("-").map((n) => Number(n.trim()));
|
||||
if (!Number.isFinite(a)) return false;
|
||||
return b === undefined ? port === a : Number.isFinite(b) && port >= a! && port <= b;
|
||||
});
|
||||
}
|
||||
|
||||
/** Listening TCP sockets from `ss -ltnH`: port → local addresses. */
|
||||
export function parseSs(text: string): Map<number, string[]> {
|
||||
const out = new Map<number, string[]>();
|
||||
for (const line of text.split("\n")) {
|
||||
const cols = line.trim().split(/\s+/);
|
||||
const local = cols[3];
|
||||
if (!local) continue;
|
||||
const i = local.lastIndexOf(":");
|
||||
const port = Number(local.slice(i + 1));
|
||||
const addr = local.slice(0, i).replace(/^\[|\]$/g, "").replace(/%.*$/, "");
|
||||
if (!Number.isFinite(port)) continue;
|
||||
out.set(port, [...(out.get(port) ?? []), addr]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const LOOPBACK = new Set(["127.0.0.1", "::1", "::ffff:127.0.0.1"]);
|
||||
|
||||
export interface PortCheck {
|
||||
port: number;
|
||||
allowed: boolean;
|
||||
listening: boolean;
|
||||
loopbackOnly: boolean;
|
||||
ok: boolean;
|
||||
}
|
||||
|
||||
export function checkPorts(ports: number[], frps: FrpsSettings | null, listening: Map<number, string[]>): { ports: PortCheck[]; proxyBindAddr: string | null; warnings: string[] } {
|
||||
const warnings: string[] = [];
|
||||
if (!frps) warnings.push(`No frps.ini at ${paths.frpsIni}.`);
|
||||
const bind = frps?.proxyBindAddr ?? null;
|
||||
if (frps && (!bind || !LOOPBACK.has(bind))) warnings.push(`proxy_bind_addr is ${bind ? `"${bind}"` : "not set"}: tunnels can be reached straight from the internet. Set proxy_bind_addr = 127.0.0.1 in frps.ini.`);
|
||||
const results = ports.map((port) => {
|
||||
const addrs = listening.get(port) ?? [];
|
||||
const allowed = portAllowed(port, frps?.allowPorts ?? null);
|
||||
const loopbackOnly = addrs.length > 0 && addrs.every((a) => LOOPBACK.has(a));
|
||||
return { port, allowed, listening: addrs.length > 0, loopbackOnly, ok: allowed && addrs.length > 0 && loopbackOnly };
|
||||
});
|
||||
for (const r of results) {
|
||||
if (!r.allowed) warnings.push(`Port ${r.port} isn't in allow_ports.`);
|
||||
else if (!r.listening) warnings.push(`Port ${r.port} isn't open in frps (is the church server's frpc connected?).`);
|
||||
else if (!r.loopbackOnly) warnings.push(`Port ${r.port} is open on more than 127.0.0.1.`);
|
||||
}
|
||||
return { ports: results, proxyBindAddr: bind, warnings };
|
||||
}
|
||||
|
||||
export function listeningNow(): Promise<Map<number, string[]>> {
|
||||
return new Promise((resolve) => {
|
||||
execFile("ss", ["-ltnH"], { timeout: 10_000 }, (err, stdout) => resolve(err ? new Map() : parseSs(String(stdout))));
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import { agent } from "@crosscode/contracts";
|
||||
import { generateEd25519, generateX25519 } from "@crosscode/contracts/signing";
|
||||
import { configFile, ed25519File, loadConfig, saveConfig, writePrivate, x25519File } from "./config";
|
||||
import { hubClient, postUnsigned } from "./hub";
|
||||
import { Agent } from "./loop";
|
||||
import { AGENT_VERSION } from "./stats";
|
||||
|
||||
/**
|
||||
* crosscode-agent (AGENT-1.0-PLAN §3)
|
||||
* crosscode-agent join --hub https://hub.crosscode.co.za --token <one-time token> [--central]
|
||||
* crosscode-agent run what systemd runs (crosscode-agent.service)
|
||||
* crosscode-agent version
|
||||
*/
|
||||
|
||||
const USAGE = `crosscode-agent ${AGENT_VERSION}
|
||||
|
||||
crosscode-agent join --hub <url> --token <token> [--central] [--force]
|
||||
Join this server to the Hub with a one-time token (Hub → Servers → Add server).
|
||||
Makes the server's key pairs; the private keys never leave it.
|
||||
crosscode-agent run poll the Hub for jobs and send stats (run by systemd)
|
||||
crosscode-agent version
|
||||
`;
|
||||
|
||||
function flag(args: string[], name: string): string | undefined {
|
||||
const i = args.indexOf(name);
|
||||
return i >= 0 ? args[i + 1] : undefined;
|
||||
}
|
||||
|
||||
export async function join(args: string[]): Promise<void> {
|
||||
const hub = flag(args, "--hub");
|
||||
const token = flag(args, "--token");
|
||||
const mode: agent.AgentMode = args.includes("--central") ? "central" : "church";
|
||||
if (!hub || !/^https?:\/\//.test(hub)) throw new Error("--hub <url> is required (https://hub.crosscode.co.za).");
|
||||
if (!token) throw new Error("--token <token> is required (Hub → Servers → Add server).");
|
||||
if (fs.existsSync(configFile()) && !args.includes("--force")) {
|
||||
throw new Error(`This server has already joined (${configFile()}). Use --force to join again (the Hub then sees a new server).`);
|
||||
}
|
||||
const ed = generateEd25519();
|
||||
const x = generateX25519();
|
||||
const reply = agent.joinResponseSchema.parse(await postUnsigned(hub, "/api/agent/v1/join", {
|
||||
agentApiVersion: agent.AGENT_API_VERSION,
|
||||
token,
|
||||
mode,
|
||||
hostname: os.hostname().slice(0, 253),
|
||||
agentVersion: AGENT_VERSION,
|
||||
publicKeys: { ed25519: ed.publicKey, x25519: x.publicKey },
|
||||
}));
|
||||
writePrivate(ed25519File(), ed.privateKeyPem);
|
||||
writePrivate(x25519File(), x.privateKeyPem);
|
||||
saveConfig({ hubUrl: hub, serverId: reply.serverId, mode, hubJobKey: reply.hubJobKey, pollSeconds: reply.pollSeconds, statsSeconds: reply.statsSeconds, joinedAt: new Date().toISOString() });
|
||||
console.log(`Joined the Hub as ${reply.serverId} (${mode}). Start the agent: systemctl enable --now crosscode-agent`);
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)): Promise<void> {
|
||||
const [command, ...args] = argv;
|
||||
switch (command) {
|
||||
case "version":
|
||||
case "--version":
|
||||
console.log(AGENT_VERSION);
|
||||
return;
|
||||
case "join":
|
||||
await join(args);
|
||||
return;
|
||||
case "run": {
|
||||
if (process.getuid && process.getuid() !== 0 && !process.env.CROSSCODE_AGENT_ALLOW_NON_ROOT) {
|
||||
throw new Error("Run the agent as root (it runs crosscode, which needs root).");
|
||||
}
|
||||
const { config, keys } = loadConfig();
|
||||
const a = new Agent({ config, keys, hub: hubClient(config, keys) });
|
||||
for (const sig of ["SIGTERM", "SIGINT"] as const) process.on(sig, () => { a.stop(); setTimeout(() => process.exit(0), 1000).unref(); });
|
||||
await a.start();
|
||||
return;
|
||||
}
|
||||
default:
|
||||
process.stdout.write(USAGE);
|
||||
if (command && command !== "help" && command !== "--help") process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((e) => {
|
||||
console.error(`crosscode-agent: ${(e as Error).message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { agent } from "@crosscode/contracts";
|
||||
|
||||
/**
|
||||
* The `crosscode` arguments for a job (AGENT-1.0-PLAN §4): built from the checked arguments only,
|
||||
* as an array passed straight to the process (never through a shell). Jobs the agent handles
|
||||
* itself (update-agent, check-ports) have none.
|
||||
*
|
||||
* `--config <file>` is added by the runner when the job brings config; `--json` always.
|
||||
*/
|
||||
export function crosscodeArgs(job: agent.AgentJob, opts: { exportFile?: string } = {}): string[] | null {
|
||||
switch (job.type) {
|
||||
case "install": {
|
||||
const a = job.args;
|
||||
if (a.products.length !== 1 || a.products[0] !== "events") throw new Error("This agent can only install Events so far.");
|
||||
const domain = a.domains.events;
|
||||
const ports = a.ports.events;
|
||||
if (!domain || !ports) throw new Error("An Events install needs its domain and ports.");
|
||||
// The church's address: the Events domain without "events." (crosscode uses it for the
|
||||
// church.env ADDRESS; the domain itself goes in --events-domain).
|
||||
const address = domain.startsWith("events.") ? domain.slice("events.".length) : domain;
|
||||
const out = ["install", a.church, "--products", "events", "--domain", address, "--events-domain", domain,
|
||||
"--api-port", String(ports.api), "--web-port", String(ports.web), "--frp"];
|
||||
const v = a.versions?.events;
|
||||
if (v) out.push("--version", v);
|
||||
return out;
|
||||
}
|
||||
case "add-product":
|
||||
return ["add", job.args.church, job.args.product];
|
||||
case "deploy": {
|
||||
const out = ["deploy", job.args.church, ...job.args.products];
|
||||
for (const v of job.args.versions ?? []) out.push("--version", v);
|
||||
if (job.args.check) out.push("--check");
|
||||
return out;
|
||||
}
|
||||
case "adopt": {
|
||||
const out = ["adopt", job.args.church, "--events-dir", job.args.eventsDir];
|
||||
if (job.args.version) out.push("--version", job.args.version);
|
||||
if (job.args.address) out.push("--address", job.args.address);
|
||||
return out;
|
||||
}
|
||||
case "adopt-undo":
|
||||
return ["adopt", job.args.church, "--undo"];
|
||||
case "restart":
|
||||
return ["restart", job.args.church, ...(job.args.product ? [job.args.product] : [])];
|
||||
case "export":
|
||||
if (!opts.exportFile) throw new Error("export needs a file to write");
|
||||
return ["export", job.args.church, "--out", opts.exportFile];
|
||||
case "uninstall":
|
||||
return ["uninstall", job.args.church, "--confirm", job.args.church];
|
||||
case "purge":
|
||||
return ["purge", job.args.church, "--confirm", job.args.church];
|
||||
case "status":
|
||||
return ["status", ...(job.args.church ? [job.args.church] : []), "--no-fetch"];
|
||||
case "update-agent":
|
||||
case "check-ports":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { z } from "zod";
|
||||
import { agent } from "@crosscode/contracts";
|
||||
import { publicKeyOf } from "@crosscode/contracts/signing";
|
||||
import { paths } from "./paths";
|
||||
|
||||
/** /etc/crosscode/agent.json, written by `crosscode-agent join`. No secrets: the keys are files next to it. */
|
||||
export const agentConfigSchema = z.object({
|
||||
hubUrl: z.url({ protocol: /^https?$/ }),
|
||||
serverId: z.string().regex(/^srv_[A-Za-z0-9_-]{8,64}$/),
|
||||
mode: agent.agentModeSchema,
|
||||
/** The Hub's job-signing key, pinned when joining. */
|
||||
hubJobKey: z.string().regex(/^[A-Za-z0-9_-]{43}$/),
|
||||
pollSeconds: z.int().min(5).max(600).default(30),
|
||||
statsSeconds: z.int().min(15).max(3600).default(60),
|
||||
joinedAt: z.string(),
|
||||
});
|
||||
export type AgentConfig = z.infer<typeof agentConfigSchema>;
|
||||
|
||||
export interface Keys {
|
||||
ed25519Pem: string;
|
||||
x25519Pem: string;
|
||||
}
|
||||
|
||||
export const configFile = () => path.join(paths.configDir, "agent.json");
|
||||
export const ed25519File = () => path.join(paths.configDir, "agent-ed25519.pem");
|
||||
export const x25519File = () => path.join(paths.configDir, "agent-x25519.pem");
|
||||
|
||||
/** Writes a file only root can read, atomically. */
|
||||
export function writePrivate(file: string, text: string): void {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
||||
const tmp = `${file}.tmp-${process.pid}`;
|
||||
fs.writeFileSync(tmp, text, { mode: 0o600 });
|
||||
fs.renameSync(tmp, file);
|
||||
}
|
||||
|
||||
export function loadConfig(): { config: AgentConfig; keys: Keys } {
|
||||
const config = agentConfigSchema.parse(JSON.parse(fs.readFileSync(configFile(), "utf8")));
|
||||
const keys = { ed25519Pem: fs.readFileSync(ed25519File(), "utf8"), x25519Pem: fs.readFileSync(x25519File(), "utf8") };
|
||||
// A broken pair would sign requests the Hub can't check; better to stop at start.
|
||||
publicKeyOf(keys.ed25519Pem);
|
||||
publicKeyOf(keys.x25519Pem);
|
||||
return { config, keys };
|
||||
}
|
||||
|
||||
export function saveConfig(config: AgentConfig): void {
|
||||
writePrivate(configFile(), `${JSON.stringify(config, null, 2)}\n`);
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { signAgentRequest } from "@crosscode/contracts/signing";
|
||||
import type { AgentConfig, Keys } from "./config";
|
||||
|
||||
/**
|
||||
* Requests to the Hub. The agent only ever makes outgoing requests (it listens on nothing,
|
||||
* AGENT-1.0-PLAN §2 rule 1); every one after joining is signed with the server's Ed25519 key.
|
||||
*/
|
||||
|
||||
export class HubRequestError extends Error {
|
||||
constructor(message: string, readonly status: number, readonly code: string | null) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export type Fetch = typeof fetch;
|
||||
|
||||
export interface HubClient {
|
||||
post(path: string, body: unknown): Promise<unknown>;
|
||||
put(path: string, body: Uint8Array, contentType: string): Promise<unknown>;
|
||||
}
|
||||
|
||||
const TIMEOUT_MS = 30_000;
|
||||
const UPLOAD_TIMEOUT_MS = 10 * 60_000;
|
||||
|
||||
async function send(fetcher: Fetch, url: string, init: RequestInit, timeoutMs: number): Promise<unknown> {
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetcher(url, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
||||
} catch (e) {
|
||||
throw new HubRequestError(`The Hub can't be reached: ${(e as Error).message}`, 0, "HUB_UNREACHABLE");
|
||||
}
|
||||
if (res.status === 204) return null;
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await res.json();
|
||||
} catch {
|
||||
body = null;
|
||||
}
|
||||
if (!res.ok) {
|
||||
const b = (body ?? {}) as { message?: string; code?: string };
|
||||
throw new HubRequestError(b.message || `The Hub answered ${res.status}`, res.status, b.code ?? null);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
/** An unsigned request (only `join`, where the one-time token is the proof). */
|
||||
export function postUnsigned(hubUrl: string, path: string, body: unknown, fetcher: Fetch = fetch): Promise<unknown> {
|
||||
return send(fetcher, `${hubUrl.replace(/\/$/, "")}${path}`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", "user-agent": "crosscode-agent" },
|
||||
body: JSON.stringify(body),
|
||||
}, TIMEOUT_MS);
|
||||
}
|
||||
|
||||
export function hubClient(config: AgentConfig, keys: Keys, fetcher: Fetch = fetch): HubClient {
|
||||
const base = config.hubUrl.replace(/\/$/, "");
|
||||
const signed = (raw: string | Uint8Array) => signAgentRequest({ serverId: config.serverId, privateKeyPem: keys.ed25519Pem, rawBody: raw });
|
||||
return {
|
||||
post(path, body) {
|
||||
const raw = JSON.stringify(body);
|
||||
return send(fetcher, `${base}${path}`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", "user-agent": "crosscode-agent", ...signed(raw) },
|
||||
body: raw,
|
||||
}, TIMEOUT_MS);
|
||||
},
|
||||
put(path, body, contentType) {
|
||||
return send(fetcher, `${base}${path}`, {
|
||||
method: "PUT",
|
||||
headers: { "content-type": contentType, "user-agent": "crosscode-agent", ...signed(body) },
|
||||
body,
|
||||
}, UPLOAD_TIMEOUT_MS);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { agent } from "@crosscode/contracts";
|
||||
import { encryptExport } from "@crosscode/contracts/signing";
|
||||
import { checkPorts, listeningNow, readFrps } from "./central";
|
||||
import { crosscodeArgs } from "./command";
|
||||
import type { AgentConfig, Keys } from "./config";
|
||||
import { type HubClient, HubRequestError } from "./hub";
|
||||
import { paths } from "./paths";
|
||||
import { runCrosscode, type RunResult } from "./runner";
|
||||
import { AGENT_VERSION, collectStats } from "./stats";
|
||||
import { confirmUpdate, updateAgent } from "./update";
|
||||
import { checkJob, SeenJobs } from "./verify";
|
||||
|
||||
/**
|
||||
* The agent's life: poll the Hub every 30 s for jobs due, send stats every 60 s, run one job at a
|
||||
* time and send each result. Reports that can't be sent are kept in an outbox and sent later, so
|
||||
* a Hub outage never loses a job's result. Nothing here listens: every request goes out.
|
||||
*/
|
||||
|
||||
const log = (...a: unknown[]) => console.log(new Date().toISOString(), ...a);
|
||||
|
||||
export interface AgentDeps {
|
||||
config: AgentConfig;
|
||||
keys: Keys;
|
||||
hub: HubClient;
|
||||
seen?: SeenJobs;
|
||||
/** For tests: replaces running crosscode and the stats. */
|
||||
run?: (args: string[], job: agent.AgentJob) => Promise<RunResult>;
|
||||
stats?: () => Promise<agent.AgentStats>;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
const outboxDir = () => path.join(paths.stateDir, "outbox");
|
||||
|
||||
function report(job: { id: string; type: agent.JobType }, serverId: string, r: Omit<RunResult, "log"> & { log?: string[] }): agent.AgentReport {
|
||||
return {
|
||||
schema: "crosscode.agent-report/1",
|
||||
jobId: job.id,
|
||||
serverId,
|
||||
type: job.type,
|
||||
status: r.status,
|
||||
startedAt: r.startedAt,
|
||||
finishedAt: r.finishedAt,
|
||||
steps: r.steps.slice(0, 200),
|
||||
result: r.result,
|
||||
...(r.message ? { message: r.message.slice(0, 2000) } : {}),
|
||||
log: (r.log ?? []).slice(-200),
|
||||
};
|
||||
}
|
||||
|
||||
export class Agent {
|
||||
private seen: SeenJobs;
|
||||
private stopped = false;
|
||||
private confirmed = false;
|
||||
private failures = 0;
|
||||
|
||||
constructor(private deps: AgentDeps) {
|
||||
this.seen = deps.seen ?? new SeenJobs();
|
||||
}
|
||||
|
||||
private now(): Date {
|
||||
return this.deps.now?.() ?? new Date();
|
||||
}
|
||||
|
||||
/** Sends a report, or keeps it in the outbox to send later. */
|
||||
async send(r: agent.AgentReport): Promise<void> {
|
||||
try {
|
||||
await this.deps.hub.post("/api/agent/v1/report", r);
|
||||
} catch (e) {
|
||||
log(`[report] ${r.jobId}: couldn't send (${(e as Error).message}); kept to send later.`);
|
||||
fs.mkdirSync(outboxDir(), { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(path.join(outboxDir(), `${r.jobId}.json`), JSON.stringify(r), { mode: 0o600 });
|
||||
}
|
||||
}
|
||||
|
||||
async flushOutbox(): Promise<void> {
|
||||
let files: string[];
|
||||
try { files = fs.readdirSync(outboxDir()).filter((f) => f.endsWith(".json")); } catch { return; }
|
||||
for (const f of files) {
|
||||
const file = path.join(outboxDir(), f);
|
||||
try {
|
||||
await this.deps.hub.post("/api/agent/v1/report", JSON.parse(fs.readFileSync(file, "utf8")));
|
||||
fs.rmSync(file, { force: true });
|
||||
} catch (e) {
|
||||
// A report the Hub refuses outright (4xx) would never go: drop it rather than retry forever.
|
||||
if (e instanceof HubRequestError && e.status >= 400 && e.status < 500) fs.rmSync(file, { force: true });
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** One poll: fetch the jobs due and run them, one after the other. Returns the next delay (s). */
|
||||
async pollOnce(): Promise<number> {
|
||||
const { config } = this.deps;
|
||||
let reply: { jobs: unknown[]; pollSeconds: number };
|
||||
try {
|
||||
reply = agent.pollResponseSchema.parse(await this.deps.hub.post("/api/agent/v1/poll", {
|
||||
agentApiVersion: agent.AGENT_API_VERSION, agentVersion: AGENT_VERSION, mode: config.mode, runningJobId: null,
|
||||
}));
|
||||
} catch (e) {
|
||||
this.failures++;
|
||||
const delay = Math.min(300, config.pollSeconds * 2 ** Math.min(this.failures, 4));
|
||||
log(`[poll] ${(e as Error).message}; next try in ${delay}s.`);
|
||||
return delay;
|
||||
}
|
||||
this.failures = 0;
|
||||
if (!this.confirmed) {
|
||||
confirmUpdate();
|
||||
this.confirmed = true;
|
||||
}
|
||||
await this.flushOutbox();
|
||||
for (const raw of reply.jobs) {
|
||||
if (this.stopped) break;
|
||||
await this.handle(raw);
|
||||
}
|
||||
return reply.pollSeconds || config.pollSeconds;
|
||||
}
|
||||
|
||||
async handle(raw: unknown): Promise<void> {
|
||||
const { config } = this.deps;
|
||||
const verdict = checkJob(raw, config, this.seen, this.now());
|
||||
const at = this.now().toISOString();
|
||||
if (!verdict.ok) {
|
||||
log(`[job] ${verdict.id ?? "(no id)"} ${verdict.status}: ${verdict.message}`);
|
||||
if (verdict.id && verdict.type && !this.seen.has(verdict.id)) {
|
||||
this.seen.add(verdict.id);
|
||||
await this.send(report({ id: verdict.id, type: verdict.type }, config.serverId, { status: verdict.status, steps: [], result: {}, message: verdict.message, startedAt: at, finishedAt: at }));
|
||||
}
|
||||
return;
|
||||
}
|
||||
const job = verdict.job;
|
||||
// Seen before it runs, so a crash part-way can never run it twice.
|
||||
this.seen.add(job.id);
|
||||
log(`[job] ${job.id} ${job.type} ${JSON.stringify(job.args)}`);
|
||||
const result = await this.execute(job);
|
||||
log(`[job] ${job.id} ${result.status}${result.message ? `: ${result.message}` : ""}`);
|
||||
await this.send(report(job, config.serverId, result));
|
||||
if ((result as RunResult & { restart?: boolean }).restart) {
|
||||
log("[agent] Exiting so systemd starts the new version.");
|
||||
this.stopped = true;
|
||||
setTimeout(() => process.exit(0), 500);
|
||||
}
|
||||
}
|
||||
|
||||
private async execute(job: agent.AgentJob): Promise<RunResult & { restart?: boolean }> {
|
||||
const startedAt = this.now().toISOString();
|
||||
const limit = agent.JOB_TIME_LIMIT_MINUTES[job.type];
|
||||
try {
|
||||
switch (job.type) {
|
||||
case "update-agent": {
|
||||
const u = await updateAgent(job.args.version, AGENT_VERSION);
|
||||
return { status: u.status, steps: u.steps, result: u.result, message: u.message, log: [], startedAt, finishedAt: new Date().toISOString(), restart: u.restart };
|
||||
}
|
||||
case "check-ports": {
|
||||
const res = checkPorts(job.args.ports, readFrps(), await listeningNow());
|
||||
const ok = res.ports.every((p) => p.ok) && res.warnings.length === 0;
|
||||
return { status: ok ? "succeeded" : "failed", steps: [], result: res as unknown as Record<string, unknown>, message: res.warnings.join(" ") || undefined, log: [], startedAt, finishedAt: new Date().toISOString() };
|
||||
}
|
||||
case "export":
|
||||
return await this.runExport(job, limit);
|
||||
default: {
|
||||
const args = crosscodeArgs(job);
|
||||
if (!args) throw new Error(`No command for ${job.type}`);
|
||||
return await this.run(args, job, limit);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
return { status: "failed", steps: [], result: {}, message: (e as Error).message, log: [], startedAt, finishedAt: new Date().toISOString() };
|
||||
}
|
||||
}
|
||||
|
||||
private run(args: string[], job: agent.AgentJob, limit: number): Promise<RunResult> {
|
||||
if (this.deps.run) return this.deps.run(args, job);
|
||||
return runCrosscode({ args, sealedConfig: job.config ?? null, x25519Pem: this.deps.keys.x25519Pem, timeLimitMinutes: limit });
|
||||
}
|
||||
|
||||
/**
|
||||
* export: crosscode writes the ZIP, the agent encrypts it with a fresh key and uploads it. The
|
||||
* key goes to the Hub only in this job's report, so it can email the download link; the Hub
|
||||
* doesn't keep it (AGENT-1.0-PLAN §9).
|
||||
*/
|
||||
private async runExport(job: Extract<agent.AgentJob, { type: "export" }>, limit: number): Promise<RunResult> {
|
||||
const dir = path.join(paths.stateDir, "exports");
|
||||
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
const zip = path.join(dir, `${job.args.exportId}.zip`);
|
||||
fs.rmSync(zip, { force: true });
|
||||
try {
|
||||
const r = await this.run(crosscodeArgs(job, { exportFile: zip })!, job, limit);
|
||||
if (r.status !== "succeeded") return r;
|
||||
const { key, file } = encryptExport(fs.readFileSync(zip));
|
||||
await this.deps.hub.put(`/api/agent/v1/exports/${job.args.exportId}`, file, "application/octet-stream");
|
||||
return { ...r, result: { ...r.result, file: undefined, encryptedBytes: file.length, key } };
|
||||
} finally {
|
||||
fs.rmSync(zip, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async statsOnce(): Promise<void> {
|
||||
try {
|
||||
const stats = await (this.deps.stats ?? (() => collectStats(this.deps.config.mode)))();
|
||||
await this.deps.hub.post("/api/agent/v1/stats", stats);
|
||||
} catch (e) {
|
||||
log(`[stats] ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs until stopped: polls and stats on their own timers, never at the same time as a job. */
|
||||
async start(): Promise<void> {
|
||||
log(`[agent] ${AGENT_VERSION} (${this.deps.config.mode}) for ${this.deps.config.serverId}, Hub ${this.deps.config.hubUrl}`);
|
||||
let nextStats = 0;
|
||||
while (!this.stopped) {
|
||||
if (Date.now() >= nextStats) {
|
||||
await this.statsOnce();
|
||||
nextStats = Date.now() + this.deps.config.statsSeconds * 1000;
|
||||
}
|
||||
const delay = await this.pollOnce();
|
||||
await new Promise((r) => setTimeout(r, Math.max(5, Math.min(delay, this.deps.config.statsSeconds)) * 1000));
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* Where the agent keeps things on a server (docs/AGENT-1.0-PLAN.md §3). Each can be moved with an
|
||||
* environment variable, which the tests use.
|
||||
*/
|
||||
const env = (name: string, fallback: string): string => process.env[name] || fallback;
|
||||
|
||||
export const paths = {
|
||||
/** agent.json and the two private keys, root only (mode 600). */
|
||||
get configDir() { return env("CROSSCODE_AGENT_CONFIG_DIR", "/etc/crosscode"); },
|
||||
/** Seen job ids (replay protection) and exports waiting to upload. */
|
||||
get stateDir() { return env("CROSSCODE_AGENT_STATE_DIR", "/var/lib/crosscode-agent"); },
|
||||
/** A job's opened config, only while the job runs (tmpfs). */
|
||||
get runDir() { return env("CROSSCODE_AGENT_RUN_DIR", "/run/crosscode-agent"); },
|
||||
/** The crosscode script (a checkout of the monorepo in /opt/crosscode/tools). */
|
||||
get crosscode() { return env("CROSSCODE_BIN", "/usr/local/bin/crosscode"); },
|
||||
/** Agent releases: releases/<version>/ (checkouts), current → the running one, previous. */
|
||||
get agentRoot() { return env("CROSSCODE_AGENT_ROOT", "/opt/crosscode/agent"); },
|
||||
/** frps.ini on the central server. */
|
||||
get frpsIni() { return env("CROSSCODE_FRPS_INI", "/etc/frp/frps.ini"); },
|
||||
/** The churches on a church server. */
|
||||
get sites() { return env("CROSSCODE_SITES", "/var/www/sites"); },
|
||||
};
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* Log lines are filtered for anything that looks like a secret before they leave the server
|
||||
* (AGENT-1.0-PLAN §4, rule 7). The crosscode script never prints .env values, but tools it runs
|
||||
* (npm, Prisma, git) might print a URL with a password or a token in an error.
|
||||
*/
|
||||
|
||||
const RULES: [RegExp, string][] = [
|
||||
// user:password@ in any URL (postgresql://, https://).
|
||||
[/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s@/]+@/gi, "$1***@"],
|
||||
// KEY=value and KEY: value for keys that hold secrets.
|
||||
[/\b([A-Z0-9_]*(SECRET|PASSWORD|PASSWD|TOKEN|API_KEY|PRIVATE_KEY|DSN|LICENCE_KEY|LICENSE_KEY)[A-Z0-9_]*)(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/g, "$1$3***"],
|
||||
// Authorization headers and bearer tokens.
|
||||
[/\b(authorization\s*:\s*)(bearer|basic)\s+\S+/gi, "$1$2 ***"],
|
||||
[/\bbearer\s+[A-Za-z0-9._~+/=-]{16,}/gi, "Bearer ***"],
|
||||
// PEM blocks.
|
||||
[/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, "[private key]"],
|
||||
// Cross Code licence keys (CEH-7K2M-…).
|
||||
[/\b(CEH|CCA|CCW|CCP)-[A-Z0-9]{4}(-[A-Z0-9]{4}){3}\b/g, "$1-****"],
|
||||
// Long random-looking strings (hex or base64 of 32+ characters): likely keys or tokens.
|
||||
[/\b[A-Fa-f0-9]{40,}\b/g, "***"],
|
||||
[/\b[A-Za-z0-9+/_-]{48,}={0,2}/g, "***"],
|
||||
];
|
||||
|
||||
export function redact(line: string): string {
|
||||
let out = line;
|
||||
for (const [re, to] of RULES) out = out.replace(re, to);
|
||||
return out.length > 1000 ? `${out.slice(0, 997)}...` : out;
|
||||
}
|
||||
|
||||
/** Keeps the last `max` lines of output, redacted, without holding all of it in memory. */
|
||||
export class LogTail {
|
||||
private lines: string[] = [];
|
||||
private partial = "";
|
||||
constructor(private max = 200) {}
|
||||
push(chunk: string): void {
|
||||
const parts = (this.partial + chunk).split(/\r?\n/);
|
||||
this.partial = parts.pop() ?? "";
|
||||
for (const p of parts) this.add(p);
|
||||
}
|
||||
private add(line: string): void {
|
||||
this.lines.push(redact(line));
|
||||
if (this.lines.length > this.max) this.lines.shift();
|
||||
}
|
||||
end(): string[] {
|
||||
if (this.partial) {
|
||||
this.add(this.partial);
|
||||
this.partial = "";
|
||||
}
|
||||
return [...this.lines];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { agent } from "@crosscode/contracts";
|
||||
import { openConfig } from "@crosscode/contracts/signing";
|
||||
import { paths } from "./paths";
|
||||
import { LogTail, redact } from "./redact";
|
||||
|
||||
/**
|
||||
* Runs `crosscode … --json` for a job: argument array (no shell), the job's config opened to a
|
||||
* file only root can read and removed as soon as the job ends, a time limit per type (the whole
|
||||
* process group is stopped past it), and the last 200 log lines, redacted.
|
||||
*/
|
||||
|
||||
export interface RunResult {
|
||||
status: agent.JobStatus;
|
||||
steps: agent.ReportStep[];
|
||||
result: Record<string, unknown>;
|
||||
message?: string;
|
||||
log: string[];
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
}
|
||||
|
||||
export interface RunOptions {
|
||||
args: string[];
|
||||
sealedConfig?: agent.Sealed | null;
|
||||
x25519Pem: string;
|
||||
timeLimitMinutes: number;
|
||||
/** For tests. */
|
||||
bin?: string;
|
||||
}
|
||||
|
||||
const iso = (d = new Date()) => d.toISOString();
|
||||
|
||||
function writeConfig(sealed: agent.Sealed, x25519Pem: string): string {
|
||||
const opened = agent.jobConfigSchema.parse(openConfig(sealed, x25519Pem));
|
||||
fs.mkdirSync(paths.runDir, { recursive: true, mode: 0o700 });
|
||||
const file = path.join(paths.runDir, `config-${process.pid}-${Date.now()}.json`);
|
||||
fs.writeFileSync(file, JSON.stringify(opened), { mode: 0o600 });
|
||||
return file;
|
||||
}
|
||||
|
||||
export async function runCrosscode(opts: RunOptions): Promise<RunResult> {
|
||||
const startedAt = iso();
|
||||
let configFile: string | null = null;
|
||||
try {
|
||||
if (opts.sealedConfig) {
|
||||
try {
|
||||
configFile = writeConfig(opts.sealedConfig, opts.x25519Pem);
|
||||
} catch (e) {
|
||||
return { status: "refused", steps: [], result: {}, message: `The job's config couldn't be opened with this server's key: ${(e as Error).message}`, log: [], startedAt, finishedAt: iso() };
|
||||
}
|
||||
}
|
||||
const args = [...opts.args, "--json", ...(configFile ? ["--config", configFile] : [])];
|
||||
return await new Promise<RunResult>((resolve) => {
|
||||
const tail = new LogTail(200);
|
||||
let stdout = "";
|
||||
const child = spawn(opts.bin ?? paths.crosscode, args, {
|
||||
detached: true,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: { ...process.env, CROSSCODE_NO_UPDATE: process.env.CROSSCODE_NO_UPDATE ?? "", LC_ALL: "C.UTF-8" },
|
||||
});
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
try { process.kill(-child.pid!, "SIGTERM"); } catch { /* already gone */ }
|
||||
setTimeout(() => { try { process.kill(-child.pid!, "SIGKILL"); } catch { /* gone */ } }, 10_000).unref();
|
||||
}, opts.timeLimitMinutes * 60_000);
|
||||
child.stdout.setEncoding("utf8").on("data", (d: string) => { stdout += d; if (stdout.length > 5_000_000) stdout = stdout.slice(-1_000_000); });
|
||||
child.stderr.setEncoding("utf8").on("data", (d: string) => tail.push(d));
|
||||
child.on("error", (e) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ status: "failed", steps: [], result: {}, message: `crosscode couldn't be started: ${e.message}`, log: tail.end(), startedAt, finishedAt: iso() });
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timer);
|
||||
const log = tail.end();
|
||||
const report = parseReport(stdout);
|
||||
if (timedOut) {
|
||||
resolve({ status: "failed", steps: report?.steps ?? [], result: report?.result ?? {}, message: `Stopped after the time limit (${opts.timeLimitMinutes} minutes).`, log, startedAt, finishedAt: iso() });
|
||||
return;
|
||||
}
|
||||
if (!report) {
|
||||
resolve({ status: code === 0 ? "succeeded" : "failed", steps: [], result: {}, message: code === 0 ? undefined : `crosscode ended with code ${code} without a report.`, log, startedAt, finishedAt: iso() });
|
||||
return;
|
||||
}
|
||||
resolve({
|
||||
status: report.status,
|
||||
steps: report.steps,
|
||||
result: report.result,
|
||||
message: report.message ? redact(report.message) : undefined,
|
||||
log,
|
||||
startedAt: report.startedAt,
|
||||
finishedAt: report.finishedAt,
|
||||
});
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
if (configFile) fs.rmSync(configFile, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** The last line of stdout that's a valid script report. */
|
||||
export function parseReport(stdout: string): agent.ScriptReport | null {
|
||||
const lines = stdout.trim().split("\n").reverse();
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const parsed = agent.scriptReportSchema.safeParse(JSON.parse(line));
|
||||
if (parsed.success) return parsed.data;
|
||||
} catch {
|
||||
// not JSON
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { agent } from "@crosscode/contracts";
|
||||
import { readFrps } from "./central";
|
||||
import { paths } from "./paths";
|
||||
|
||||
/**
|
||||
* The server's stats every 60 s (AGENT-1.0-PLAN §10, crosscode.agent-stats/1): RAM, CPU, disks,
|
||||
* uptime, versions, frpc, and each PM2 process with its church and product. Read from /proc and
|
||||
* the tools that are there; anything missing is null rather than an error.
|
||||
*/
|
||||
|
||||
export const AGENT_VERSION = "1.0.0";
|
||||
|
||||
function run(cmd: string, args: string[], timeoutMs = 10_000): Promise<string | null> {
|
||||
return new Promise((resolve) => {
|
||||
execFile(cmd, args, { timeout: timeoutMs, maxBuffer: 10_000_000 }, (err, stdout) => resolve(err ? null : String(stdout)));
|
||||
});
|
||||
}
|
||||
|
||||
export function parseMeminfo(text: string): agent.AgentStats["ram"] {
|
||||
const kb = (key: string) => Number(new RegExp(`^${key}:\\s+(\\d+)`, "m").exec(text)?.[1] ?? 0) * 1024;
|
||||
const total = kb("MemTotal");
|
||||
const available = kb("MemAvailable");
|
||||
return { totalBytes: total, usedBytes: Math.max(0, total - available), availableBytes: available, swapTotalBytes: kb("SwapTotal"), swapUsedBytes: Math.max(0, kb("SwapTotal") - kb("SwapFree")) };
|
||||
}
|
||||
|
||||
/** CPU busy % between two /proc/stat readings. */
|
||||
export function cpuPercent(a: string, b: string): number {
|
||||
const parse = (t: string) => {
|
||||
const f = (/^cpu\s+(.+)$/m.exec(t)?.[1] ?? "").trim().split(/\s+/).map(Number);
|
||||
const idle = (f[3] ?? 0) + (f[4] ?? 0);
|
||||
return { idle, total: f.reduce((s, n) => s + (Number.isFinite(n) ? n : 0), 0) };
|
||||
};
|
||||
const x = parse(a), y = parse(b);
|
||||
const total = y.total - x.total;
|
||||
if (total <= 0) return 0;
|
||||
return Math.min(100, Math.max(0, Math.round(((total - (y.idle - x.idle)) / total) * 1000) / 10));
|
||||
}
|
||||
|
||||
async function disks(): Promise<agent.AgentStats["disks"]> {
|
||||
const out = await run("df", ["-P", "-B1", "-x", "tmpfs", "-x", "devtmpfs", "-x", "squashfs", "-x", "overlay"]);
|
||||
if (!out) return [];
|
||||
return out.trim().split("\n").slice(1).map((l) => l.trim().split(/\s+/)).filter((c) => c.length >= 6 && c[5]!.startsWith("/"))
|
||||
.slice(0, 30).map((c) => ({ mount: c[5]!, totalBytes: Number(c[1]), usedBytes: Number(c[2]), freeBytes: Number(c[3]) }));
|
||||
}
|
||||
|
||||
function osName(): string {
|
||||
try {
|
||||
const t = fs.readFileSync("/etc/os-release", "utf8");
|
||||
return /^PRETTY_NAME="?([^"\n]+)"?/m.exec(t)?.[1] ?? `${os.type()} ${os.release()}`;
|
||||
} catch {
|
||||
return `${os.type()} ${os.release()}`;
|
||||
}
|
||||
}
|
||||
|
||||
const versionIn = (text: string | null) => (text ? /(\d+\.\d+(\.\d+)?)/.exec(text)?.[1] ?? null : null);
|
||||
|
||||
/** church.env → { app user → [{ church, product, names }] }, read as data like the script does. */
|
||||
export function churchProcesses(sitesDir = paths.sites): Map<string, { church: string; product: agent.AgentStats["processes"][number]["product"]; names: string[] }[]> {
|
||||
const byUser = new Map<string, { church: string; product: "events" | "accounts" | "website" | "app"; names: string[] }[]>();
|
||||
let entries: string[];
|
||||
try { entries = fs.readdirSync(sitesDir); } catch { return byUser; }
|
||||
for (const church of entries) {
|
||||
if (!/^[a-z][a-z0-9-]{0,39}$/.test(church)) continue;
|
||||
let text: string;
|
||||
try { text = fs.readFileSync(path.join(sitesDir, church, "church.env"), "utf8"); } catch { continue; }
|
||||
const vars = new Map<string, string>();
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
const m = /^([A-Z][A-Z0-9_]*)=("([^"]*)"|'([^']*)'|([^#\s]*))/.exec(line);
|
||||
if (m) vars.set(m[1]!, m[3] ?? m[4] ?? m[5] ?? "");
|
||||
}
|
||||
for (const product of ["accounts", "events", "website", "app"] as const) {
|
||||
const p = product.toUpperCase();
|
||||
const user = vars.get(`${p}_USER`);
|
||||
const names = (vars.get(`${p}_PM2`) ?? "").split(/\s+/).filter(Boolean);
|
||||
if (!user || !names.length || !/^[a-z_][a-z0-9_-]{0,31}$/.test(user)) continue;
|
||||
const list = byUser.get(user) ?? [];
|
||||
list.push({ church, product, names });
|
||||
byUser.set(user, list);
|
||||
}
|
||||
}
|
||||
return byUser;
|
||||
}
|
||||
|
||||
interface Pm2Proc { name?: string; monit?: { memory?: number; cpu?: number }; pm2_env?: { status?: string; restart_time?: number; pm_uptime?: number } }
|
||||
|
||||
async function pm2Processes(): Promise<agent.AgentStats["processes"]> {
|
||||
const out: agent.AgentStats["processes"] = [];
|
||||
for (const [user, installs] of churchProcesses()) {
|
||||
const json = await run("sudo", ["-n", "-u", user, "-H", "--", "pm2", "jlist"]);
|
||||
let list: Pm2Proc[];
|
||||
try { list = JSON.parse(json ?? "[]") as Pm2Proc[]; } catch { list = []; }
|
||||
for (const p of list) {
|
||||
const owner = installs.find((i) => i.names.includes(p.name ?? ""));
|
||||
out.push({
|
||||
name: String(p.name ?? "?").slice(0, 100),
|
||||
church: owner?.church ?? null,
|
||||
product: owner?.product ?? null,
|
||||
status: String(p.pm2_env?.status ?? "unknown").slice(0, 40),
|
||||
memoryBytes: Number(p.monit?.memory ?? 0),
|
||||
cpuPercent: Number(p.monit?.cpu ?? 0),
|
||||
restarts: Math.max(0, Math.trunc(Number(p.pm2_env?.restart_time ?? 0))),
|
||||
uptimeSeconds: p.pm2_env?.pm_uptime ? Math.max(0, (Date.now() - p.pm2_env.pm_uptime) / 1000) : 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out.slice(0, 200);
|
||||
}
|
||||
|
||||
/** A systemd unit is active; null when there's no systemctl. (is-active exits non-zero when inactive.) */
|
||||
function active(unit: string): Promise<boolean | null> {
|
||||
return new Promise((resolve) => {
|
||||
execFile("systemctl", ["is-active", unit], { timeout: 10_000 }, (err, stdout) => {
|
||||
if (err && (err as NodeJS.ErrnoException).code === "ENOENT") resolve(null);
|
||||
else resolve(String(stdout).trim() === "active");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function collectStats(mode: agent.AgentMode): Promise<agent.AgentStats> {
|
||||
const read = (f: string) => { try { return fs.readFileSync(f, "utf8"); } catch { return ""; } };
|
||||
const stat1 = read("/proc/stat");
|
||||
await new Promise((r) => setTimeout(r, 1000));
|
||||
const stat2 = read("/proc/stat");
|
||||
const [load1, load5, load15] = os.loadavg();
|
||||
const [diskList, postgres, frpc, processes, frpcRunning] = await Promise.all([
|
||||
disks(),
|
||||
run("psql", ["--version"]),
|
||||
run("frpc", ["-v"]),
|
||||
mode === "church" ? pm2Processes() : Promise.resolve([]),
|
||||
mode === "church" ? active("frpc") : Promise.resolve(null),
|
||||
]);
|
||||
const stats: agent.AgentStats = {
|
||||
schema: "crosscode.agent-stats/1",
|
||||
at: new Date().toISOString(),
|
||||
ram: parseMeminfo(read("/proc/meminfo")),
|
||||
cpu: { percent: cpuPercent(stat1, stat2), load1: load1 ?? 0, load5: load5 ?? 0, load15: load15 ?? 0, cores: Math.max(1, os.cpus().length) },
|
||||
disks: diskList,
|
||||
uptimeSeconds: os.uptime(),
|
||||
versions: { os: osName().slice(0, 200), node: process.versions.node, postgres: versionIn(postgres), frpc: versionIn(frpc), agent: AGENT_VERSION },
|
||||
frpcRunning,
|
||||
processes,
|
||||
};
|
||||
if (mode === "central") {
|
||||
const frps = readFrps();
|
||||
stats.central = { proxyBindAddr: frps?.proxyBindAddr ?? null, allowPorts: frps?.allowPorts ?? null, frpsRunning: await active("frps") };
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { agent } from "@crosscode/contracts";
|
||||
import { paths } from "./paths";
|
||||
|
||||
/**
|
||||
* update-agent (AGENT-1.0-PLAN §3): the newest agent@x.y.z is checked out and built next to the
|
||||
* running copy, `current` is switched to it and the agent exits, so systemd starts the new one.
|
||||
* A one-shot systemd timer switches back to `previous` unless the new agent clears
|
||||
* update-pending.json by polling the Hub within 150 seconds.
|
||||
*
|
||||
* /opt/crosscode/agent/releases/<version>/ checkouts of the monorepo at agent@<version>
|
||||
* /opt/crosscode/agent/current, previous symlinks to releases
|
||||
* /opt/crosscode/agent/revert.sh written by the install script
|
||||
*/
|
||||
|
||||
const REPO = process.env.CROSSCODE_REPO || "https://c-git.crosscode.co.za/joshua/crosscode-platform.git";
|
||||
const CONFIRM_SECONDS = 150;
|
||||
|
||||
const pendingFile = () => path.join(paths.stateDir, "update-pending.json");
|
||||
|
||||
function exec(cmd: string, args: string[], cwd?: string, timeoutMs = 15 * 60_000): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(cmd, args, { cwd, timeout: timeoutMs, maxBuffer: 20_000_000, env: { ...process.env, NODE_ENV: "development" } }, (err, stdout, stderr) => {
|
||||
if (err) reject(new Error(`${cmd} ${args.join(" ")}: ${String(stderr).trim().split("\n").slice(-3).join(" ") || err.message}`));
|
||||
else resolve(String(stdout));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export interface UpdateOutcome {
|
||||
status: agent.JobStatus;
|
||||
message: string;
|
||||
steps: agent.ReportStep[];
|
||||
result: Record<string, unknown>;
|
||||
/** Exit after reporting, so systemd starts the new version. */
|
||||
restart: boolean;
|
||||
}
|
||||
|
||||
export async function updateAgent(tag: string, currentVersion: string): Promise<UpdateOutcome> {
|
||||
const version = tag.replace(/^agent@/, "");
|
||||
const steps: agent.ReportStep[] = [];
|
||||
const step = async (name: string, fn: () => Promise<void>) => {
|
||||
const s: agent.ReportStep = { name, startedAt: new Date().toISOString(), finishedAt: null, outcome: "ok" };
|
||||
steps.push(s);
|
||||
try {
|
||||
await fn();
|
||||
} catch (e) {
|
||||
s.outcome = "failed";
|
||||
s.detail = (e as Error).message.slice(0, 2000);
|
||||
throw e;
|
||||
} finally {
|
||||
s.finishedAt = new Date().toISOString();
|
||||
}
|
||||
};
|
||||
if (version === currentVersion) return { status: "succeeded", message: `Already on ${tag}.`, steps, result: { version }, restart: false };
|
||||
|
||||
const root = paths.agentRoot;
|
||||
const dir = path.join(root, "releases", version);
|
||||
try {
|
||||
await step(`Checking out ${tag}`, async () => {
|
||||
if (!fs.existsSync(path.join(dir, ".git"))) {
|
||||
fs.mkdirSync(path.dirname(dir), { recursive: true });
|
||||
await exec("git", ["clone", "--quiet", "--depth", "1", "--branch", tag, REPO, dir]);
|
||||
}
|
||||
});
|
||||
await step("Installing and building", async () => {
|
||||
await exec("npm", ["ci", "--no-audit", "--no-fund", "--include=dev", "-w", "packages/contracts", "-w", "apps/agent"], dir);
|
||||
await exec("npm", ["run", "build", "-w", "packages/contracts"], dir);
|
||||
await exec("npm", ["run", "build", "-w", "apps/agent"], dir);
|
||||
});
|
||||
await step("Checking the new build", async () => {
|
||||
const out = await exec(process.execPath, [path.join(dir, "apps/agent/dist/cli.js"), "version"], dir, 30_000);
|
||||
if (out.trim() !== version) throw new Error(`The new build says it's ${out.trim()}, not ${version}.`);
|
||||
});
|
||||
await step("Switching over", async () => {
|
||||
const current = path.join(root, "current");
|
||||
const previous = path.join(root, "previous");
|
||||
const old = fs.readlinkSync(current);
|
||||
fs.writeFileSync(pendingFile(), JSON.stringify({ from: currentVersion, to: version, at: new Date().toISOString() }), { mode: 0o600 });
|
||||
replaceLink(previous, old);
|
||||
replaceLink(current, dir);
|
||||
// If the new agent doesn't poll within CONFIRM_SECONDS, revert.sh puts `previous` back.
|
||||
await exec("systemd-run", [`--on-active=${CONFIRM_SECONDS}`, `--unit=crosscode-agent-revert-${Date.now()}`, path.join(root, "revert.sh")]);
|
||||
});
|
||||
} catch (e) {
|
||||
return { status: "failed", message: `The agent wasn't updated: ${(e as Error).message}`, steps, result: { from: currentVersion, to: version }, restart: false };
|
||||
}
|
||||
return { status: "succeeded", message: `Switching to ${tag}; it confirms by polling within ${CONFIRM_SECONDS} seconds.`, steps, result: { from: currentVersion, to: version }, restart: true };
|
||||
}
|
||||
|
||||
function replaceLink(link: string, target: string): void {
|
||||
const tmp = `${link}.tmp-${process.pid}`;
|
||||
fs.rmSync(tmp, { force: true });
|
||||
fs.symlinkSync(target, tmp);
|
||||
fs.renameSync(tmp, link);
|
||||
}
|
||||
|
||||
/** Called after the first successful poll: this version works, so it stays. */
|
||||
export function confirmUpdate(): void {
|
||||
fs.rmSync(pendingFile(), { force: true });
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { agent } from "@crosscode/contracts";
|
||||
import { verifyJob } from "@crosscode/contracts/signing";
|
||||
import type { AgentConfig } from "./config";
|
||||
import { paths } from "./paths";
|
||||
|
||||
/**
|
||||
* Whether to run a job (AGENT-1.0-PLAN §4). Checked in this order, and a job is refused at the
|
||||
* first check it fails:
|
||||
* 1. it's an object with an id and type the agent can report on,
|
||||
* 2. the signature verifies with the Hub key pinned when the server joined,
|
||||
* 3. it's for this server,
|
||||
* 4. it's within notBefore … expiresAt (an expired job is reported as `expired`, never run late),
|
||||
* 5. its id hasn't been seen in the last 90 days (replays),
|
||||
* 6. its type is one this agent runs (a central agent only runs check-ports, status, update-agent),
|
||||
* 7. every argument passes that type's schema (unknown arguments refused).
|
||||
*/
|
||||
|
||||
export type Verdict =
|
||||
| { ok: true; job: agent.AgentJob }
|
||||
| { ok: false; status: "refused" | "expired"; message: string; id: string | null; type: agent.JobType | null };
|
||||
|
||||
const SEEN_DAYS = 90;
|
||||
|
||||
/** Job ids already run or refused, with when (replay protection). */
|
||||
export class SeenJobs {
|
||||
private seen = new Map<string, number>();
|
||||
constructor(private file = path.join(paths.stateDir, "seen-jobs.json")) {
|
||||
try {
|
||||
const data = JSON.parse(fs.readFileSync(this.file, "utf8")) as Record<string, number>;
|
||||
for (const [id, at] of Object.entries(data)) this.seen.set(id, at);
|
||||
} catch {
|
||||
// First run, or an unreadable file: start empty (signatures and expiry still apply).
|
||||
}
|
||||
}
|
||||
has(id: string): boolean {
|
||||
return this.seen.has(id);
|
||||
}
|
||||
add(id: string, now = Date.now()): void {
|
||||
this.seen.set(id, now);
|
||||
const cutoff = now - SEEN_DAYS * 86_400_000;
|
||||
for (const [k, at] of this.seen) if (at < cutoff) this.seen.delete(k);
|
||||
fs.mkdirSync(path.dirname(this.file), { recursive: true, mode: 0o700 });
|
||||
const tmp = `${this.file}.tmp-${process.pid}`;
|
||||
fs.writeFileSync(tmp, JSON.stringify(Object.fromEntries(this.seen)), { mode: 0o600 });
|
||||
fs.renameSync(tmp, this.file);
|
||||
}
|
||||
}
|
||||
|
||||
const typeOf = (v: unknown): agent.JobType | null => (agent.jobTypeSchema.safeParse(v).success ? (v as agent.JobType) : null);
|
||||
|
||||
export function checkJob(raw: unknown, config: AgentConfig, seen: SeenJobs, now = new Date()): Verdict {
|
||||
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return { ok: false, status: "refused", message: "Not a job.", id: null, type: null };
|
||||
const j = raw as Record<string, unknown>;
|
||||
const id = typeof j.id === "string" && /^job_[A-Za-z0-9_-]{8,64}$/.test(j.id) ? j.id : null;
|
||||
const type = typeOf(j.type);
|
||||
const refuse = (message: string): Verdict => ({ ok: false, status: "refused", message, id, type });
|
||||
|
||||
if (!id) return refuse("The job has no valid id.");
|
||||
if (!verifyJob(j, config.hubJobKey)) return refuse("The signature doesn't match the Hub's key.");
|
||||
if (j.serverId !== config.serverId) return refuse(`The job is for ${String(j.serverId)}, not this server.`);
|
||||
const notBefore = Date.parse(String(j.notBefore));
|
||||
const expiresAt = Date.parse(String(j.expiresAt));
|
||||
if (!Number.isFinite(notBefore) || !Number.isFinite(expiresAt) || expiresAt <= notBefore) return refuse("The job's times aren't valid.");
|
||||
if (now.getTime() > expiresAt) return { ok: false, status: "expired", message: `The job expired at ${new Date(expiresAt).toISOString()}; it wasn't run.`, id, type };
|
||||
// A minute's leeway for clocks that differ a little.
|
||||
if (now.getTime() < notBefore - 60_000) return refuse(`The job isn't due until ${new Date(notBefore).toISOString()}.`);
|
||||
if (seen.has(id)) return refuse("This job was already received (a replay).");
|
||||
if (!type) return refuse(`Unknown job type ${JSON.stringify(j.type)}.`);
|
||||
if (config.mode === "central" && !agent.CENTRAL_JOB_TYPES.includes(type)) return refuse(`A central agent doesn't run ${type} jobs.`);
|
||||
const parsed = agent.agentJobSchema.safeParse(raw);
|
||||
if (!parsed.success) {
|
||||
const issue = parsed.error.issues[0];
|
||||
return refuse(`The job's ${issue?.path.join(".") || "content"} isn't valid: ${issue?.message ?? "unknown problem"}.`);
|
||||
}
|
||||
return { ok: true, job: parsed.data };
|
||||
}
|
||||
Reference in New Issue
Block a user