Central servers only get the agent (install.sh --central), not the crosscode script, but a central agent accepts status jobs and ran `crosscode status` for them, which failed with `spawn /usr/local/bin/crosscode ENOENT`. A central agent now reports its own stats (frps and the agent version) for status; church servers still run crosscode status. Also ignores temp/ (local screenshots). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
236 lines
9.6 KiB
TypeScript
236 lines
9.6 KiB
TypeScript
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { agent } from "@crosscode/contracts";
|
|
import { encryptExport } from "@crosscode/contracts/signing";
|
|
import { checkPorts, listeningNow, readFrps } from "./central";
|
|
import { crosscodeArgs } from "./command";
|
|
import type { AgentConfig, Keys } from "./config";
|
|
import { type HubClient, HubRequestError } from "./hub";
|
|
import { paths } from "./paths";
|
|
import { runCrosscode, type RunResult } from "./runner";
|
|
import { AGENT_VERSION, collectStats } from "./stats";
|
|
import { confirmUpdate, updateAgent } from "./update";
|
|
import { checkJob, SeenJobs } from "./verify";
|
|
|
|
/**
|
|
* The agent's life: poll the Hub every 30 s for jobs due, send stats every 60 s, run one job at a
|
|
* time and send each result. Reports that can't be sent are kept in an outbox and sent later, so
|
|
* a Hub outage never loses a job's result. Nothing here listens: every request goes out.
|
|
*/
|
|
|
|
const log = (...a: unknown[]) => console.log(new Date().toISOString(), ...a);
|
|
|
|
export interface AgentDeps {
|
|
config: AgentConfig;
|
|
keys: Keys;
|
|
hub: HubClient;
|
|
seen?: SeenJobs;
|
|
/** For tests: replaces running crosscode and the stats. */
|
|
run?: (args: string[], job: agent.AgentJob) => Promise<RunResult>;
|
|
stats?: () => Promise<agent.AgentStats>;
|
|
now?: () => Date;
|
|
}
|
|
|
|
const outboxDir = () => path.join(paths.stateDir, "outbox");
|
|
|
|
function report(job: { id: string; type: agent.JobType }, serverId: string, r: Omit<RunResult, "log"> & { log?: string[] }): agent.AgentReport {
|
|
return {
|
|
schema: "crosscode.agent-report/1",
|
|
jobId: job.id,
|
|
serverId,
|
|
type: job.type,
|
|
status: r.status,
|
|
startedAt: r.startedAt,
|
|
finishedAt: r.finishedAt,
|
|
steps: r.steps.slice(0, 200),
|
|
result: r.result,
|
|
...(r.message ? { message: r.message.slice(0, 2000) } : {}),
|
|
log: (r.log ?? []).slice(-200),
|
|
};
|
|
}
|
|
|
|
export class Agent {
|
|
private seen: SeenJobs;
|
|
private stopped = false;
|
|
private confirmed = false;
|
|
private failures = 0;
|
|
|
|
constructor(private deps: AgentDeps) {
|
|
this.seen = deps.seen ?? new SeenJobs();
|
|
}
|
|
|
|
private now(): Date {
|
|
return this.deps.now?.() ?? new Date();
|
|
}
|
|
|
|
/** Sends a report, or keeps it in the outbox to send later. */
|
|
async send(r: agent.AgentReport): Promise<void> {
|
|
try {
|
|
await this.deps.hub.post("/api/agent/v1/report", r);
|
|
} catch (e) {
|
|
log(`[report] ${r.jobId}: couldn't send (${(e as Error).message}); kept to send later.`);
|
|
fs.mkdirSync(outboxDir(), { recursive: true, mode: 0o700 });
|
|
fs.writeFileSync(path.join(outboxDir(), `${r.jobId}.json`), JSON.stringify(r), { mode: 0o600 });
|
|
}
|
|
}
|
|
|
|
async flushOutbox(): Promise<void> {
|
|
let files: string[];
|
|
try { files = fs.readdirSync(outboxDir()).filter((f) => f.endsWith(".json")); } catch { return; }
|
|
for (const f of files) {
|
|
const file = path.join(outboxDir(), f);
|
|
try {
|
|
await this.deps.hub.post("/api/agent/v1/report", JSON.parse(fs.readFileSync(file, "utf8")));
|
|
fs.rmSync(file, { force: true });
|
|
} catch (e) {
|
|
// A report the Hub refuses outright (4xx) would never go: drop it rather than retry forever.
|
|
if (e instanceof HubRequestError && e.status >= 400 && e.status < 500) fs.rmSync(file, { force: true });
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/** One poll: fetch the jobs due and run them, one after the other. Returns the next delay (s). */
|
|
async pollOnce(): Promise<number> {
|
|
const { config } = this.deps;
|
|
let reply: { jobs: unknown[]; pollSeconds: number };
|
|
try {
|
|
reply = agent.pollResponseSchema.parse(await this.deps.hub.post("/api/agent/v1/poll", {
|
|
agentApiVersion: agent.AGENT_API_VERSION, agentVersion: AGENT_VERSION, mode: config.mode, runningJobId: null,
|
|
}));
|
|
} catch (e) {
|
|
this.failures++;
|
|
const delay = Math.min(300, config.pollSeconds * 2 ** Math.min(this.failures, 4));
|
|
log(`[poll] ${(e as Error).message}; next try in ${delay}s.`);
|
|
return delay;
|
|
}
|
|
this.failures = 0;
|
|
if (!this.confirmed) {
|
|
confirmUpdate();
|
|
this.confirmed = true;
|
|
}
|
|
await this.flushOutbox();
|
|
for (const raw of reply.jobs) {
|
|
if (this.stopped) break;
|
|
await this.handle(raw);
|
|
}
|
|
return reply.pollSeconds || config.pollSeconds;
|
|
}
|
|
|
|
async handle(raw: unknown): Promise<void> {
|
|
const { config } = this.deps;
|
|
const verdict = checkJob(raw, config, this.seen, this.now());
|
|
const at = this.now().toISOString();
|
|
if (!verdict.ok) {
|
|
log(`[job] ${verdict.id ?? "(no id)"} ${verdict.status}: ${verdict.message}`);
|
|
if (verdict.id && verdict.type && !this.seen.has(verdict.id)) {
|
|
this.seen.add(verdict.id);
|
|
await this.send(report({ id: verdict.id, type: verdict.type }, config.serverId, { status: verdict.status, steps: [], result: {}, message: verdict.message, startedAt: at, finishedAt: at }));
|
|
}
|
|
return;
|
|
}
|
|
const job = verdict.job;
|
|
// Seen before it runs, so a crash part-way can never run it twice.
|
|
this.seen.add(job.id);
|
|
log(`[job] ${job.id} ${job.type} ${JSON.stringify(job.args)}`);
|
|
const result = await this.execute(job);
|
|
log(`[job] ${job.id} ${result.status}${result.message ? `: ${result.message}` : ""}`);
|
|
await this.send(report(job, config.serverId, result));
|
|
if ((result as RunResult & { restart?: boolean }).restart) {
|
|
log("[agent] Exiting so systemd starts the new version.");
|
|
this.stopped = true;
|
|
setTimeout(() => process.exit(0), 500);
|
|
}
|
|
}
|
|
|
|
private async execute(job: agent.AgentJob): Promise<RunResult & { restart?: boolean }> {
|
|
const startedAt = this.now().toISOString();
|
|
const limit = agent.JOB_TIME_LIMIT_MINUTES[job.type];
|
|
try {
|
|
switch (job.type) {
|
|
case "update-agent": {
|
|
const u = await updateAgent(job.args.version, AGENT_VERSION);
|
|
return { status: u.status, steps: u.steps, result: u.result, message: u.message, log: [], startedAt, finishedAt: new Date().toISOString(), restart: u.restart };
|
|
}
|
|
case "check-ports": {
|
|
const res = checkPorts(job.args.ports, readFrps(), await listeningNow());
|
|
const ok = res.ports.every((p) => p.ok) && res.warnings.length === 0;
|
|
return { status: ok ? "succeeded" : "failed", steps: [], result: res as unknown as Record<string, unknown>, message: res.warnings.join(" ") || undefined, log: [], startedAt, finishedAt: new Date().toISOString() };
|
|
}
|
|
case "export":
|
|
return await this.runExport(job, limit);
|
|
case "status":
|
|
// The central server has no crosscode script (install.sh --central): its status is the
|
|
// agent's own stats, with frps.
|
|
if (this.deps.config.mode === "central") {
|
|
const stats = await (this.deps.stats ?? (() => collectStats("central")))();
|
|
const warning = stats.central && !stats.central.frpsRunning ? "frps isn't running." : undefined;
|
|
return { status: "succeeded", steps: [], result: { stats } as unknown as Record<string, unknown>, message: warning, log: [], startedAt, finishedAt: new Date().toISOString() };
|
|
}
|
|
return await this.run(crosscodeArgs(job)!, job, limit);
|
|
default: {
|
|
const args = crosscodeArgs(job);
|
|
if (!args) throw new Error(`No command for ${job.type}`);
|
|
return await this.run(args, job, limit);
|
|
}
|
|
}
|
|
} catch (e) {
|
|
return { status: "failed", steps: [], result: {}, message: (e as Error).message, log: [], startedAt, finishedAt: new Date().toISOString() };
|
|
}
|
|
}
|
|
|
|
private run(args: string[], job: agent.AgentJob, limit: number): Promise<RunResult> {
|
|
if (this.deps.run) return this.deps.run(args, job);
|
|
return runCrosscode({ args, sealedConfig: job.config ?? null, x25519Pem: this.deps.keys.x25519Pem, timeLimitMinutes: limit });
|
|
}
|
|
|
|
/**
|
|
* export: crosscode writes the ZIP, the agent encrypts it with a fresh key and uploads it. The
|
|
* key goes to the Hub only in this job's report, so it can email the download link; the Hub
|
|
* doesn't keep it (AGENT-1.0-PLAN §9).
|
|
*/
|
|
private async runExport(job: Extract<agent.AgentJob, { type: "export" }>, limit: number): Promise<RunResult> {
|
|
const dir = path.join(paths.stateDir, "exports");
|
|
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
const zip = path.join(dir, `${job.args.exportId}.zip`);
|
|
fs.rmSync(zip, { force: true });
|
|
try {
|
|
const r = await this.run(crosscodeArgs(job, { exportFile: zip })!, job, limit);
|
|
if (r.status !== "succeeded") return r;
|
|
const { key, file } = encryptExport(fs.readFileSync(zip));
|
|
await this.deps.hub.put(`/api/agent/v1/exports/${job.args.exportId}`, file, "application/octet-stream");
|
|
return { ...r, result: { ...r.result, file: undefined, encryptedBytes: file.length, key } };
|
|
} finally {
|
|
fs.rmSync(zip, { force: true });
|
|
}
|
|
}
|
|
|
|
async statsOnce(): Promise<void> {
|
|
try {
|
|
const stats = await (this.deps.stats ?? (() => collectStats(this.deps.config.mode)))();
|
|
await this.deps.hub.post("/api/agent/v1/stats", stats);
|
|
} catch (e) {
|
|
log(`[stats] ${(e as Error).message}`);
|
|
}
|
|
}
|
|
|
|
/** Runs until stopped: polls and stats on their own timers, never at the same time as a job. */
|
|
async start(): Promise<void> {
|
|
log(`[agent] ${AGENT_VERSION} (${this.deps.config.mode}) for ${this.deps.config.serverId}, Hub ${this.deps.config.hubUrl}`);
|
|
let nextStats = 0;
|
|
while (!this.stopped) {
|
|
if (Date.now() >= nextStats) {
|
|
await this.statsOnce();
|
|
nextStats = Date.now() + this.deps.config.statsSeconds * 1000;
|
|
}
|
|
const delay = await this.pollOnce();
|
|
await new Promise((r) => setTimeout(r, Math.max(5, Math.min(delay, this.deps.config.statsSeconds)) * 1000));
|
|
}
|
|
}
|
|
|
|
stop(): void {
|
|
this.stopped = true;
|
|
}
|
|
}
|
|
|