From 570f2dda5a34e29f4cb85c5efe1804d1dc8a5a42 Mon Sep 17 00:00:00 2001 From: Joshua Oosthuizen Date: Tue, 17 Nov 2026 16:40:12 +0200 Subject: [PATCH] PM2 runs under its own systemd unit, never as a child of the agent Found in the VM test: a site's PM2 daemon started from the agent's install job belonged to the agent's service, so stopping or updating the agent took down every site it had installed. Before any PM2 command as a site user, the scripts now make sure that user's daemon runs under pm2- (moving one that runs elsewhere, keeping its process list). The agent's stats no longer start an empty daemon by asking one that isn't running. Uninstall stops the unit before removing it. Co-Authored-By: Claude Opus 5.5 --- apps/agent/src/stats.ts | 17 ++++++++++++++++- scripts/lib/common.sh | 27 +++++++++++++++++++++++++++ scripts/lib/install.sh | 1 - scripts/lib/manage.sh | 1 + 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/apps/agent/src/stats.ts b/apps/agent/src/stats.ts index b1af290..494488d 100644 --- a/apps/agent/src/stats.ts +++ b/apps/agent/src/stats.ts @@ -1,4 +1,4 @@ -import { execFile } from "node:child_process"; +import { execFile, execFileSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -87,9 +87,24 @@ export function churchProcesses(sitesDir = paths.sites): Map { const out: agent.AgentStats["processes"] = []; for (const [user, installs] of churchProcesses()) { + // Only ask a daemon that's running: `pm2 jlist` would otherwise start an empty one inside this + // agent's service, which dies with the agent (found in the VM test). + if (!pm2Running(user)) continue; const json = await run("sudo", ["-n", "-u", user, "-H", "--", "pm2", "jlist"]); let list: Pm2Proc[]; try { list = JSON.parse(json ?? "[]") as Pm2Proc[]; } catch { list = []; } diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 28fef7d..daca61e 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -66,8 +66,35 @@ upper() { printf '%s' "$1" | tr '[:lower:]-' '[:upper:]_'; } # ── Running things as the app user ─────────────────────────────────────────────────────────── # run_as : runs the command in as (directly when we already # are that user), with that user's HOME so PM2 and npm use their own folders. +# pm2_service: makes sure the pm2 daemon of user $1 runs under its own systemd unit (pm2-), +# never as a child of whatever ran this script. A daemon started from a command (the agent's job, +# an SSH session) belongs to that command's service and dies with it: stopping or updating the +# agent took down every site it had installed (found in the VM test). Done once per user per run. +PM2_SERVICE_DONE=" " +pm2_service() { + local user="$1" home pid n + case "$PM2_SERVICE_DONE" in *" $user "*) return 0 ;; esac + PM2_SERVICE_DONE="$PM2_SERVICE_DONE$user " + { command -v systemctl >/dev/null && [ -d /run/systemd/system ] && [ "$(id -u)" = 0 ]; } || return 0 + home="$(getent passwd "$user" | cut -d: -f6)" + systemctl cat "pm2-$user" >/dev/null 2>&1 || env PATH="$PATH:/usr/bin" pm2 startup systemd -u "$user" --hp "$home" >/dev/null + systemctl is-active -q "pm2-$user" && return 0 + pid="$(cat "$home/.pm2/pm2.pid" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + # A daemon outside its unit: keep its list (an empty one doesn't replace the saved list), + # stop it, and let the unit bring the list back. Its sites are down for a few seconds. + n="$(sudo -u "$user" -H -- pm2 jlist 2>/dev/null | node -e 'let s = ""; process.stdin.on("data", (d) => (s += d)).on("end", () => { try { console.log(JSON.parse(s).length); } catch { console.log(0); } })')" + if [ "${n:-0}" -gt 0 ]; then sudo -u "$user" -H -- pm2 save --force >/dev/null 2>&1 || true; fi + sudo -u "$user" -H -- pm2 kill >/dev/null 2>&1 || true + info "PM2 for $user now runs under systemd (pm2-$user)." + fi + systemctl start "pm2-$user" || warn "pm2-$user didn't start: check systemctl status pm2-$user." +} + run_as() { local user="$1" dir="$2" + # Any PM2 command runs against the daemon in its own unit (see pm2_service). + if [ "$3" = pm2 ]; then pm2_service "$user"; fi shift 2 if [ "$(id -un)" = "$user" ]; then (cd -- "$dir" && "$@") diff --git a/scripts/lib/install.sh b/scripts/lib/install.sh index f6e21cb..312b145 100644 --- a/scripts/lib/install.sh +++ b/scripts/lib/install.sh @@ -261,7 +261,6 @@ install_events() { undo_add pm2 "$user" "$name-backend $name-frontend" run_as "$user" "$app" pm2 startOrReload ecosystem.config.js run_as "$user" "$app" pm2 save - env PATH="$PATH:/usr/bin" pm2 startup systemd -u "$user" --hp "$(getent passwd "$user" | cut -d: -f6)" >/dev/null if [ "$frp" = 1 ]; then step "Events: frpc tunnels" diff --git a/scripts/lib/manage.sh b/scripts/lib/manage.sh index 517be78..479ccb9 100644 --- a/scripts/lib/manage.sh +++ b/scripts/lib/manage.sh @@ -170,6 +170,7 @@ cmd_uninstall() { rm -rf -- "$(church_dir "$church")" # The app user goes too when crosscode made it for this church (cc-). if [ "$user" = "cc-$church" ] && id "$user" >/dev/null 2>&1; then + systemctl stop "pm2-$user" >/dev/null 2>&1 || true pm2 unstartup systemd -u "$user" >/dev/null 2>&1 || true pkill -u "$user" 2>/dev/null || true userdel -r "$user" >/dev/null 2>&1 || warn "The user $user couldn't be removed (userdel -r $user)."