From 6493ba3b3d5feea572a7a693911c2cce9530408b Mon Sep 17 00:00:00 2001 From: InayaYousfi Date: Sun, 9 Aug 2026 18:01:46 +0200 Subject: [PATCH 1/7] feat(server): t3 service runs T3 Code in the background on Windows The background service was Linux-only, because it installs a systemd user unit. Windows gets a shortcut in the per-user Startup folder instead. The shortcut runs PowerShell hidden, PowerShell starts the launcher with no console window, then exits. The systemd path is untouched. Every addition to the shared launcher is gated on a flag the generated logon script sets and systemd never does. Three gaps had to be covered because Windows has no init system: - Nothing supervises a Startup folder entry, so the launcher restarts its own child on the same terms as RestartSec and StartLimitBurst. - Nothing captures the launcher's output, so the logon script redirects it through cmd.exe. - Windows has no SIGTERM, so stopping goes through a request file the launcher watches. A pid file is what tells the CLI whether a launcher is running, so an install can never write over one it could not confirm dead. Stopping is not graceful on Windows: the child is terminated without its shutdown finalizer. That, and the fact that the service starts at sign-in rather than boot and stops at sign-out, are documented. Install refuses when a path contains a percent sign, which cmd.exe would expand and silently break, and when the Startup entry is switched off in Windows Settings. --- apps/server/src/cli/service.test.ts | 16 +- apps/server/src/cli/service.ts | 77 +- apps/server/src/cloud/bootService.ts | 18 +- .../src/cloud/bootServiceWindows.test.ts | 326 +++++++++ apps/server/src/cloud/bootServiceWindows.ts | 664 ++++++++++++++++++ apps/server/src/cloud/serviceProtocol.ts | 14 + apps/server/src/serviceLauncher.test.ts | 137 ++++ apps/server/src/serviceLauncher.ts | 184 ++++- docs/user/background-service.md | 35 +- 9 files changed, 1442 insertions(+), 29 deletions(-) create mode 100644 apps/server/src/cloud/bootServiceWindows.test.ts create mode 100644 apps/server/src/cloud/bootServiceWindows.ts diff --git a/apps/server/src/cli/service.test.ts b/apps/server/src/cli/service.test.ts index e91e10000b6..e1ca7c51945 100644 --- a/apps/server/src/cli/service.test.ts +++ b/apps/server/src/cli/service.test.ts @@ -6,10 +6,18 @@ const status = { supported: true, installed: true, current: true, + kind: "systemd", unitPath: "/home/me/.config/systemd/user/t3code.service", logPath: "/home/me/.t3/userdata/logs/boot-service.log", } as const; +const windowsStatus = { + ...status, + kind: "win32-startup-shortcut", + unitPath: + "C:\\Users\\me\\AppData\\Roaming\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\T3 Code Server.lnk", +} as const; + it("reports the installed service version and host paths", () => { assert.equal( formatServiceStatus(status, "0.0.29"), @@ -32,6 +40,12 @@ it("gives a direct repair command for a stale service", () => { it("explains service availability without systemd", () => { assert.include( formatServiceStatus({ ...status, supported: false, installed: false }, "0.0.29"), - "Supported on: Linux with systemd", + "Supported on: Linux with systemd, or Windows", ); }); + +it("calls the Windows definition a shortcut, not a unit", () => { + const output = formatServiceStatus(windowsStatus, "0.0.29"); + assert.include(output, ` Shortcut: ${windowsStatus.unitPath}`); + assert.notInclude(output, "Unit:"); +}); diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index d55b270f183..4588bc33f39 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -1,3 +1,4 @@ +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Console from "effect/Console"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -10,12 +11,23 @@ import type * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts"; -export const bootServiceLayer = (config: ServerConfig.ServerConfig["Service"]) => - BootService.layer({ +export const bootServiceLayer = (config: ServerConfig.ServerConfig["Service"]) => { + const input = { baseDir: config.baseDir, logsDir: config.logsDir, cliVersion: packageJson.version, - }).pipe(Layer.provide(ProcessRunner.layer)); + }; + // Windows has no systemd, so it gets its own backend. Loading it lazily keeps + // the Linux path free of any Windows import. + return Layer.unwrap( + Effect.gen(function* () { + const platform = yield* HostProcessPlatform; + if (platform !== "win32") return BootService.layer(input); + const windows = yield* Effect.promise(() => import("../cloud/bootServiceWindows.ts")); + return windows.layer(input); + }), + ).pipe(Layer.provide(ProcessRunner.layer)); +}; export type ServiceReconcileResult = | { @@ -48,7 +60,7 @@ export function formatServiceStatus( cliVersion: string, ): string { if (!status.supported) { - return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd"; + return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd, or Windows"; } if (!status.installed) { return "T3 Code service\n Status: not installed\n Next: Run `t3 service install`."; @@ -56,7 +68,7 @@ export function formatServiceStatus( return [ "T3 Code service", ` Status: ${status.current ? `installed ยท t3@${cliVersion}` : "needs an update or repair"}`, - ` Unit: ${status.unitPath}`, + ` ${status.kind === "systemd" ? "Unit" : "Shortcut"}: ${status.unitPath}`, ` Logs: ${status.logPath}`, ...(status.current ? [] : [" Next: Run `npx t3@latest service update`."]), ].join("\n"); @@ -71,6 +83,30 @@ const runServiceCommand = Effect.fn("cli.service.run")(function* ( return yield* run.pipe(Effect.provide(bootServiceLayer(config))); }); +/** Windows only. The blink at sign-in looks alarming until you know what it is. */ +const WINDOWS_SERVICE_NOTICE = [ + 'A small window named "T3 Code Server" blinks once when you sign in. That is expected.', + "It appears under Startup apps in Windows Settings, where you can switch it off.", + "It starts at sign-in, not at boot, and it stops when you sign out.", +].join("\n"); + +/** Shared by install and update. The Linux output is unchanged. */ +const reportReconcileResult = Effect.fn("cli.service.report")(function* ( + result: ServiceReconcileResult, + unchangedMessage: string, +) { + if (!result.changed) { + yield* Console.log(unchangedMessage); + return; + } + yield* Console.log( + `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`, + ); + if ((yield* HostProcessPlatform) === "win32") { + yield* Console.log(WINDOWS_SERVICE_NOTICE); + } +}); + const serviceInstallCommand = Command.make("install", projectLocationFlags).pipe( Command.withDescription("Install T3 Code as a background service for this user."), Command.withHandler((flags) => @@ -78,14 +114,9 @@ const serviceInstallCommand = Command.make("install", projectLocationFlags).pipe flags, Effect.gen(function* () { const result = yield* reconcileService(); - if (!result.changed) { - yield* Console.log( - `T3 Code service is already installed with t3@${packageJson.version}.`, - ); - return; - } - yield* Console.log( - `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`, + yield* reportReconcileResult( + result, + `T3 Code service is already installed with t3@${packageJson.version}.`, ); }), ), @@ -101,12 +132,9 @@ const serviceUpdateCommand = Command.make("update", projectLocationFlags).pipe( flags, Effect.gen(function* () { const result = yield* reconcileService(); - if (!result.changed) { - yield* Console.log(`T3 Code service is already using t3@${packageJson.version}.`); - return; - } - yield* Console.log( - `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`, + yield* reportReconcileResult( + result, + `T3 Code service is already using t3@${packageJson.version}.`, ); }), ), @@ -152,12 +180,18 @@ export const offerServiceDuringOnboarding = Effect.gen(function* () { yield* Console.log("T3 Code is already set up to run in the background on this machine."); return true; } + // Windows starts the service at sign-in, not at boot, and it stops at + // sign-out. Promising otherwise here would be a lie. + const windows = (yield* HostProcessPlatform) === "win32"; const wanted = yield* Prompt.run( Prompt.confirm({ message: installed ? "The installed T3 Code service needs an update or repair. Update it now?" - : "Run T3 Code in the background whenever this machine boots? " + - "It stays reachable through T3 Connect even after you log out.", + : windows + ? "Run T3 Code in the background whenever you sign in to Windows? " + + "It starts again after every reboot, and stops when you sign out." + : "Run T3 Code in the background whenever this machine boots? " + + "It stays reachable through T3 Connect even after you log out.", initial: true, }), ); @@ -169,6 +203,7 @@ export const offerServiceDuringOnboarding = Effect.gen(function* () { yield* Console.log( `Background service ${result.previouslyInstalled ? "updated" : "installed"}. Logs: ${result.plan.logPath}`, ); + if (windows) yield* Console.log(WINDOWS_SERVICE_NOTICE); } return true; }); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 7eef6feba50..72af5c10d7b 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -88,7 +88,7 @@ export class BootServiceUnsupportedError extends Schema.TaggedErrorClass new BootServiceInstallError({ cause })), diff --git a/apps/server/src/cloud/bootServiceWindows.test.ts b/apps/server/src/cloud/bootServiceWindows.test.ts new file mode 100644 index 00000000000..eb56920af94 --- /dev/null +++ b/apps/server/src/cloud/bootServiceWindows.test.ts @@ -0,0 +1,326 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { + HostProcessArguments, + HostProcessExecutablePath, + HostProcessPlatform, +} from "@t3tools/shared/hostProcess"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +import * as ProcessRunner from "../processRunner.ts"; +import * as BootServiceWindows from "./bootServiceWindows.ts"; +import { pinnedRuntimePaths } from "./pinnedRuntime.ts"; +import { + parseServiceState, + SERVICE_LAUNCHER_PROTOCOL, + SERVICE_PID_FILE, +} from "./serviceProtocol.ts"; + +const plan = { + nodePath: "C:\\Program Files\\nodejs\\node.exe", + launcherPath: "C:\\Users\\me\\.t3\\runtime\\service-launcher.mjs", + baseDir: "C:\\Users\\me\\.t3", + logPath: "C:\\Users\\me\\.t3\\userdata\\logs\\boot-service.log", + unitPath: + "C:\\Users\\me\\AppData\\Roaming\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\T3 Code Server.lnk", + startupScriptPath: "C:\\Users\\me\\.t3\\runtime\\service-startup.ps1", + shortcutScriptPath: "C:\\Users\\me\\.t3\\runtime\\service-shortcut.ps1", +} as const; + +it("escapes a quote in a PowerShell literal by doubling it", () => { + expect(BootServiceWindows.quotePowerShellLiteral("C:\\Users\\o'brien\\.t3")).toBe( + "'C:\\Users\\o''brien\\.t3'", + ); +}); + +it("starts the launcher with no window and redirects both streams to the log", () => { + const script = BootServiceWindows.renderStartupScript(plan); + + expect(script).toContain("$startInfo.CreateNoWindow = $true"); + expect(script).toContain("$startInfo.UseShellExecute = $false"); + // Doubling the outer quote is what carries a fully quoted command through cmd. + expect(script).toContain( + `/c ""${plan.nodePath}" "${plan.launcherPath}" >> "${plan.logPath}" 2>&1"`, + ); +}); + +it("tells the launcher to supervise itself, because nothing else will", () => { + expect(BootServiceWindows.renderStartupScript(plan)).toContain( + "$env:T3_SERVICE_SELF_SUPERVISE = '1'", + ); +}); + +it("names the window so the blink at sign-in does not look like malware", () => { + const script = BootServiceWindows.renderStartupScript(plan); + const shortcut = BootServiceWindows.renderShortcutScript(plan); + + expect(script).toContain("$Host.UI.RawUI.WindowTitle = 'T3 Code Server'"); + expect(shortcut).toContain("$shortcut.Description = 'T3 Code Server, started at sign-in'"); +}); + +it("points the shortcut at a hidden PowerShell running the startup script", () => { + const shortcut = BootServiceWindows.renderShortcutScript(plan); + + expect(shortcut).toContain("System32\\WindowsPowerShell\\v1.0\\powershell.exe"); + expect(shortcut).toContain(`-WindowStyle Hidden -File "${plan.startupScriptPath}"`); + expect(shortcut).toContain("$shortcut.WindowStyle = 7"); +}); + +it("reads both findings out of the probe output", () => { + expect(BootServiceWindows.parseProbeOutput("shell=True\r\ndisabled=False\r\n")).toEqual({ + shellRunning: true, + entryDisabled: false, + }); + expect(BootServiceWindows.parseProbeOutput("shell=False\ndisabled=True\n")).toEqual({ + shellRunning: false, + entryDisabled: true, + }); +}); + +it("refuses to guess when the probe output is unreadable", () => { + // Defaulting to false would read as "the entry is not disabled", so a probe + // that never ran would look exactly like one that passed. + expect(BootServiceWindows.parseProbeOutput("")).toBeUndefined(); + expect(BootServiceWindows.parseProbeOutput("shell=True\n")).toBeUndefined(); +}); + +it("spots a percent sign, which the command shell would rewrite silently", () => { + expect( + BootServiceWindows.findPercentInPaths([ + ["the Node executable", "C:\\node.exe"], + ["the data directory", "C:\\pct %TEMP% dir\\launcher.mjs"], + ]), + ).toBe("the data directory"); + expect( + BootServiceWindows.findPercentInPaths([["the Node executable", "C:\\node.exe"]]), + ).toBeUndefined(); +}); + +const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function* ( + platform: NodeJS.Platform = "win32", +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boot-service-win-test-" }); + const appData = path.join(home, "AppData", "Roaming"); + const baseDir = path.join(home, ".t3"); + const sourceLauncher = path.join(home, "service-launcher.mjs"); + const runtimeDir = path.join(baseDir, "runtime"); + const statePath = path.join(runtimeDir, "service-state.json"); + const startupScriptPath = path.join(runtimeDir, "service-startup.ps1"); + const shortcutPath = path.join( + appData, + "Microsoft", + "Windows", + "Start Menu", + "Programs", + "Startup", + BootServiceWindows.SHORTCUT_FILE, + ); + yield* fs.writeFileString(sourceLauncher, "export {};\n"); + const runtime = pinnedRuntimePaths(path, baseDir, "1.2.3"); + yield* fs.makeDirectory(path.dirname(runtime.entryPath), { recursive: true }); + yield* fs.writeFileString(runtime.entryPath, "export {};\n"); + yield* fs.writeFileString(runtime.sentinelPath, "1.2.3\n"); + + const pidPath = path.join(runtimeDir, SERVICE_PID_FILE); + const stopRequestPath = path.join(runtimeDir, ".service-stop-request"); + const commands: string[] = []; + const control = { + shell: "True", + disabled: "False", + failCommand: undefined as string | undefined, + /** Set to false to model a launcher that never answers the stop request. */ + launcherStopsOnRequest: true, + }; + // The fake stands in for PowerShell: creating and removing the shortcut is a + // plain file write here, which is all the module observes. + const runner = ProcessRunner.ProcessRunner.of({ + run: (input) => + Effect.gen(function* () { + const command = `${input.command} ${input.args.join(" ")}`; + commands.push(command); + const action = input.args.at(-1); + if (command.includes("service-startup")) { + // Standing in for the logon script: a launcher starts and claims the + // pid file, exactly as the real one does before anything else. + yield* fs.makeDirectory(runtimeDir, { recursive: true }); + yield* fs.writeFileString(pidPath, `${process.pid}\n`); + } + if (command.includes("service-shortcut")) { + if (action === "Install") { + yield* fs.makeDirectory(path.dirname(shortcutPath), { recursive: true }); + yield* fs.writeFileString(shortcutPath, "shortcut"); + } + if (action === "Remove") { + yield* fs.remove(shortcutPath, { force: true }); + } + } + return { + stdout: + action === "Probe" + ? `shell=${control.shell}\ndisabled=${control.disabled}\n` + : input.args[1] === "--version" + ? "t3 v1.2.3\n" + : "", + stderr: "", + code: ChildProcessSpawner.ExitCode(command === control.failCommand ? 1 : 0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }; + }).pipe(Effect.orDie), + }); + + const service = yield* BootServiceWindows.make({ + baseDir, + logsDir: path.join(baseDir, "userdata", "logs"), + cliVersion: "1.2.3", + host: { execPath: "C:\\node.exe", launcherSourcePath: sourceLauncher }, + stopRequestTimeout: Duration.millis(30), + }).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.provide( + Layer.mergeAll( + Layer.succeed(HostProcessPlatform, platform), + Layer.succeed(HostProcessExecutablePath, "C:\\node.exe"), + Layer.succeed(HostProcessArguments, ["C:\\node.exe", path.join(home, "bin.mjs")]), + ConfigProvider.layer(ConfigProvider.fromEnv({ env: { APPDATA: appData } })), + ), + ), + ); + // Stands in for a running launcher. The real one drops its pid file when it + // stops, and that disappearance is the only thing the CLI accepts as proof. + yield* Effect.forkScoped( + Effect.gen(function* () { + while (true) { + yield* Effect.sleep(Duration.millis(5)); + if (!control.launcherStopsOnRequest) continue; + const asked = yield* fs.exists(stopRequestPath).pipe(Effect.orElseSucceed(() => false)); + if (!asked) continue; + yield* fs.remove(stopRequestPath, { force: true }).pipe(Effect.ignore); + yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore); + } + }), + ); + + return { + service, + fs, + statePath, + startupScriptPath, + shortcutPath, + pidPath, + commands, + control, + }; +}); + +it.layer(NodeServices.layer)("windows boot service", (it) => { + it.effect("installs, reports current state, and uninstalls", () => + Effect.gen(function* () { + const { service, fs, statePath, shortcutPath } = yield* makeHarness(); + const installed = yield* service.install; + + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + }); + expect(yield* fs.readFileString(installed.launcherPath)).toBe("export {};\n"); + expect(yield* fs.exists(shortcutPath)).toBe(true); + expect((yield* service.status).current).toBe(true); + expect((yield* service.status).kind).toBe("win32-startup-shortcut"); + + expect(yield* service.uninstall).toBe(true); + expect((yield* service.status).installed).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("creates the shortcut only after the Startup folder passes its probe", () => + Effect.gen(function* () { + const { service, commands } = yield* makeHarness(); + yield* service.install; + + const actions = commands + .filter((command) => command.includes("service-shortcut.ps1")) + .map((command) => command.split(" ").at(-1)); + expect(actions).toEqual(["Probe", "Install"]); + }).pipe(TestClock.withLive), + ); + + it.effect("goes stale when the startup script drifts from what it should be", () => + Effect.gen(function* () { + const { service, fs, startupScriptPath } = yield* makeHarness(); + yield* service.install; + expect((yield* service.status).current).toBe(true); + + yield* fs.writeFileString(startupScriptPath, "# edited by hand\n"); + expect((yield* service.status).current).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("refuses to install over an entry switched off in Windows Settings", () => + Effect.gen(function* () { + const { service, control } = yield* makeHarness(); + control.disabled = "True"; + + const error = yield* service.install.pipe(Effect.flip); + expect(error._tag).toBe("BootServiceInstallError"); + expect(String(error.cause)).toContain("Startup apps"); + }).pipe(TestClock.withLive), + ); + + it.effect("reinstalls over a running launcher only after it has actually gone", () => + Effect.gen(function* () { + const { service, fs, pidPath } = yield* makeHarness(); + yield* service.install; + // The stand-in launcher claimed the pid file when the startup script ran. + expect(yield* fs.exists(pidPath)).toBe(true); + + yield* service.install; + expect((yield* service.status).current).toBe(true); + }).pipe(TestClock.withLive), + ); + + it.effect("refuses to reinstall when the running launcher will not stop", () => + Effect.gen(function* () { + const { service, control } = yield* makeHarness(); + yield* service.install; + // Writing over a launcher we cannot confirm dead would leave two servers + // on one database, so this must fail rather than carry on. + control.launcherStopsOnRequest = false; + + const error = yield* service.install.pipe(Effect.flip); + expect(error._tag).toBe("BootServiceCommandError"); + expect(error.message).toContain("did not exit"); + }).pipe(TestClock.withLive), + ); + + it.effect("removes the shortcut without needing the generated script", () => + Effect.gen(function* () { + const { service, fs, shortcutPath, startupScriptPath } = yield* makeHarness(); + yield* service.install; + // A user who cleared the runtime directory still needs uninstall to work, + // because the shortcut is the thing they asked to be rid of. + yield* fs.remove(startupScriptPath, { force: true }); + + expect(yield* service.uninstall).toBe(true); + expect(yield* fs.exists(shortcutPath)).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("fails closed off Windows", () => + Effect.gen(function* () { + const { service } = yield* makeHarness("linux"); + expect((yield* service.status).supported).toBe(false); + expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUnsupportedError"); + }).pipe(TestClock.withLive), + ); +}); diff --git a/apps/server/src/cloud/bootServiceWindows.ts b/apps/server/src/cloud/bootServiceWindows.ts new file mode 100644 index 00000000000..bd6d562acc2 --- /dev/null +++ b/apps/server/src/cloud/bootServiceWindows.ts @@ -0,0 +1,664 @@ +/** + * Windows boot service. Everything in this file is the Windows path. + * + * Windows has no systemd, so the equivalent of the user unit is a shortcut in + * the per-user Startup folder. The shell (`explorer.exe`) runs that shortcut at + * every sign-in. The shortcut points at PowerShell, PowerShell spawns the + * launcher with its console hidden, then PowerShell exits. + * + * Three differences from the Linux path are deliberate and load bearing: + * + * - Nothing supervises the launcher, so the launcher restarts its own child. + * The generated logon script sets `SERVICE_SELF_SUPERVISE_ENV` to say so. + * - Nothing captures the launcher's output, so the logon script redirects it + * through the command shell (`cmd.exe`) rather than the launcher opening it. + * - Windows has no SIGTERM, so stopping goes through a request file that the + * launcher watches. That stop is not graceful: the child is terminated + * without running its shutdown finalizer. + */ +import { HostProcessExecutablePath, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Config from "effect/Config"; +import * as Console from "effect/Console"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; + +import * as ProcessRunner from "../processRunner.ts"; +import { + BootService, + BootServiceCommandError, + BootServiceInstallError, + BootServiceUnsupportedError, + BootServiceUpdatePendingError, + type BootServiceHost, + type BootServicePlan, +} from "./bootService.ts"; +import { + ensurePinnedRuntimeInstalled, + pinnedRuntimePaths, + PinnedRuntimeInstallError, +} from "./pinnedRuntime.ts"; +import { + SERVICE_LAUNCHER_FILE, + SERVICE_LAUNCHER_PROTOCOL, + SERVICE_PID_FILE, + SERVICE_STATE_FILE, + SERVICE_STOP_REQUEST_FILE, + parseServiceState, + serviceStateHasPendingUpdate, + type ServiceState, +} from "./serviceProtocol.ts"; + +/** Windows only. The shortcut basename doubles as the title of the window that + blinks once at sign-in, because a console launched from a shortcut takes its + title from the shortcut. "Server" rather than "Code" because the desktop app + is a separate thing the user may also start. */ +export const SHORTCUT_NAME = "T3 Code Server"; +export const SHORTCUT_FILE = `${SHORTCUT_NAME}.lnk`; +const STARTUP_SCRIPT_FILE = "service-startup.ps1"; +const SHORTCUT_SCRIPT_FILE = "service-shortcut.ps1"; + +/** + * Windows only. How long the CLI waits for a running launcher to shut down. + * + * It must comfortably exceed the launcher's own worst case. That is a restart + * backoff (5s, now interruptible) plus one watch interval (2s), so 30s leaves + * room for a slow database backup holding the launcher's transition queue. + */ +const STOP_REQUEST_TIMEOUT = Duration.seconds(30); +/** Windows only. How often the CLI re-checks whether the launcher has gone. */ +const STOP_REQUEST_ACK_POLL = Duration.millis(250); +const POWERSHELL_TIMEOUT = Duration.seconds(30); + +/** + * Windows only. The absolute interpreter path, rather than trusting PATH. + * The shortcut has to spell it out anyway, so every call site uses the same one. + */ +const powershellPath = (systemRoot: string) => + `${systemRoot}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`; + +/** + * Windows only. The command shell expands `%VAR%` on its command line, even + * inside double quotes, and there is no escape that works there. A path holding + * a percent sign would therefore be rewritten before the launcher ever starts, + * silently, with a successful install and nothing running. The Linux renderer + * guards the same hazard for systemd in `escapeSystemdSpecifiers`; here the only + * honest option is to refuse. + */ +export function findPercentInPaths( + paths: ReadonlyArray, +): string | undefined { + const offender = paths.find(([, value]) => value.includes("%")); + return offender === undefined ? undefined : offender[0]; +} + +/** + * Windows only. Signal 0 asks the kernel whether a process exists without + * touching it. Node implements it on Windows too. A permission error means the + * process is there but not ours, which still counts as running. + */ +const processIsAlive = (pid: number): Effect.Effect => + Effect.sync(() => { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException | undefined)?.code === "EPERM"; + } + }); + +/** Windows only. PowerShell single-quoted strings escape a quote by doubling it. */ +export function quotePowerShellLiteral(value: string): string { + return `'${value.replaceAll("'", "''")}'`; +} + +export interface WindowsBootServicePlan extends BootServicePlan { + readonly startupScriptPath: string; + readonly shortcutScriptPath: string; +} + +/** Windows only. What the preflight learned about this machine. */ +export interface WindowsPreflight { + readonly shellRunning: boolean; + readonly entryDisabled: boolean; +} + +/** + * Windows only. Pure renderer for the script the shortcut runs at sign-in. + * + * The command shell does the redirect because PowerShell cannot append, and + * cannot send both streams to the same file. Doubling the outer quote is the + * documented way to pass a fully quoted command line through `cmd /c`. + */ +export function renderStartupScript(plan: WindowsBootServicePlan): string { + const command = `""${plan.nodePath}" "${plan.launcherPath}" >> "${plan.logPath}" 2>&1"`; + return [ + "# Generated by `t3 service install`. Do not edit; it is rewritten on update.", + "$ErrorActionPreference = 'Stop'", + "# Belt and braces. The blink is usually titled from the shortcut name.", + `try { $Host.UI.RawUI.WindowTitle = ${quotePowerShellLiteral(SHORTCUT_NAME)} } catch { }`, + `$env:T3CODE_HOME = ${quotePowerShellLiteral(plan.baseDir)}`, + "# Nothing supervises a Startup folder entry, so the launcher supervises itself.", + "$env:T3_SERVICE_SELF_SUPERVISE = '1'", + `$logDir = Split-Path -Parent ${quotePowerShellLiteral(plan.logPath)}`, + "if (-not (Test-Path -LiteralPath $logDir)) {", + " New-Item -ItemType Directory -Path $logDir -Force | Out-Null", + "}", + "$startInfo = New-Object System.Diagnostics.ProcessStartInfo", + "$startInfo.FileName = $env:ComSpec", + `$startInfo.Arguments = ${quotePowerShellLiteral(`/c ${command}`)}`, + "$startInfo.UseShellExecute = $false", + "# The whole point: the launcher runs with no console window at all.", + "$startInfo.CreateNoWindow = $true", + `$startInfo.WorkingDirectory = ${quotePowerShellLiteral(plan.baseDir)}`, + "[System.Diagnostics.Process]::Start($startInfo) | Out-Null", + "", + ].join("\n"); +} + +/** + * Windows only. Pure renderer for the script that probes, creates and removes + * the shortcut. One script owns every shortcut interaction so there is a single + * place to read when the Startup entry misbehaves. + */ +export function renderShortcutScript(plan: WindowsBootServicePlan): string { + return [ + "# Generated by `t3 service install`. Do not edit; it is rewritten on update.", + "param([Parameter(Mandatory = $true)][ValidateSet('Probe', 'Install')][string]$Action)", + "$ErrorActionPreference = 'Stop'", + `$shortcutPath = ${quotePowerShellLiteral(plan.unitPath)}`, + `$shortcutFile = ${quotePowerShellLiteral(SHORTCUT_FILE)}`, + "", + "if ($Action -eq 'Probe') {", + " # The shell is what runs Startup folder entries. A different shell is a", + " # deliberate choice, so this is reported and not treated as a failure.", + " $shell = @(Get-Process -Name 'explorer' -ErrorAction SilentlyContinue).Count -gt 0", + " # Windows records entries disabled through Settings here. The low bit of", + " # the first byte is the disabled flag.", + " $key = 'HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\StartupApproved\\StartupFolder'", + " $disabled = $false", + " if (Test-Path -LiteralPath $key) {", + " $value = (Get-ItemProperty -LiteralPath $key -Name $shortcutFile -ErrorAction SilentlyContinue).$shortcutFile", + " if ($null -ne $value -and $value.Length -gt 0) { $disabled = ($value[0] -band 1) -eq 1 }", + " }", + ' Write-Output "shell=$shell"', + ' Write-Output "disabled=$disabled"', + " exit 0", + "}", + "", + "$startupDir = Split-Path -Parent $shortcutPath", + "if (-not (Test-Path -LiteralPath $startupDir)) {", + " New-Item -ItemType Directory -Path $startupDir -Force | Out-Null", + "}", + "$shell = New-Object -ComObject WScript.Shell", + "$shortcut = $shell.CreateShortcut($shortcutPath)", + "$shortcut.TargetPath = Join-Path $env:SystemRoot 'System32\\WindowsPowerShell\\v1.0\\powershell.exe'", + `$shortcut.Arguments = ${quotePowerShellLiteral( + `-NoLogo -NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden -File "${plan.startupScriptPath}"`, + )}`, + "$shortcut.WorkingDirectory = $env:USERPROFILE", + `$shortcut.Description = ${quotePowerShellLiteral(`${SHORTCUT_NAME}, started at sign-in`)}`, + "# Minimized, because PowerShell paints a console before it hides itself.", + "$shortcut.WindowStyle = 7", + "$shortcut.Save()", + "", + ].join("\n"); +} + +/** + * Windows only. Parses the two lines the Probe action writes. + * + * Returns undefined when either line is missing. Defaulting to false would read + * as "the entry is not disabled", so a probe that failed to run would look + * exactly like a probe that passed, and the install would sail past a check it + * never actually performed. + */ +export function parseProbeOutput(stdout: string): WindowsPreflight | undefined { + const read = (key: string) => { + const match = new RegExp(`^${key}=(True|False)\\s*$`, "im").exec(stdout)?.[1]; + return match === undefined ? undefined : match === "True"; + }; + const shellRunning = read("shell"); + const entryDisabled = read("disabled"); + return shellRunning === undefined || entryDisabled === undefined + ? undefined + : { shellRunning, entryDisabled }; +} + +export interface WindowsBootServiceInput { + readonly baseDir: string; + readonly logsDir: string; + readonly cliVersion: string; + readonly host?: BootServiceHost; + /** Overridable so tests do not sit through the real wait. */ + readonly stopRequestTimeout?: Duration.Duration; +} + +export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( + input: WindowsBootServiceInput, +) { + const hostExecPath = yield* HostProcessExecutablePath; + const platform = yield* HostProcessPlatform; + const appData = yield* Config.string("APPDATA").pipe(Config.withDefault("")); + const systemRoot = yield* Config.string("SystemRoot").pipe(Config.withDefault("C:\\Windows")); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const runner = yield* ProcessRunner.ProcessRunner; + const host = input.host ?? { execPath: hostExecPath }; + + const startupDir = path.join( + appData, + "Microsoft", + "Windows", + "Start Menu", + "Programs", + "Startup", + ); + const unitPath = path.join(startupDir, SHORTCUT_FILE); + const logPath = path.join(input.logsDir, "boot-service.log"); + const runtimeDir = path.join(input.baseDir, "runtime"); + const launcherPath = path.join(runtimeDir, SERVICE_LAUNCHER_FILE); + const statePath = path.join(runtimeDir, SERVICE_STATE_FILE); + const stopRequestPath = path.join(runtimeDir, SERVICE_STOP_REQUEST_FILE); + const pidPath = path.join(runtimeDir, SERVICE_PID_FILE); + const startupScriptPath = path.join(runtimeDir, STARTUP_SCRIPT_FILE); + const shortcutScriptPath = path.join(runtimeDir, SHORTCUT_SCRIPT_FILE); + const runtimePaths = pinnedRuntimePaths(path, input.baseDir, input.cliVersion); + const launcherSourcePath = + host.launcherSourcePath ?? + path.join(path.dirname(runtimePaths.entryPath), SERVICE_LAUNCHER_FILE); + + const plan: WindowsBootServicePlan = { + nodePath: host.execPath, + launcherPath, + baseDir: input.baseDir, + logPath, + unitPath, + startupScriptPath, + shortcutScriptPath, + }; + + /** + * Windows only. A copy of the Linux durable write, minus its final directory + * sync. Windows cannot open a directory as a file, so that sync always fails + * here. Every other step is identical on purpose. + */ + const writeDurably = (filePath: string, contents: string) => + Effect.scoped( + Effect.gen(function* () { + const directory = path.dirname(filePath); + yield* fs.makeDirectory(directory, { recursive: true }); + const tempPath = yield* fs.makeTempFileScoped({ directory, prefix: ".service-write-" }); + yield* fs.writeFileString(tempPath, contents, { mode: 0o600 }); + yield* (yield* fs.open(tempPath, { flag: "r" })).sync; + yield* fs.rename(tempPath, filePath); + }), + ).pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + + const requireWindows = Effect.gen(function* () { + if (platform !== "win32" || appData === "") { + return yield* new BootServiceUnsupportedError({ platform }); + } + }); + + /** Windows only. A copy of the Linux step runner, duplicated for the same + reason as the pinned-runtime step below: extracting it would restructure + the systemd path. Fix bugs here and in bootService.ts together. */ + const runStep = Effect.fn("cloud.boot_service_windows.run_step")(function* ( + step: string, + command: string, + args: ReadonlyArray, + options?: { readonly timeout?: Duration.Input }, + ) { + return yield* runner.run({ command, args, timeout: options?.timeout }).pipe( + Effect.mapError((cause) => new BootServiceCommandError({ step, cause })), + Effect.filterOrFail( + (result) => result.code === 0, + (result) => + new BootServiceCommandError({ + step, + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ), + Effect.tapError((error) => + DateTime.now.pipe( + Effect.flatMap((now) => + fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, { + flag: "a", + }), + ), + Effect.ignore, + ), + ), + ); + }); + + const powershell = powershellPath(systemRoot); + + const runPowerShellScript = ( + step: string, + scriptPath: string, + extraArgs: ReadonlyArray = [], + ) => + runStep( + step, + powershell, + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + scriptPath, + ...extraArgs, + ], + { timeout: POWERSHELL_TIMEOUT }, + ); + + const runShortcutScript = (step: string, action: "Probe" | "Install") => + runPowerShellScript(step, shortcutScriptPath, ["-Action", action]); + + /** + * Windows only. Proves the Startup folder is usable and reports what would + * silently stop the entry from ever running. The shortcut script must already + * be on disk, which is safe: it lives under the data directory, not the + * Startup folder, so nothing outside our own tree has been touched yet. + */ + const preflight: Effect.Effect< + WindowsPreflight, + BootServiceUnsupportedError | BootServiceInstallError | BootServiceCommandError + > = Effect.gen(function* () { + yield* requireWindows; + yield* fs + .makeDirectory(startupDir, { recursive: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + // Prove it is writable now, rather than failing halfway through an install. + yield* Effect.scoped( + fs.makeTempFileScoped({ directory: startupDir, prefix: ".t3-service-probe-" }), + ).pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + + const probe = yield* runShortcutScript("checking the Windows Startup folder", "Probe"); + const findings = parseProbeOutput(probe.stdout); + if (findings === undefined) { + return yield* new BootServiceCommandError({ + step: "reading the Windows Startup folder check", + stdoutLength: probe.stdout.length, + stderrLength: probe.stderr.length, + }); + } + return findings; + }); + + /** + * Windows only. Asks a running launcher to stop, because Windows has no + * SIGTERM. The stop is not graceful: the launcher terminates its child + * without letting it run its shutdown finalizer. + * + * The pid file is what makes the answer unambiguous. Its absence means no + * launcher is running, and its disappearance means one stopped. Treating a + * vanished request file as proof instead would let a slow launcher look + * stopped, and the install would then rewrite the runtime underneath it and + * start a second server on the same database. + */ + const requestStop = Effect.gen(function* () { + const recordedPid = yield* fs.readFileString(pidPath).pipe(Effect.option); + if (Option.isNone(recordedPid)) return; + const pid = Number.parseInt(recordedPid.value.trim(), 10); + if (!Number.isInteger(pid) || pid <= 0 || !(yield* processIsAlive(pid))) { + // The launcher died without cleaning up. Nothing to wait for. + yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore); + return; + } + + // A missing runtime directory must not block an uninstall: the shortcut is + // the thing the user asked to remove, and it lives elsewhere. + const requested = yield* fs.writeFileString(stopRequestPath, "").pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + if (!requested) return; + + const timeoutMs = Duration.toMillis(input.stopRequestTimeout ?? STOP_REQUEST_TIMEOUT); + const pollMs = Math.min(timeoutMs, Duration.toMillis(STOP_REQUEST_ACK_POLL)); + const attempts = Math.max(1, Math.ceil(timeoutMs / pollMs)); + for (let attempt = 0; attempt < attempts; attempt += 1) { + yield* Effect.sleep(Duration.millis(pollMs)); + const stillRunning = yield* fs.exists(pidPath).pipe(Effect.orElseSucceed(() => true)); + if (!stillRunning) return; + } + // Refusing here is the whole point. Carrying on would write over a launcher + // we could not confirm dead. + return yield* new BootServiceCommandError({ + step: `stopping the running service (process ${pid} did not exit in ${Math.round(timeoutMs / 1000)}s)`, + }); + }); + + /** Windows only. A copy of the Linux install's pinned-runtime step, which + validates the runtime by asking it for its own version. It is identical, + and it is duplicated rather than extracted because pulling it out of the + Linux `make` would restructure the systemd path. That path is deliberately + left alone. Fix bugs here and in bootService.ts together. */ + const installPinnedRuntime = ensurePinnedRuntimeInstalled({ + baseDir: input.baseDir, + version: input.cliVersion, + fs, + path, + runner, + validate: (runtime) => + runner + .run({ + command: host.execPath, + args: [runtime.entryPath, "--version"], + timeout: Duration.seconds(30), + }) + .pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ step: "verifying the pinned t3 runtime", cause }), + ), + Effect.flatMap((result) => { + const reportedVersion = /\bv(\S+)\s*$/.exec(result.stdout)?.[1]; + return result.code === 0 && reportedVersion === input.cliVersion + ? Effect.void + : Effect.fail( + new PinnedRuntimeInstallError({ + step: "verifying the pinned t3 runtime", + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ); + }), + ), + }).pipe( + Effect.mapError((error) => + error._tag === "PinnedRuntimeInstallError" + ? new BootServiceCommandError({ + step: error.step, + exitCode: error.exitCode, + stdoutLength: error.stdoutLength, + stderrLength: error.stderrLength, + cause: error, + }) + : new BootServiceInstallError({ cause: error }), + ), + ); + + const install: BootService["Service"]["install"] = Effect.gen(function* () { + yield* requireWindows; + yield* fs + .makeDirectory(input.logsDir, { recursive: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + + const percentIn = findPercentInPaths([ + ["the Node executable", plan.nodePath], + ["the data directory", plan.launcherPath], + ["the log directory", plan.logPath], + ]); + if (percentIn !== undefined) { + return yield* new BootServiceInstallError({ + cause: new Error( + `The path to ${percentIn} contains a percent sign, and the Windows command ` + + "shell would rewrite it before the service could start. Move T3 Code to a " + + "path without one, or set T3CODE_HOME to such a path, then run this again.", + ), + }); + } + + // Write the shortcut script first so the preflight has something to run. + // It lives under the data directory, so nothing user-visible changes yet. + yield* writeDurably(shortcutScriptPath, renderShortcutScript(plan)); + const checks = yield* preflight; + if (!checks.shellRunning) { + // Not fatal. Running a different shell is a deliberate choice, and it is + // the user's to make. Saying nothing would leave them with a service that + // silently never starts. + yield* Console.warn( + "Windows Explorer is not running, and it is what starts Startup folder entries.\n" + + "If you use a different shell, T3 Code may never start when you sign in.", + ); + } + if (checks.entryDisabled) { + return yield* new BootServiceInstallError({ + cause: new Error( + `"${SHORTCUT_NAME}" is switched off under Startup apps in Windows Settings. ` + + "Turn it back on, then run this again.", + ), + }); + } + + // Prepare every immutable artifact before stopping a running launcher. + yield* installPinnedRuntime; + const launcherSource = yield* fs + .readFileString(launcherSourcePath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + + const installed = yield* fs + .exists(unitPath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + if (installed) { + yield* requestStop; + } + + yield* Effect.gen(function* () { + if (installed) { + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); + if ( + Option.isSome(previousStateText) && + serviceStateHasPendingUpdate(previousStateText.value) + ) { + return yield* new BootServiceUpdatePendingError(); + } + } + yield* writeDurably(launcherPath, launcherSource); + yield* writeDurably( + statePath, + // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned document. + `${JSON.stringify( + { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: input.cliVersion, + } satisfies ServiceState, + null, + 2, + )}\n`, + ); + yield* writeDurably(startupScriptPath, renderStartupScript(plan)); + + yield* runShortcutScript("creating the Windows Startup shortcut", "Install"); + // Start last. No administrative state write occurs after this succeeds. + yield* runPowerShellScript("starting the service", startupScriptPath); + }).pipe( + Effect.tapError(() => + installed + ? runPowerShellScript( + "restarting the service after a failed update", + startupScriptPath, + ).pipe(Effect.ignore) + : Effect.void, + ), + ); + return plan satisfies BootServicePlan; + }).pipe(Effect.withSpan("cloud.boot_service_windows.install")); + + const uninstall: BootService["Service"]["uninstall"] = Effect.gen(function* () { + yield* requireWindows; + if ( + !(yield* fs + .exists(unitPath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })))) + ) + return false; + + yield* requestStop; + // A shortcut is an ordinary file, so removing it needs no PowerShell. That + // also means a missing or broken generated script cannot strand the entry. + yield* fs + .remove(unitPath, { force: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + yield* fs.remove(startupScriptPath, { force: true }).pipe(Effect.ignore); + yield* fs.remove(shortcutScriptPath, { force: true }).pipe(Effect.ignore); + return true; + }).pipe(Effect.withSpan("cloud.boot_service_windows.uninstall")); + + const status: BootService["Service"]["status"] = Effect.gen(function* () { + const base = { kind: "win32-startup-shortcut", unitPath, logPath } as const; + if (platform !== "win32" || appData === "") { + return { supported: false, installed: false, current: false, ...base }; + } + if (!(yield* fs.exists(unitPath))) { + return { supported: true, installed: false, current: false, ...base }; + } + const [ + startupScript, + shortcutScript, + launcherExists, + runtimeEntryExists, + runtimeSentinel, + stateText, + ] = yield* Effect.all([ + fs.readFileString(startupScriptPath).pipe(Effect.option), + fs.readFileString(shortcutScriptPath).pipe(Effect.option), + fs.exists(launcherPath), + fs.exists(runtimePaths.entryPath), + fs.readFileString(runtimePaths.sentinelPath).pipe(Effect.option), + fs.readFileString(statePath).pipe(Effect.option), + ]); + const state = Option.isSome(stateText) ? parseServiceState(stateText.value) : undefined; + // Duplicated from the Linux status. The runtime and state checks are the + // same, but the Linux copy also compares the systemd unit, so it cannot be + // shared without changing the Linux path. + return { + supported: true, + installed: true, + current: + Option.isSome(startupScript) && + startupScript.value === renderStartupScript(plan) && + // The shortcut script embeds the shortcut path, the interpreter and the + // window style, so comparing it is what catches shortcut drift. + Option.isSome(shortcutScript) && + shortcutScript.value === renderShortcutScript(plan) && + launcherExists && + runtimeEntryExists && + Option.isSome(runtimeSentinel) && + runtimeSentinel.value.trim() === input.cliVersion && + state?.activeVersion === input.cliVersion && + state?.update?.status !== "pending", + ...base, + }; + }).pipe( + Effect.mapError((cause) => new BootServiceInstallError({ cause })), + Effect.withSpan("cloud.boot_service_windows.status"), + ); + + return BootService.of({ install, uninstall, status }); +}); + +export const layer = (input: WindowsBootServiceInput) => Layer.effect(BootService, make(input)); diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 0faf8894837..46ba3ec7efe 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -9,6 +9,20 @@ export const SERVICE_STATE_FILE = "service-state.json"; the child can tell "the service is going away" from "the launcher is about to start my replacement" while a pending update is recorded. */ export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; +/** Windows only. Set by the generated logon script because a Startup folder + shortcut has no supervisor. systemd never sets it, so the Linux launcher + keeps failing fast and letting `Restart=always` bring the whole unit back. */ +export const SERVICE_SELF_SUPERVISE_ENV = "T3_SERVICE_SELF_SUPERVISE"; +/** Windows only. Written by the CLI to ask a self-supervising launcher to stop, + because Windows has no SIGTERM. The launcher deletes it once it has stopped, + which is how the CLI learns the stop finished. */ +export const SERVICE_STOP_REQUEST_FILE = ".service-stop-request"; +/** Windows only. Holds a self-supervising launcher's process id while it runs. + It is what lets the CLI tell "the service stopped" apart from "no service was + running" and from "it is still running and did not answer". Without it, an + install could rewrite the runtime under a live launcher and end up with two + servers on one database. */ +export const SERVICE_PID_FILE = ".service-pid"; export interface PendingServiceUpdate { readonly id: string; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 45c472af1fc..9695cc20393 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -3,6 +3,7 @@ import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; import { Launcher, readServiceState, writeServiceState } from "./serviceLauncher.ts"; import { @@ -11,6 +12,7 @@ import { isExactServiceVersion, SERVICE_LAUNCHER_PROTOCOL, SERVICE_STOP_MARKER_FILE, + SERVICE_STOP_REQUEST_FILE, } from "./cloud/serviceProtocol.ts"; it("accepts only exact semantic versions", () => { @@ -290,3 +292,138 @@ if (context.update?.status === "pending") { }), ); }); + +it.layer(NodeServices.layer)("self-supervising launcher", (it) => { + /** A child that records every start, then exits so the launcher sees a crash. */ + const crashingChild = ` +import { appendFileSync } from "node:fs"; +appendFileSync(process.env.T3_TEST_START_LOG, "start\\n"); +process.exit(1); +`; + + const seedRuntime = Effect.fn("test.seed_launcher_runtime")(function* ( + root: string, + source: string, + ) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, source); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + return statePath; + }); + + it.effect("fails fast when something else supervises it, as on Linux", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-supervised-" }); + const startLog = path.join(root, "starts.log"); + process.env.T3_TEST_START_LOG = startLog; + const statePath = yield* seedRuntime(root, crashingChild); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: false }, + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + // One start, then it hands the problem to the init system by exiting. + assert.equal((yield* fs.readFileString(startLog)).trim().split("\n").length, 1); + }), + ); + + it.effect("restarts its own child, then gives up on the systemd burst limit", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-selfsuper-" }); + const startLog = path.join(root, "starts.log"); + process.env.T3_TEST_START_LOG = startLog; + const statePath = yield* seedRuntime(root, crashingChild); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + // Five starts in total, which is what StartLimitBurst=5 permits: the + // first one plus four restarts. + assert.equal((yield* fs.readFileString(startLog)).trim().split("\n").length, 5); + }), + ); + + it.effect("stops when the CLI writes a stop request, then clears it", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-stopreq-" }); + const statePath = yield* seedRuntime(root, "setInterval(() => {}, 1_000);\n"); + const requestPath = path.join(root, "runtime", SERVICE_STOP_REQUEST_FILE); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + const running = launcher.run(); + yield* fs.writeFileString(requestPath, ""); + yield* Effect.promise(() => running); + + // Deleting it is how the CLI learns the stop actually happened. + assert.isFalse(yield* fs.exists(requestPath)); + assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); + }), + ); + + it.effect("ignores a request left over from before it started", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-stalereq-" }); + const statePath = yield* seedRuntime(root, "setInterval(() => {}, 1_000);\n"); + const requestPath = path.join(root, "runtime", SERVICE_STOP_REQUEST_FILE); + yield* fs.writeFileString(requestPath, ""); + // Backdate it well past the grace window, which is what makes it stale. + // A request written just before startup is deliberately treated as real. + // A fixed date, so this does not depend on the clock the test runs under. + const staleEpochMs = 1_577_836_800_000; // 2020-01-01 + yield* fs.utimes(requestPath, staleEpochMs, staleEpochMs); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + const running = launcher.run(); + yield* Effect.sleep("100 millis"); + // Still running: the stale request was cleared, not obeyed. + assert.isFalse(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); + + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }).pipe(TestClock.withLive), + ); +}); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 3641593ecf0..ad1b12f567a 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -24,14 +24,43 @@ import { parseServiceState, SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_LAUNCHER_PROTOCOL, + SERVICE_PID_FILE, + SERVICE_SELF_SUPERVISE_ENV, SERVICE_STATE_FILE, SERVICE_STOP_MARKER_FILE, + SERVICE_STOP_REQUEST_FILE, } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; const PREPARED_TIMEOUT_MS = 120_000; const TERMINATE_GRACE_MS = 5_000; +/** + * Windows only. A Startup folder shortcut has no supervisor, so the launcher + * does the job the systemd unit does on Linux. These three mirror RestartSec=5, + * StartLimitBurst=5 and StartLimitIntervalSec=300, so a self-supervising + * launcher gives up at exactly the point systemd would. The burst counts the + * first start as well, the way systemd counts it, so five means four restarts. + */ +const RESTART_DELAY_MS = 5_000; +const RESTART_BURST = 5; +const RESTART_WINDOW_MS = 300_000; +/** Windows only. How often the launcher looks for a stop request. Directory + watching misses events, so this poll is the real delivery guarantee. */ +const STOP_REQUEST_WATCH_INTERVAL_MS = 2_000; +/** + * Windows only. A request older than this when the launcher starts was left by + * a stop nobody completed, so obeying it would stop a launcher the request was + * never meant for. The window is generous on purpose: some filesystems report + * a coarse modification time, and wrongly ignoring a real stop is far worse + * than wrongly obeying an old one. Ignoring one lets an install rewrite the + * runtime under a launcher that is still running. + */ +const STOP_REQUEST_GRACE_MS = 5_000; + +/** Windows only. Set by the generated logon script. systemd never sets it. */ +const isSelfSupervising = (): boolean => process.env[SERVICE_SELF_SUPERVISE_ENV] === "1"; + type TerminalStatus = "committed" | "rolled-back" | "failed"; type ChildRole = "active" | "trial"; @@ -262,9 +291,29 @@ async function terminateChild( const stopMarkerPath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); +/** Windows only. Written by the CLI to ask for a stop, because there is no SIGTERM. */ +const stopRequestPath = (baseDir: string) => + NodePath.join(baseDir, "runtime", SERVICE_STOP_REQUEST_FILE); + +/** Windows only. Present exactly while a self-supervising launcher is running. */ +const pidFilePath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_PID_FILE); + +export interface LauncherOptions { + /** + * Windows only. Whether this launcher must restart its own child and watch + * for stop requests. Defaults to the flag the generated logon script sets, + * which systemd never sets. + */ + readonly selfSupervise?: boolean; + /** Overridable so tests do not sit through the real wait. */ + readonly restartDelayMs?: number; +} + export class Launcher { readonly #baseDir: string; readonly #statePath: string; + readonly #selfSupervise: boolean; + readonly #restartDelayMs: number; #state: ServiceState; #child: ManagedChild | null = null; #timer: NodeJS.Timeout | undefined; @@ -272,12 +321,30 @@ export class Launcher { #stopRequested = false; #stopping = false; #done = false; + /** Windows only. Restart times inside the current window. */ + #restartAttempts: Array = []; + /** Windows only. Used to tell a fresh stop request from a leftover one. */ + #startedAt = 0; + /** Windows only. Modification time of a request already judged stale. */ + #dismissedRequestAt: number | undefined; + /** + * Windows only. Recovery clears a stale stop marker, so acting on a request + * before it runs would let recovery wipe the marker the stop just wrote. The + * child would then read "a replacement server is coming" while none is. + */ + #recovered = false; + #stopWatcher: NodeFS.FSWatcher | undefined; + #stopPoll: NodeJS.Timeout | undefined; + /** Windows only. Lets a stop cut a restart backoff short instead of queueing behind it. */ + #restartWait: { readonly timer: NodeJS.Timeout; readonly resolve: () => void } | undefined; readonly #completion = Promise.withResolvers(); - constructor(baseDir: string, state: ServiceState) { + constructor(baseDir: string, state: ServiceState, options?: LauncherOptions) { this.#baseDir = baseDir; this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); this.#state = state; + this.#selfSupervise = options?.selfSupervise ?? isSelfSupervising(); + this.#restartDelayMs = options?.restartDelayMs ?? RESTART_DELAY_MS; } async run(): Promise { @@ -285,15 +352,118 @@ export class Launcher { const onSigint = () => void this.stop("SIGINT"); process.once("SIGTERM", onSigterm); process.once("SIGINT", onSigint); + const supervising = this.#selfSupervise; + this.#startedAt = Date.now(); + // The first start counts toward the burst, the way systemd counts it. + this.#restartAttempts = [this.#startedAt]; try { + if (supervising) { + // Written before anything else, so the CLI can never see a running + // launcher as absent and rewrite the runtime underneath it. + await NodeFSP.mkdir(NodePath.dirname(pidFilePath(this.#baseDir)), { + recursive: true, + mode: 0o700, + }).catch(() => undefined); + await NodeFSP.writeFile(pidFilePath(this.#baseDir), `${process.pid}\n`, { + mode: 0o600, + }).catch(() => undefined); + this.#watchStopRequest(); + } this.#enqueue(() => this.#recover()); + this.#enqueue(async () => { + this.#recovered = true; + }); await this.#completion.promise; } finally { process.off("SIGTERM", onSigterm); process.off("SIGINT", onSigint); + this.#stopWatchingStopRequest(); + if (supervising) { + // Removing these before run() returns is how the CLI learns the stop + // finished. Doing it after would race the process exiting. + await NodeFSP.rm(stopRequestPath(this.#baseDir), { force: true }).catch(() => undefined); + await NodeFSP.rm(pidFilePath(this.#baseDir), { force: true }).catch(() => undefined); + } } } + /** + * Windows only. Watches for the CLI's stop request, because Windows has no + * SIGTERM. The stop it triggers is not graceful: the child is terminated + * without running its shutdown finalizer. + */ + #watchStopRequest(): void { + const target = stopRequestPath(this.#baseDir); + const check = () => { + if (!this.#recovered || this.#stopRequested || this.#stopping) return; + void NodeFSP.stat(target).then( + async (stats) => { + if (this.#stopRequested || this.#stopping) return; + if (this.#dismissedRequestAt === stats.mtimeMs) return; + if (stats.mtimeMs >= this.#startedAt - STOP_REQUEST_GRACE_MS) { + await this.stop("SIGTERM"); + return; + } + // Left behind by a stop nobody completed. Remember it rather than + // deleting it: the CLI reads a vanished pid file as proof that a + // launcher stopped, and deleting files here must never look like that. + this.#dismissedRequestAt = stats.mtimeMs; + }, + () => undefined, + ); + }; + try { + this.#stopWatcher = NodeFS.watch(NodePath.dirname(target), () => check()); + // An error event with no listener is an uncaught exception, and Windows + // emits EPERM here when the watched directory goes away. The poll below + // is the real delivery guarantee, so losing the watcher costs nothing. + this.#stopWatcher.on("error", () => undefined); + } catch { + // Best effort, for the same reason. + } + this.#stopPoll = setInterval(check, STOP_REQUEST_WATCH_INTERVAL_MS); + this.#stopPoll.unref(); + check(); + } + + #stopWatchingStopRequest(): void { + this.#stopWatcher?.close(); + this.#stopWatcher = undefined; + clearInterval(this.#stopPoll); + this.#stopPoll = undefined; + } + + /** + * Windows only. Waits out the restart backoff, but returns early when a stop + * arrives. The wait runs inside a queued transition, so sleeping through it + * would delay the stop by the full delay and eat the CLI's patience. + */ + #waitBeforeRestart(): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.#restartWait = undefined; + resolve(); + }, this.#restartDelayMs); + this.#restartWait = { + timer, + resolve: () => { + clearTimeout(timer); + this.#restartWait = undefined; + resolve(); + }, + }; + }); + } + + /** Windows only. False once the burst limit is reached inside the window. */ + #recordRestartAttempt(): boolean { + const now = Date.now(); + this.#restartAttempts = this.#restartAttempts.filter((at) => now - at < RESTART_WINDOW_MS); + if (this.#restartAttempts.length >= RESTART_BURST) return false; + this.#restartAttempts.push(now); + return true; + } + #enqueue(transition: () => Promise): void { this.#transitions = this.#transitions .then(transition, transition) @@ -328,6 +498,9 @@ export class Launcher { return; } this.#stopRequested = true; + // Windows only. A restart backoff holds the transition queue, so a stop + // queued behind it would wait the full delay before it even started. + this.#restartWait?.resolve(); this.#clearTimer(); this.#enqueue(async () => { // Let an update transition already in progress start its replacement @@ -572,6 +745,15 @@ export class Launcher { await this.#startTrial(pending); return; } + // Windows only. Nothing supervises a Startup folder entry, so the launcher + // does what Restart=always does on Linux, and gives up on the same terms. + // Without the flag this still throws, so the Linux path is unchanged. + if (this.#selfSupervise && this.#recordRestartAttempt()) { + await this.#waitBeforeRestart(); + if (this.#stopping || this.#done || this.#stopRequested) return; + await this.#startChild(this.#state.activeVersion, "active", this.#state.update); + return; + } throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); } diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 299e3b641f1..4252e77da02 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -3,6 +3,9 @@ On a Linux host, T3 Code can run as a background service for your user. It starts when the machine boots and keeps running after you log out. +On Windows it starts when you sign in and stops when you sign out. See +[On Windows](#on-windows) for the full list of differences. + ## Manage the Service Install it with the latest T3 Code release: @@ -46,4 +49,34 @@ out. This is only an onboarding shortcut: the service and T3 Connect are managed Signing out of T3 Connect does not remove the service. Use `t3 service uninstall` when you no longer want T3 Code to start in the background. -The background service currently requires Linux with systemd. +The background service requires Linux with systemd, or Windows. macOS is not supported yet. + +## On Windows + +Windows has no systemd, so `t3 service install` puts a shortcut named `T3 Code Server` in your +personal Startup folder. Windows Explorer runs that shortcut every time you sign in. The shortcut +starts PowerShell, PowerShell starts the T3 Code server with no window, then PowerShell exits. + +It is called `T3 Code Server` rather than `T3 Code` because it only starts the server. The desktop +app is separate, and you start that yourself. + +The same four commands work: `install`, `status`, `update` and `uninstall`. + +Windows differs from Linux in ways worth knowing before you rely on it: + +- **It starts at sign-in, not at boot.** Nothing runs while the machine sits at the sign-in screen. +- **It stops when you sign out.** Windows ends your session's processes, and the server is one. +- **Stopping is not graceful.** `update` and `uninstall` terminate the server rather than asking it + to shut down. In-flight agent work is cut, and the server does not get to release its T3 Connect + link, so the host can look online for a short while after it has gone. +- **A small window blinks once at sign-in.** It is named `T3 Code Server`. That flash is PowerShell + starting, and it is expected. Nothing stays on your taskbar afterwards. +- **Windows Settings can switch it off.** It appears under Startup apps. If you disable it there, + `t3 service status` still reports it as installed, because the shortcut is still on disk, and + `t3 service install` still reports nothing to do. Check Startup apps first if the service stops + coming back after you sign in. An install that does have work to do refuses to write over a + disabled entry and tells you to turn it back on. +- **Windows Explorer must be your shell.** That is the default. If you replaced it, Startup folder + entries may never run, and `t3 service install` warns you about that. + +Let agent work finish before running `update` on Windows, because the stop is not graceful. From dd3bebe56246c04fae2c3850b720f797e7f3005e Mon Sep 17 00:00:00 2001 From: InayaYousfi Date: Sun, 9 Aug 2026 18:15:01 +0200 Subject: [PATCH 2/7] fix(server): close the Windows service review findings Six review findings, all about the same class of failure: something looks successful while a second server ends up on one SQLite database, or while nothing runs at all. - The pid file is claimed exclusively with `wx`, and released only by the process that owns it. A Startup shortcut that fires twice now leaves the second launcher exiting instead of starting a rival server. - Failing to publish the pid file no longer continues silently. A live launcher without one is invisible to the CLI, which reads absence as proof that nothing is running. - A failed stop-request write fails the operation once a live launcher is confirmed, rather than reading as "nothing to stop". - install and uninstall decide whether to stop from the pid file, not from the shortcut. A launcher outlives a shortcut someone deleted by hand. - `status` reads the shortcut back through the Probe action and compares its target and arguments. Comparing the script that wrote the shortcut said nothing about a shortcut edited or replaced since. - Connect onboarding no longer promises Windows users the service survives sign-out, which contradicted the prompt they had just accepted. --- apps/server/src/cli/connect.ts | 11 ++- .../src/cloud/bootServiceWindows.test.ts | 43 ++++++++- apps/server/src/cloud/bootServiceWindows.ts | 96 ++++++++++++------- apps/server/src/serviceLauncher.test.ts | 41 ++++++++ apps/server/src/serviceLauncher.ts | 83 +++++++++++++--- 5 files changed, 223 insertions(+), 51 deletions(-) diff --git a/apps/server/src/cli/connect.ts b/apps/server/src/cli/connect.ts index ef15e650a6f..c0d46ac9f01 100644 --- a/apps/server/src/cli/connect.ts +++ b/apps/server/src/cli/connect.ts @@ -5,6 +5,7 @@ import { type RelayClientInstallProgressStage, } from "@t3tools/contracts"; import { RelayOkResponse } from "@t3tools/contracts/relay"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as RelayClient from "@t3tools/shared/relayClient"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import * as Cause from "effect/Cause"; @@ -694,9 +695,13 @@ export const connectCommand = Command.make("connect", { // fail the command, just tell the user what happened and move on. const background = yield* recoverServiceOnboardingOffer(offerServiceDuringOnboarding); if (background) { - yield* Console.log( - "\nโœ“ Background service ready\n\nT3 Code will stay reachable after you log out.", - ); + // Windows starts the service at sign-in and stops it at sign-out, so + // the Linux promise would contradict the prompt the user just agreed to. + const reach = + (yield* HostProcessPlatform) === "win32" + ? "T3 Code will start again every time you sign in to Windows." + : "T3 Code will stay reachable after you log out."; + yield* Console.log(`\nโœ“ Background service ready\n\n${reach}`); return; } const serveCommand = yield* resolveCliCommand("serve"); diff --git a/apps/server/src/cloud/bootServiceWindows.test.ts b/apps/server/src/cloud/bootServiceWindows.test.ts index eb56920af94..ded28455c47 100644 --- a/apps/server/src/cloud/bootServiceWindows.test.ts +++ b/apps/server/src/cloud/bootServiceWindows.test.ts @@ -133,12 +133,20 @@ const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function const pidPath = path.join(runtimeDir, SERVICE_PID_FILE); const stopRequestPath = path.join(runtimeDir, ".service-stop-request"); const commands: string[] = []; + const powershell = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"; + const expectedArguments = BootServiceWindows.expectedShortcutArguments({ + ...plan, + startupScriptPath, + }); const control = { shell: "True", disabled: "False", failCommand: undefined as string | undefined, /** Set to false to model a launcher that never answers the stop request. */ launcherStopsOnRequest: true, + /** What the fake shortcut currently points at, so drift can be modelled. */ + shortcutTarget: powershell, + shortcutArguments: expectedArguments, }; // The fake stands in for PowerShell: creating and removing the shortcut is a // plain file write here, which is all the module observes. @@ -166,7 +174,10 @@ const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function return { stdout: action === "Probe" - ? `shell=${control.shell}\ndisabled=${control.disabled}\n` + ? `shell=${control.shell}\ndisabled=${control.disabled}\n` + + ((yield* fs.exists(shortcutPath).pipe(Effect.orElseSucceed(() => false))) + ? `target=${control.shortcutTarget}\narguments=${control.shortcutArguments}\n` + : "") : input.args[1] === "--version" ? "t3 v1.2.3\n" : "", @@ -192,7 +203,9 @@ const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function Layer.succeed(HostProcessPlatform, platform), Layer.succeed(HostProcessExecutablePath, "C:\\node.exe"), Layer.succeed(HostProcessArguments, ["C:\\node.exe", path.join(home, "bin.mjs")]), - ConfigProvider.layer(ConfigProvider.fromEnv({ env: { APPDATA: appData } })), + ConfigProvider.layer( + ConfigProvider.fromEnv({ env: { APPDATA: appData, SystemRoot: "C:\\Windows" } }), + ), ), ), ); @@ -316,6 +329,32 @@ it.layer(NodeServices.layer)("windows boot service", (it) => { }).pipe(TestClock.withLive), ); + it.effect("goes stale when the shortcut itself is edited, not just the script", () => + Effect.gen(function* () { + const { service, control } = yield* makeHarness(); + yield* service.install; + expect((yield* service.status).current).toBe(true); + + // The generated scripts still match. Only the .lnk was tampered with, and + // that is the thing sign-in actually runs. + control.shortcutArguments = "-File C:\\somewhere\\else.ps1"; + expect((yield* service.status).current).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("stops a running launcher even when the shortcut was deleted by hand", () => + Effect.gen(function* () { + const { service, fs, shortcutPath, pidPath } = yield* makeHarness(); + yield* service.install; + // A launcher outlives its shortcut, so keying the stop on the shortcut + // would leave a server running with nothing left to manage it. + yield* fs.remove(shortcutPath, { force: true }); + + expect(yield* service.uninstall).toBe(true); + expect(yield* fs.exists(pidPath)).toBe(false); + }).pipe(TestClock.withLive), + ); + it.effect("fails closed off Windows", () => Effect.gen(function* () { const { service } = yield* makeHarness("linux"); diff --git a/apps/server/src/cloud/bootServiceWindows.ts b/apps/server/src/cloud/bootServiceWindows.ts index bd6d562acc2..8943811d744 100644 --- a/apps/server/src/cloud/bootServiceWindows.ts +++ b/apps/server/src/cloud/bootServiceWindows.ts @@ -125,6 +125,14 @@ export interface WindowsBootServicePlan extends BootServicePlan { export interface WindowsPreflight { readonly shellRunning: boolean; readonly entryDisabled: boolean; + /** Absent when no shortcut exists yet. */ + readonly shortcutTarget?: string; + readonly shortcutArguments?: string; +} + +/** Windows only. What the shortcut must point at for the service to work. */ +export function expectedShortcutArguments(plan: WindowsBootServicePlan): string { + return `-NoLogo -NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden -File "${plan.startupScriptPath}"`; } /** @@ -197,9 +205,7 @@ export function renderShortcutScript(plan: WindowsBootServicePlan): string { "$shell = New-Object -ComObject WScript.Shell", "$shortcut = $shell.CreateShortcut($shortcutPath)", "$shortcut.TargetPath = Join-Path $env:SystemRoot 'System32\\WindowsPowerShell\\v1.0\\powershell.exe'", - `$shortcut.Arguments = ${quotePowerShellLiteral( - `-NoLogo -NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden -File "${plan.startupScriptPath}"`, - )}`, + `$shortcut.Arguments = ${quotePowerShellLiteral(expectedShortcutArguments(plan))}`, "$shortcut.WorkingDirectory = $env:USERPROFILE", `$shortcut.Description = ${quotePowerShellLiteral(`${SHORTCUT_NAME}, started at sign-in`)}`, "# Minimized, because PowerShell paints a console before it hides itself.", @@ -222,11 +228,18 @@ export function parseProbeOutput(stdout: string): WindowsPreflight | undefined { const match = new RegExp(`^${key}=(True|False)\\s*$`, "im").exec(stdout)?.[1]; return match === undefined ? undefined : match === "True"; }; + const readText = (key: string) => new RegExp(`^${key}=(.*)$`, "im").exec(stdout)?.[1]?.trim(); const shellRunning = read("shell"); const entryDisabled = read("disabled"); - return shellRunning === undefined || entryDisabled === undefined - ? undefined - : { shellRunning, entryDisabled }; + if (shellRunning === undefined || entryDisabled === undefined) return undefined; + const shortcutTarget = readText("target"); + const shortcutArguments = readText("arguments"); + return { + shellRunning, + entryDisabled, + ...(shortcutTarget === undefined ? {} : { shortcutTarget }), + ...(shortcutArguments === undefined ? {} : { shortcutArguments }), + }; } export interface WindowsBootServiceInput { @@ -409,21 +422,20 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( */ const requestStop = Effect.gen(function* () { const recordedPid = yield* fs.readFileString(pidPath).pipe(Effect.option); - if (Option.isNone(recordedPid)) return; + if (Option.isNone(recordedPid)) return false; const pid = Number.parseInt(recordedPid.value.trim(), 10); if (!Number.isInteger(pid) || pid <= 0 || !(yield* processIsAlive(pid))) { // The launcher died without cleaning up. Nothing to wait for. yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore); - return; + return false; } - // A missing runtime directory must not block an uninstall: the shortcut is - // the thing the user asked to remove, and it lives elsewhere. - const requested = yield* fs.writeFileString(stopRequestPath, "").pipe( - Effect.as(true), - Effect.orElseSucceed(() => false), - ); - if (!requested) return; + // A launcher is confirmed alive, so a failed write must fail the whole + // operation. Swallowing it would let the caller carry on and start a second + // server against the same database. + yield* fs + .writeFileString(stopRequestPath, "") + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); const timeoutMs = Duration.toMillis(input.stopRequestTimeout ?? STOP_REQUEST_TIMEOUT); const pollMs = Math.min(timeoutMs, Duration.toMillis(STOP_REQUEST_ACK_POLL)); @@ -431,7 +443,7 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( for (let attempt = 0; attempt < attempts; attempt += 1) { yield* Effect.sleep(Duration.millis(pollMs)); const stillRunning = yield* fs.exists(pidPath).pipe(Effect.orElseSucceed(() => true)); - if (!stillRunning) return; + if (!stillRunning) return true; } // Refusing here is the whole point. Carrying on would write over a launcher // we could not confirm dead. @@ -543,19 +555,18 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( const installed = yield* fs .exists(unitPath) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - if (installed) { - yield* requestStop; - } + // Keyed on the pid file, never on the shortcut. A launcher outlives a + // manually deleted shortcut, and starting a second one alongside it would + // put two servers on the same database. + const stopped = yield* requestStop; yield* Effect.gen(function* () { - if (installed) { - const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); - if ( - Option.isSome(previousStateText) && - serviceStateHasPendingUpdate(previousStateText.value) - ) { - return yield* new BootServiceUpdatePendingError(); - } + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); + if ( + Option.isSome(previousStateText) && + serviceStateHasPendingUpdate(previousStateText.value) + ) { + return yield* new BootServiceUpdatePendingError(); } yield* writeDurably(launcherPath, launcherSource); yield* writeDurably( @@ -577,7 +588,7 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( yield* runPowerShellScript("starting the service", startupScriptPath); }).pipe( Effect.tapError(() => - installed + installed || stopped ? runPowerShellScript( "restarting the service after a failed update", startupScriptPath, @@ -590,14 +601,16 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( const uninstall: BootService["Service"]["uninstall"] = Effect.gen(function* () { yield* requireWindows; - if ( - !(yield* fs - .exists(unitPath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })))) - ) - return false; + const installed = yield* fs + .exists(unitPath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + + // Stop first, and decide that on the pid file rather than the shortcut. A + // launcher survives someone deleting the shortcut by hand, and leaving it + // running is exactly what the user asked us not to do. + const stopped = yield* requestStop; + if (!installed && !stopped) return false; - yield* requestStop; // A shortcut is an ordinary file, so removing it needs no PowerShell. That // also means a missing or broken generated script cannot strand the entry. yield* fs @@ -632,6 +645,18 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( fs.readFileString(statePath).pipe(Effect.option), ]); const state = Option.isSome(stateText) ? parseServiceState(stateText.value) : undefined; + // Read the shortcut itself back, because the script that wrote it matching + // proves nothing about the .lnk someone may have edited or replaced since. + const shortcut = yield* runShortcutScript( + "checking the Windows Startup shortcut", + "Probe", + ).pipe( + Effect.map((probe) => parseProbeOutput(probe.stdout)), + Effect.orElseSucceed(() => undefined), + ); + const shortcutMatches = + shortcut?.shortcutTarget === powershell && + shortcut.shortcutArguments === expectedShortcutArguments(plan); // Duplicated from the Linux status. The runtime and state checks are the // same, but the Linux copy also compares the systemd unit, so it cannot be // shared without changing the Linux path. @@ -639,6 +664,7 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( supported: true, installed: true, current: + shortcutMatches && Option.isSome(startupScript) && startupScript.value === renderStartupScript(plan) && // The shortcut script embeds the shortcut path, the interpreter and the diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 9695cc20393..cf776ab9721 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -11,6 +11,7 @@ import { decodeServiceState, isExactServiceVersion, SERVICE_LAUNCHER_PROTOCOL, + SERVICE_PID_FILE, SERVICE_STOP_MARKER_FILE, SERVICE_STOP_REQUEST_FILE, } from "./cloud/serviceProtocol.ts"; @@ -427,3 +428,43 @@ process.exit(1); }).pipe(TestClock.withLive), ); }); + +it.layer(NodeServices.layer)("self-supervising pid ownership", (it) => { + it.effect("a second launcher exits rather than running a second server", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-pid-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const state = yield* Effect.promise(() => readServiceState(statePath)); + + const first = new Launcher(root, state, { selfSupervise: true, restartDelayMs: 5 }); + const firstRunning = first.run(); + yield* Effect.sleep("100 millis"); + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + assert.isTrue(yield* fs.exists(pidPath)); + + // The Startup shortcut firing twice must not put two servers on one + // database. The second launcher sees a live owner and gives up. + const second = new Launcher(root, state, { selfSupervise: true, restartDelayMs: 5 }); + yield* Effect.promise(() => second.run()); + // It also must not take the first launcher's pid file with it on the way out. + assert.isTrue(yield* fs.exists(pidPath)); + + yield* Effect.promise(() => first.stop("SIGTERM")); + yield* Effect.promise(() => firstRunning); + assert.isFalse(yield* fs.exists(pidPath)); + }).pipe(TestClock.withLive), + ); +}); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index ad1b12f567a..e38d26e8505 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -61,6 +61,20 @@ const STOP_REQUEST_GRACE_MS = 5_000; /** Windows only. Set by the generated logon script. systemd never sets it. */ const isSelfSupervising = (): boolean => process.env[SERVICE_SELF_SUPERVISE_ENV] === "1"; +/** + * Windows only. Signal 0 asks the kernel whether a process exists without + * touching it. A permission error means it exists but is not ours, which still + * counts as running. + */ +function processIsAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException | undefined)?.code === "EPERM"; + } +} + type TerminalStatus = "committed" | "rolled-back" | "failed"; type ChildRole = "active" | "trial"; @@ -353,20 +367,23 @@ export class Launcher { process.once("SIGTERM", onSigterm); process.once("SIGINT", onSigint); const supervising = this.#selfSupervise; + let owned = false; this.#startedAt = Date.now(); // The first start counts toward the burst, the way systemd counts it. this.#restartAttempts = [this.#startedAt]; try { if (supervising) { - // Written before anything else, so the CLI can never see a running - // launcher as absent and rewrite the runtime underneath it. - await NodeFSP.mkdir(NodePath.dirname(pidFilePath(this.#baseDir)), { - recursive: true, - mode: 0o700, - }).catch(() => undefined); - await NodeFSP.writeFile(pidFilePath(this.#baseDir), `${process.pid}\n`, { - mode: 0o600, - }).catch(() => undefined); + // Claimed before anything else, and never swallowed on failure. The CLI + // reads an absent pid file as proof nothing is running, so a live + // launcher without one would let an install rewrite the runtime + // underneath it and start a second server on the same database. + owned = await this.#claimPidFile(); + if (!owned) { + // Another launcher is already running, which happens when the Startup + // shortcut fires twice. Leave it alone and exit. + process.stderr.write("[service-launcher] another launcher is already running\n"); + return; + } this.#watchStopRequest(); } this.#enqueue(() => this.#recover()); @@ -378,13 +395,57 @@ export class Launcher { process.off("SIGTERM", onSigterm); process.off("SIGINT", onSigint); this.#stopWatchingStopRequest(); - if (supervising) { + if (owned) { // Removing these before run() returns is how the CLI learns the stop // finished. Doing it after would race the process exiting. await NodeFSP.rm(stopRequestPath(this.#baseDir), { force: true }).catch(() => undefined); - await NodeFSP.rm(pidFilePath(this.#baseDir), { force: true }).catch(() => undefined); + await this.#releasePidFile(); + } + } + } + + /** + * Windows only. Claims the pid file exclusively. Returns false when another + * live launcher already owns it, which is what stops a Startup shortcut that + * fires twice from running two servers against one database. + */ + async #claimPidFile(): Promise { + const target = pidFilePath(this.#baseDir); + await NodeFSP.mkdir(NodePath.dirname(target), { recursive: true, mode: 0o700 }); + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + const handle = await NodeFSP.open(target, "wx", 0o600); + try { + await handle.writeFile(`${process.pid}\n`, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + return true; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; + // A live owner wins, even when it is this very process. Two Launcher + // instances sharing a process would still be two servers on one + // database, so there is nothing to exempt. + const owner = await this.#readPidFile(); + if (owner !== undefined && processIsAlive(owner)) return false; + // The recorded owner is gone, so the file is leftover. Clear and retry. + await NodeFSP.rm(target, { force: true }); } } + return false; + } + + async #readPidFile(): Promise { + const contents = await NodeFSP.readFile(pidFilePath(this.#baseDir), "utf8").catch(() => ""); + const pid = Number.parseInt(contents.trim(), 10); + return Number.isInteger(pid) && pid > 0 ? pid : undefined; + } + + /** Windows only. Only clears the file if this process still owns it. */ + async #releasePidFile(): Promise { + if ((await this.#readPidFile()) !== process.pid) return; + await NodeFSP.rm(pidFilePath(this.#baseDir), { force: true }).catch(() => undefined); } /** From d43c2013d9eb36268083c0858e4ab61f6023ec19 Mon Sep 17 00:00:00 2001 From: InayaYousfi Date: Sun, 9 Aug 2026 18:21:07 +0200 Subject: [PATCH 3/7] fix(server): make Windows service failures visible and stop stale disabled flags Five more review findings, four of them about errors the user never sees. The percent-sign guard and the disabled-entry guard both wrapped a synthetic Error inside BootServiceInstallError, whose message is fixed text. The CLI prints message, so both carefully worded explanations were invisible. They are now dedicated tagged errors that derive their message from a field, and the onboarding recovery path prints them directly. The stop timeout put a pid and a duration into the step string, which every other step in this module and in bootService.ts keeps as a stable literal. They are structured attributes on BootServiceCommandError now. The disabled flag is read from a registry value that outlives the shortcut, so uninstalling a disabled entry and reinstalling later told the user to re-enable something Startup apps no longer lists. It is only read when the shortcut is actually there. The module imports bootService.ts as a namespace, matching how every other service-boundary consumer in the repo imports it. Also fixes a real bug the tests could not see: the probe never emitted the shortcut readback, because an escaping mistake dropped those lines from the generated script. The test fake produced them itself, so status looked right while real Windows would have reported "needs repair" forever. A test now pins the generated script rather than the fake. --- apps/server/src/cli/service.ts | 6 ++ apps/server/src/cloud/bootService.ts | 47 +++++++- .../src/cloud/bootServiceWindows.test.ts | 36 ++++++- apps/server/src/cloud/bootServiceWindows.ts | 100 +++++++++--------- 4 files changed, 131 insertions(+), 58 deletions(-) diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index 4588bc33f39..ca01274ae4e 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -222,6 +222,12 @@ export const recoverServiceOnboardingOffer = ( Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), BootServiceUpdatePendingError: (error) => Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), + // Both carry guidance the user has to act on, so print the message itself + // rather than the generic "did not finish" wrapper. + BootServicePathHasPercentError: (error) => + Console.warn(`Skipping background setup: ${error.message}`).pipe(Effect.as(false)), + BootServiceStartupEntryDisabledError: (error) => + Console.warn(`Skipping background setup: ${error.message}`).pipe(Effect.as(false)), }), ); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 72af5c10d7b..ae02666651a 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -99,13 +99,20 @@ export class BootServiceCommandError extends Schema.TaggedErrorClass()( + "BootServicePathHasPercentError", + { pathLabel: Schema.String }, +) { + override get message(): string { + return ( + `The path to ${this.pathLabel} contains a percent sign, and the Windows command ` + + "shell would rewrite it before the service could start. Move T3 Code to a path " + + "without one, or set T3CODE_HOME to such a path, then run this again." + ); + } +} + +/** Windows only. The Startup entry exists but Windows Settings has it switched off. */ +export class BootServiceStartupEntryDisabledError extends Schema.TaggedErrorClass()( + "BootServiceStartupEntryDisabledError", + { shortcutName: Schema.String }, +) { + override get message(): string { + return ( + `"${this.shortcutName}" is switched off under Startup apps in Windows Settings. ` + + "Turn it back on, then run this again." + ); + } +} + export type BootServiceError = | BootServiceUnsupportedError | BootServiceCommandError | BootServiceInstallError - | BootServiceUpdatePendingError; + | BootServiceUpdatePendingError + | BootServicePathHasPercentError + | BootServiceStartupEntryDisabledError; /** * Which backend answered. The name carries the platform on purpose: a bare diff --git a/apps/server/src/cloud/bootServiceWindows.test.ts b/apps/server/src/cloud/bootServiceWindows.test.ts index ded28455c47..4d72bca55f9 100644 --- a/apps/server/src/cloud/bootServiceWindows.test.ts +++ b/apps/server/src/cloud/bootServiceWindows.test.ts @@ -73,6 +73,19 @@ it("points the shortcut at a hidden PowerShell running the startup script", () = expect(shortcut).toContain("$shortcut.WindowStyle = 7"); }); +it("has the probe report the shortcut it finds, and the disabled flag only with it", () => { + // The harness fake produces these lines itself, so without this the real + // script could stop emitting them and every test would still pass while + // Windows reported "needs repair" forever. + const shortcut = BootServiceWindows.renderShortcutScript(plan); + + expect(shortcut).toContain('Write-Output "target=$($existing.TargetPath)"'); + expect(shortcut).toContain('Write-Output "arguments=$($existing.Arguments)"'); + // A disabled flag outlives the shortcut, so it is only read alongside one. + expect(shortcut).toContain("$present = Test-Path -LiteralPath $shortcutPath"); + expect(shortcut).toContain("if ($present -and (Test-Path -LiteralPath $key)) {"); +}); + it("reads both findings out of the probe output", () => { expect(BootServiceWindows.parseProbeOutput("shell=True\r\ndisabled=False\r\n")).toEqual({ shellRunning: true, @@ -105,6 +118,7 @@ it("spots a percent sign, which the command shell would rewrite silently", () => const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function* ( platform: NodeJS.Platform = "win32", + execPath = "C:\\node.exe", ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -194,7 +208,7 @@ const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function baseDir, logsDir: path.join(baseDir, "userdata", "logs"), cliVersion: "1.2.3", - host: { execPath: "C:\\node.exe", launcherSourcePath: sourceLauncher }, + host: { execPath, launcherSourcePath: sourceLauncher }, stopRequestTimeout: Duration.millis(30), }).pipe( Effect.provideService(ProcessRunner.ProcessRunner, runner), @@ -285,8 +299,10 @@ it.layer(NodeServices.layer)("windows boot service", (it) => { control.disabled = "True"; const error = yield* service.install.pipe(Effect.flip); - expect(error._tag).toBe("BootServiceInstallError"); - expect(String(error.cause)).toContain("Startup apps"); + // A dedicated tag, because the generic install error hides its cause and + // the CLI prints only `message`. The guidance has to reach the user. + expect(error._tag).toBe("BootServiceStartupEntryDisabledError"); + expect(error.message).toContain("Startup apps in Windows Settings"); }).pipe(TestClock.withLive), ); @@ -312,7 +328,7 @@ it.layer(NodeServices.layer)("windows boot service", (it) => { const error = yield* service.install.pipe(Effect.flip); expect(error._tag).toBe("BootServiceCommandError"); - expect(error.message).toContain("did not exit"); + expect(error.message).toContain("did not exit within"); }).pipe(TestClock.withLive), ); @@ -355,6 +371,18 @@ it.layer(NodeServices.layer)("windows boot service", (it) => { }).pipe(TestClock.withLive), ); + it.effect("names the offending path when it holds a percent sign", () => + Effect.gen(function* () { + const { service } = yield* makeHarness("win32", "C:\\pct %TEMP% dir\\node.exe"); + + const error = yield* service.install.pipe(Effect.flip); + // The generic install error prints fixed text, so a dedicated tag is what + // gets the user told which path to move. + expect(error._tag).toBe("BootServicePathHasPercentError"); + expect(error.message).toContain("percent sign"); + }).pipe(TestClock.withLive), + ); + it.effect("fails closed off Windows", () => Effect.gen(function* () { const { service } = yield* makeHarness("linux"); diff --git a/apps/server/src/cloud/bootServiceWindows.ts b/apps/server/src/cloud/bootServiceWindows.ts index 8943811d744..a4af9771bbc 100644 --- a/apps/server/src/cloud/bootServiceWindows.ts +++ b/apps/server/src/cloud/bootServiceWindows.ts @@ -28,15 +28,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as ProcessRunner from "../processRunner.ts"; -import { - BootService, - BootServiceCommandError, - BootServiceInstallError, - BootServiceUnsupportedError, - BootServiceUpdatePendingError, - type BootServiceHost, - type BootServicePlan, -} from "./bootService.ts"; +import * as BootService from "./bootService.ts"; import { ensurePinnedRuntimeInstalled, pinnedRuntimePaths, @@ -116,7 +108,7 @@ export function quotePowerShellLiteral(value: string): string { return `'${value.replaceAll("'", "''")}'`; } -export interface WindowsBootServicePlan extends BootServicePlan { +export interface WindowsBootServicePlan extends BootService.BootServicePlan { readonly startupScriptPath: string; readonly shortcutScriptPath: string; } @@ -186,15 +178,25 @@ export function renderShortcutScript(plan: WindowsBootServicePlan): string { " # deliberate choice, so this is reported and not treated as a failure.", " $shell = @(Get-Process -Name 'explorer' -ErrorAction SilentlyContinue).Count -gt 0", " # Windows records entries disabled through Settings here. The low bit of", - " # the first byte is the disabled flag.", + " # the first byte is the disabled flag. It only means anything while the", + " # shortcut exists, because the value outlives the .lnk. Reporting a stale", + " # one sends the user hunting for an entry Startup apps no longer shows.", " $key = 'HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\StartupApproved\\StartupFolder'", " $disabled = $false", - " if (Test-Path -LiteralPath $key) {", + " $present = Test-Path -LiteralPath $shortcutPath", + " if ($present -and (Test-Path -LiteralPath $key)) {", " $value = (Get-ItemProperty -LiteralPath $key -Name $shortcutFile -ErrorAction SilentlyContinue).$shortcutFile", " if ($null -ne $value -and $value.Length -gt 0) { $disabled = ($value[0] -band 1) -eq 1 }", " }", ' Write-Output "shell=$shell"', ' Write-Output "disabled=$disabled"', + " # Read the shortcut back. That the script which wrote it still matches", + " # says nothing about a .lnk somebody edited or replaced since.", + " if ($present) {", + " $existing = (New-Object -ComObject WScript.Shell).CreateShortcut($shortcutPath)", + ' Write-Output "target=$($existing.TargetPath)"', + ' Write-Output "arguments=$($existing.Arguments)"', + " }", " exit 0", "}", "", @@ -246,7 +248,7 @@ export interface WindowsBootServiceInput { readonly baseDir: string; readonly logsDir: string; readonly cliVersion: string; - readonly host?: BootServiceHost; + readonly host?: BootService.BootServiceHost; /** Overridable so tests do not sit through the real wait. */ readonly stopRequestTimeout?: Duration.Duration; } @@ -310,11 +312,11 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( yield* (yield* fs.open(tempPath, { flag: "r" })).sync; yield* fs.rename(tempPath, filePath); }), - ).pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + ).pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); const requireWindows = Effect.gen(function* () { if (platform !== "win32" || appData === "") { - return yield* new BootServiceUnsupportedError({ platform }); + return yield* new BootService.BootServiceUnsupportedError({ platform }); } }); @@ -328,11 +330,11 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( options?: { readonly timeout?: Duration.Input }, ) { return yield* runner.run({ command, args, timeout: options?.timeout }).pipe( - Effect.mapError((cause) => new BootServiceCommandError({ step, cause })), + Effect.mapError((cause) => new BootService.BootServiceCommandError({ step, cause })), Effect.filterOrFail( (result) => result.code === 0, (result) => - new BootServiceCommandError({ + new BootService.BootServiceCommandError({ step, exitCode: Number(result.code), stdoutLength: result.stdout.length, @@ -386,21 +388,23 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( */ const preflight: Effect.Effect< WindowsPreflight, - BootServiceUnsupportedError | BootServiceInstallError | BootServiceCommandError + | BootService.BootServiceUnsupportedError + | BootService.BootServiceInstallError + | BootService.BootServiceCommandError > = Effect.gen(function* () { yield* requireWindows; yield* fs .makeDirectory(startupDir, { recursive: true }) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); // Prove it is writable now, rather than failing halfway through an install. yield* Effect.scoped( fs.makeTempFileScoped({ directory: startupDir, prefix: ".t3-service-probe-" }), - ).pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + ).pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); const probe = yield* runShortcutScript("checking the Windows Startup folder", "Probe"); const findings = parseProbeOutput(probe.stdout); if (findings === undefined) { - return yield* new BootServiceCommandError({ + return yield* new BootService.BootServiceCommandError({ step: "reading the Windows Startup folder check", stdoutLength: probe.stdout.length, stderrLength: probe.stderr.length, @@ -435,7 +439,7 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( // server against the same database. yield* fs .writeFileString(stopRequestPath, "") - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); const timeoutMs = Duration.toMillis(input.stopRequestTimeout ?? STOP_REQUEST_TIMEOUT); const pollMs = Math.min(timeoutMs, Duration.toMillis(STOP_REQUEST_ACK_POLL)); @@ -447,8 +451,10 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( } // Refusing here is the whole point. Carrying on would write over a launcher // we could not confirm dead. - return yield* new BootServiceCommandError({ - step: `stopping the running service (process ${pid} did not exit in ${Math.round(timeoutMs / 1000)}s)`, + return yield* new BootService.BootServiceCommandError({ + step: "stopping the running service", + pid, + timeoutSeconds: Math.round(timeoutMs / 1000), }); }); @@ -492,22 +498,22 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( }).pipe( Effect.mapError((error) => error._tag === "PinnedRuntimeInstallError" - ? new BootServiceCommandError({ + ? new BootService.BootServiceCommandError({ step: error.step, exitCode: error.exitCode, stdoutLength: error.stdoutLength, stderrLength: error.stderrLength, cause: error, }) - : new BootServiceInstallError({ cause: error }), + : new BootService.BootServiceInstallError({ cause: error }), ), ); - const install: BootService["Service"]["install"] = Effect.gen(function* () { + const install: BootService.BootService["Service"]["install"] = Effect.gen(function* () { yield* requireWindows; yield* fs .makeDirectory(input.logsDir, { recursive: true }) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); const percentIn = findPercentInPaths([ ["the Node executable", plan.nodePath], @@ -515,13 +521,7 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( ["the log directory", plan.logPath], ]); if (percentIn !== undefined) { - return yield* new BootServiceInstallError({ - cause: new Error( - `The path to ${percentIn} contains a percent sign, and the Windows command ` + - "shell would rewrite it before the service could start. Move T3 Code to a " + - "path without one, or set T3CODE_HOME to such a path, then run this again.", - ), - }); + return yield* new BootService.BootServicePathHasPercentError({ pathLabel: percentIn }); } // Write the shortcut script first so the preflight has something to run. @@ -538,11 +538,8 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( ); } if (checks.entryDisabled) { - return yield* new BootServiceInstallError({ - cause: new Error( - `"${SHORTCUT_NAME}" is switched off under Startup apps in Windows Settings. ` + - "Turn it back on, then run this again.", - ), + return yield* new BootService.BootServiceStartupEntryDisabledError({ + shortcutName: SHORTCUT_NAME, }); } @@ -550,11 +547,11 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( yield* installPinnedRuntime; const launcherSource = yield* fs .readFileString(launcherSourcePath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); const installed = yield* fs .exists(unitPath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); // Keyed on the pid file, never on the shortcut. A launcher outlives a // manually deleted shortcut, and starting a second one alongside it would // put two servers on the same database. @@ -566,7 +563,7 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( Option.isSome(previousStateText) && serviceStateHasPendingUpdate(previousStateText.value) ) { - return yield* new BootServiceUpdatePendingError(); + return yield* new BootService.BootServiceUpdatePendingError(); } yield* writeDurably(launcherPath, launcherSource); yield* writeDurably( @@ -596,14 +593,14 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( : Effect.void, ), ); - return plan satisfies BootServicePlan; + return plan satisfies BootService.BootServicePlan; }).pipe(Effect.withSpan("cloud.boot_service_windows.install")); - const uninstall: BootService["Service"]["uninstall"] = Effect.gen(function* () { + const uninstall: BootService.BootService["Service"]["uninstall"] = Effect.gen(function* () { yield* requireWindows; const installed = yield* fs .exists(unitPath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); // Stop first, and decide that on the pid file rather than the shortcut. A // launcher survives someone deleting the shortcut by hand, and leaving it @@ -615,13 +612,13 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( // also means a missing or broken generated script cannot strand the entry. yield* fs .remove(unitPath, { force: true }) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); yield* fs.remove(startupScriptPath, { force: true }).pipe(Effect.ignore); yield* fs.remove(shortcutScriptPath, { force: true }).pipe(Effect.ignore); return true; }).pipe(Effect.withSpan("cloud.boot_service_windows.uninstall")); - const status: BootService["Service"]["status"] = Effect.gen(function* () { + const status: BootService.BootService["Service"]["status"] = Effect.gen(function* () { const base = { kind: "win32-startup-shortcut", unitPath, logPath } as const; if (platform !== "win32" || appData === "") { return { supported: false, installed: false, current: false, ...base }; @@ -680,11 +677,12 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( ...base, }; }).pipe( - Effect.mapError((cause) => new BootServiceInstallError({ cause })), + Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause })), Effect.withSpan("cloud.boot_service_windows.status"), ); - return BootService.of({ install, uninstall, status }); + return BootService.BootService.of({ install, uninstall, status }); }); -export const layer = (input: WindowsBootServiceInput) => Layer.effect(BootService, make(input)); +export const layer = (input: WindowsBootServiceInput) => + Layer.effect(BootService.BootService, make(input)); From f02a4bf97ad25347acfa74501aa50367db76611b Mon Sep 17 00:00:00 2001 From: InayaYousfi Date: Sun, 9 Aug 2026 18:28:25 +0200 Subject: [PATCH 4/7] fix(server): stop the Windows pid record outliving its process Three review findings, all about a pid file that means less than it looks. A process id is not identity. The file survives a crash or a hard reboot, and Windows recycles ids freely, so a stale record could name an unrelated live process. The launcher then refused to start for as long as that stranger ran, and the CLI waited on it forever. The record now carries the machine's boot time, so anything written before this boot is obviously stale on both sides. A dead launcher also left its stop request behind. The next install would start a launcher that immediately read that request and stopped itself, while the install reported success. Cleaning up after a dead launcher now clears both files. The percent-sign error told users to set T3CODE_HOME even when the offending path was the Node executable, which that variable cannot move. The message gives no relocation advice now; the path label names what to change. --- apps/server/src/cloud/bootService.ts | 6 ++- .../src/cloud/bootServiceWindows.test.ts | 43 ++++++++++++++++++- apps/server/src/cloud/bootServiceWindows.ts | 34 ++++++++++++--- apps/server/src/cloud/serviceProtocol.ts | 38 ++++++++++++++++ apps/server/src/serviceLauncher.test.ts | 24 +++++++++++ apps/server/src/serviceLauncher.ts | 37 ++++++++++++---- 6 files changed, 162 insertions(+), 20 deletions(-) diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index ae02666651a..a1a13c5708c 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -145,10 +145,12 @@ export class BootServicePathHasPercentError extends Schema.TaggedErrorClass { }).pipe(TestClock.withLive), ); + it.effect("ignores a pid record left behind by an earlier boot", () => + Effect.gen(function* () { + const { service, fs, pidPath } = yield* makeHarness(); + yield* service.install; + // Process ids are recycled across reboots, so a record from a previous + // boot can name a live stranger. Trusting it would block every install. + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ pid: process.pid, bootTimeMs: 0 }), + ); + + yield* service.install; + expect((yield* service.status).current).toBe(true); + }).pipe(TestClock.withLive), + ); + + it.effect("clears a leftover stop request when no launcher is listening", () => + Effect.gen(function* () { + const { service, fs, pidPath, stopRequestPath } = yield* makeHarness(); + yield* service.install; + // A dead launcher leaves both files. Removing only the pid record would + // let the request stop the very launcher the next install starts. + yield* fs.remove(pidPath, { force: true }); + yield* fs.writeFileString(stopRequestPath, ""); + + yield* service.install; + expect(yield* fs.exists(stopRequestPath)).toBe(false); + }).pipe(TestClock.withLive), + ); + it.effect("fails closed off Windows", () => Effect.gen(function* () { const { service } = yield* makeHarness("linux"); diff --git a/apps/server/src/cloud/bootServiceWindows.ts b/apps/server/src/cloud/bootServiceWindows.ts index a4af9771bbc..64d9b1b0b6a 100644 --- a/apps/server/src/cloud/bootServiceWindows.ts +++ b/apps/server/src/cloud/bootServiceWindows.ts @@ -26,6 +26,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as NodeOS from "node:os"; import * as ProcessRunner from "../processRunner.ts"; import * as BootService from "./bootService.ts"; @@ -39,6 +40,8 @@ import { SERVICE_LAUNCHER_PROTOCOL, SERVICE_PID_FILE, SERVICE_STATE_FILE, + decodeServiceLauncherPresence, + serviceLauncherPresenceIsFromThisBoot, SERVICE_STOP_REQUEST_FILE, parseServiceState, serviceStateHasPendingUpdate, @@ -103,6 +106,11 @@ const processIsAlive = (pid: number): Effect.Effect => } }); +/** Windows only. Milliseconds since the epoch when this machine last booted. */ +const currentBootTimeMs = DateTime.now.pipe( + Effect.map((now) => DateTime.toEpochMillis(now) - NodeOS.uptime() * 1_000), +); + /** Windows only. PowerShell single-quoted strings escape a quote by doubling it. */ export function quotePowerShellLiteral(value: string): string { return `'${value.replaceAll("'", "''")}'`; @@ -425,14 +433,24 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( * start a second server on the same database. */ const requestStop = Effect.gen(function* () { - const recordedPid = yield* fs.readFileString(pidPath).pipe(Effect.option); - if (Option.isNone(recordedPid)) return false; - const pid = Number.parseInt(recordedPid.value.trim(), 10); - if (!Number.isInteger(pid) || pid <= 0 || !(yield* processIsAlive(pid))) { - // The launcher died without cleaning up. Nothing to wait for. + const recorded = yield* fs.readFileString(pidPath).pipe(Effect.option); + if (Option.isNone(recorded)) return false; + const presence = decodeServiceLauncherPresence(recorded.value); + // Only trust the process id while the record is from this boot. Ids are + // recycled across reboots, so an old file can name an unrelated live + // process, and waiting on that would never finish. + const live = + presence !== undefined && + serviceLauncherPresenceIsFromThisBoot(presence, yield* currentBootTimeMs) && + (yield* processIsAlive(presence.pid)); + if (!live || presence === undefined) { + // No launcher is listening. Clear the request too: leaving one behind + // would stop the very launcher this install is about to start. yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore); + yield* fs.remove(stopRequestPath, { force: true }).pipe(Effect.ignore); return false; } + const pid = presence.pid; // A launcher is confirmed alive, so a failed write must fail the whole // operation. Swallowing it would let the caller carry on and start a second @@ -517,8 +535,10 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( const percentIn = findPercentInPaths([ ["the Node executable", plan.nodePath], - ["the data directory", plan.launcherPath], - ["the log directory", plan.logPath], + // Named with the variable that moves it, because the shared message + // deliberately gives no relocation advice of its own. + ["the T3 Code data directory (T3CODE_HOME)", plan.launcherPath], + ["the T3 Code log directory", plan.logPath], ]); if (percentIn !== undefined) { return yield* new BootService.BootServicePathHasPercentError({ pathLabel: percentIn }); diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 46ba3ec7efe..2b9ac00ae7e 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -24,6 +24,44 @@ export const SERVICE_STOP_REQUEST_FILE = ".service-stop-request"; servers on one database. */ export const SERVICE_PID_FILE = ".service-pid"; +/** + * Windows only. What the pid file records. + * + * A process id alone is not enough. The file survives a crash or a hard reboot, + * and the operating system reuses process ids freely, so an old file can name a + * live process that has nothing to do with T3 Code. Believing it would leave the + * launcher refusing to start for as long as that stranger runs. Recording when + * the machine booted makes any file written before this boot obviously stale. + */ +export interface ServiceLauncherPresence { + readonly pid: number; + readonly bootTimeMs: number; +} + +/** Boot time is derived from uptime, which drifts by a little between reads. */ +export const SAME_BOOT_TOLERANCE_MS = 60_000; + +export function encodeServiceLauncherPresence(presence: ServiceLauncherPresence): string { + return `${presence.pid} ${presence.bootTimeMs}\n`; +} + +export function decodeServiceLauncherPresence(text: string): ServiceLauncherPresence | undefined { + const [rawPid, rawBoot] = text.trim().split(/\s+/); + const pid = Number(rawPid); + const bootTimeMs = Number(rawBoot); + return Number.isInteger(pid) && pid > 0 && Number.isFinite(bootTimeMs) + ? { pid, bootTimeMs } + : undefined; +} + +/** False when the file predates this boot, whatever process now holds that id. */ +export function serviceLauncherPresenceIsFromThisBoot( + presence: ServiceLauncherPresence, + currentBootTimeMs: number, +): boolean { + return Math.abs(presence.bootTimeMs - currentBootTimeMs) <= SAME_BOOT_TOLERANCE_MS; +} + export interface PendingServiceUpdate { readonly id: string; readonly fromVersion: string; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index cf776ab9721..a9f94713fc2 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -10,6 +10,9 @@ import { compareExactServiceVersions, decodeServiceState, isExactServiceVersion, + decodeServiceLauncherPresence, + encodeServiceLauncherPresence, + serviceLauncherPresenceIsFromThisBoot, SERVICE_LAUNCHER_PROTOCOL, SERVICE_PID_FILE, SERVICE_STOP_MARKER_FILE, @@ -468,3 +471,24 @@ it.layer(NodeServices.layer)("self-supervising pid ownership", (it) => { }).pipe(TestClock.withLive), ); }); + +it("treats a pid record from an earlier boot as stale, whoever holds that id now", () => { + const presence = { pid: process.pid, bootTimeMs: 1_000 }; + + assert.isFalse(serviceLauncherPresenceIsFromThisBoot(presence, 9_000_000)); + assert.isTrue(serviceLauncherPresenceIsFromThisBoot(presence, 1_000)); + // Boot time comes from uptime, which drifts a little between reads. + assert.isTrue(serviceLauncherPresenceIsFromThisBoot(presence, 30_000)); +}); + +it("round-trips a pid record and rejects a malformed one", () => { + const presence = { pid: 4321, bootTimeMs: 1_700_000_000_000 }; + + assert.deepEqual( + decodeServiceLauncherPresence(encodeServiceLauncherPresence(presence)), + presence, + ); + assert.isUndefined(decodeServiceLauncherPresence("")); + // The old format carried a bare pid, and believing it would drop the guard. + assert.isUndefined(decodeServiceLauncherPresence("4321\n")); +}); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index e38d26e8505..95cb13d87c0 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -7,6 +7,7 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import type { @@ -14,6 +15,7 @@ import type { ServiceLauncherChildMessage, ServiceLauncherContext, ServiceLauncherParentMessage, + ServiceLauncherPresence, ServiceState, ServiceUpdateRecord, } from "./cloud/serviceProtocol.ts"; @@ -25,6 +27,9 @@ import { SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_LAUNCHER_PROTOCOL, SERVICE_PID_FILE, + decodeServiceLauncherPresence, + encodeServiceLauncherPresence, + serviceLauncherPresenceIsFromThisBoot, SERVICE_SELF_SUPERVISE_ENV, SERVICE_STATE_FILE, SERVICE_STOP_MARKER_FILE, @@ -61,6 +66,9 @@ const STOP_REQUEST_GRACE_MS = 5_000; /** Windows only. Set by the generated logon script. systemd never sets it. */ const isSelfSupervising = (): boolean => process.env[SERVICE_SELF_SUPERVISE_ENV] === "1"; +/** Windows only. Milliseconds since the epoch when this machine last booted. */ +const currentBootTimeMs = (): number => Date.now() - NodeOS.uptime() * 1_000; + /** * Windows only. Signal 0 asks the kernel whether a process exists without * touching it. A permission error means it exists but is not ours, which still @@ -411,12 +419,16 @@ export class Launcher { */ async #claimPidFile(): Promise { const target = pidFilePath(this.#baseDir); + const bootTimeMs = currentBootTimeMs(); await NodeFSP.mkdir(NodePath.dirname(target), { recursive: true, mode: 0o700 }); for (let attempt = 0; attempt < 2; attempt += 1) { try { const handle = await NodeFSP.open(target, "wx", 0o600); try { - await handle.writeFile(`${process.pid}\n`, "utf8"); + await handle.writeFile( + encodeServiceLauncherPresence({ pid: process.pid, bootTimeMs }), + "utf8", + ); await handle.sync(); } finally { await handle.close(); @@ -424,27 +436,34 @@ export class Launcher { return true; } catch (cause) { if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; + const owner = await this.#readPidFile(); // A live owner wins, even when it is this very process. Two Launcher // instances sharing a process would still be two servers on one - // database, so there is nothing to exempt. - const owner = await this.#readPidFile(); - if (owner !== undefined && processIsAlive(owner)) return false; - // The recorded owner is gone, so the file is leftover. Clear and retry. + // database, so there is nothing to exempt. But only trust the process + // id while the file is from this boot: ids are recycled across reboots, + // and believing a stranger would keep the service down indefinitely. + if ( + owner !== undefined && + serviceLauncherPresenceIsFromThisBoot(owner, bootTimeMs) && + processIsAlive(owner.pid) + ) { + return false; + } + // Leftover, from a dead owner or an earlier boot. Clear it and retry. await NodeFSP.rm(target, { force: true }); } } return false; } - async #readPidFile(): Promise { + async #readPidFile(): Promise { const contents = await NodeFSP.readFile(pidFilePath(this.#baseDir), "utf8").catch(() => ""); - const pid = Number.parseInt(contents.trim(), 10); - return Number.isInteger(pid) && pid > 0 ? pid : undefined; + return decodeServiceLauncherPresence(contents); } /** Windows only. Only clears the file if this process still owns it. */ async #releasePidFile(): Promise { - if ((await this.#readPidFile()) !== process.pid) return; + if ((await this.#readPidFile())?.pid !== process.pid) return; await NodeFSP.rm(pidFilePath(this.#baseDir), { force: true }).catch(() => undefined); } From 0340862022dd8000cd7c236278a8262a2eef1468 Mon Sep 17 00:00:00 2001 From: InayaYousfi Date: Sun, 9 Aug 2026 18:38:10 +0200 Subject: [PATCH 5/7] fix(server): a pid record now proves the launcher is alive, not just present Four review findings. A boot stamp catches a record from an earlier boot, but not one left by a launcher that crashed during this boot whose id has since been handed to someone else. A running launcher now refreshes its record every 15s, and a record nobody has refreshed for 90s is treated as dead on both sides. Claiming a stale record was a delete followed by a create, so two launchers racing could both delete and both claim. An exclusive takeover file picks a single winner, and it re-checks the holder under that file before deleting. A start that failed transiently escaped into the fatal path and killed the launcher outright, which defeats the supervisor over exactly the kind of failure it exists to absorb. It now spends another attempt from the same burst, so the burst limit still bounds it. install stopped the running launcher before checking for a pending remote update, so the guard meant to protect that update interrupted it first. The check now runs before anything is terminated. Also: a transient read error on the pid record no longer counts as proof the launcher is gone. Only a confirmed not-found does. --- .../src/cloud/bootServiceWindows.test.ts | 40 ++++++ apps/server/src/cloud/bootServiceWindows.ts | 37 +++-- apps/server/src/serviceLauncher.test.ts | 43 ++++++ apps/server/src/serviceLauncher.ts | 133 +++++++++++++----- 4 files changed, 211 insertions(+), 42 deletions(-) diff --git a/apps/server/src/cloud/bootServiceWindows.test.ts b/apps/server/src/cloud/bootServiceWindows.test.ts index 3e1562eae1a..8d60d39a37f 100644 --- a/apps/server/src/cloud/bootServiceWindows.test.ts +++ b/apps/server/src/cloud/bootServiceWindows.test.ts @@ -422,6 +422,46 @@ it.layer(NodeServices.layer)("windows boot service", (it) => { }).pipe(TestClock.withLive), ); + it.effect("refuses to reinstall while a remote update is still pending", () => + Effect.gen(function* () { + const { service, fs, statePath, pidPath } = yield* makeHarness(); + yield* service.install; + // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document. + const pendingState = JSON.stringify({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + update: { + id: "u", + fromVersion: "1.2.3", + targetVersion: "1.2.4", + dbPath: "C:\\state.sqlite", + status: "pending", + }, + }); + yield* fs.writeFileString(statePath, pendingState); + + const error = yield* service.install.pipe(Effect.flip); + expect(error._tag).toBe("BootServiceUpdatePendingError"); + // The guard protects the in-flight update, so it must refuse before + // anything is terminated. A surviving pid record proves nothing stopped. + expect(yield* fs.exists(pidPath)).toBe(true); + }).pipe(TestClock.withLive), + ); + + it.effect("ignores a pid record nobody has refreshed", () => + Effect.gen(function* () { + const { service, fs, pidPath } = yield* makeHarness(); + yield* service.install; + // Same boot and a live process id, but the launcher stopped refreshing + // it. Within one boot that is the only way to tell a recycled id apart. + const longAgo = 1_577_836_800_000; // 2020-01-01 + yield* fs.utimes(pidPath, longAgo, longAgo); + + yield* service.install; + expect((yield* service.status).current).toBe(true); + }).pipe(TestClock.withLive), + ); + it.effect("fails closed off Windows", () => Effect.gen(function* () { const { service } = yield* makeHarness("linux"); diff --git a/apps/server/src/cloud/bootServiceWindows.ts b/apps/server/src/cloud/bootServiceWindows.ts index 64d9b1b0b6a..630f5c97d35 100644 --- a/apps/server/src/cloud/bootServiceWindows.ts +++ b/apps/server/src/cloud/bootServiceWindows.ts @@ -68,6 +68,8 @@ const STOP_REQUEST_TIMEOUT = Duration.seconds(30); /** Windows only. How often the CLI re-checks whether the launcher has gone. */ const STOP_REQUEST_ACK_POLL = Duration.millis(250); const POWERSHELL_TIMEOUT = Duration.seconds(30); +/** Windows only. Must match the launcher's own staleness rule for its pid record. */ +const PID_RECORD_STALE_AFTER_MS = 90_000; /** * Windows only. The absolute interpreter path, rather than trusting PATH. @@ -433,15 +435,33 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( * start a second server on the same database. */ const requestStop = Effect.gen(function* () { - const recorded = yield* fs.readFileString(pidPath).pipe(Effect.option); + // Only a confirmed absence means no launcher. A transient read error, say a + // sharing violation while the launcher rewrites its record, must not be + // read as proof the launcher is gone, or the install writes over a live one. + const recorded = yield* fs.readFileString(pidPath).pipe( + Effect.map(Option.some), + Effect.catch((error) => + error.reason._tag === "NotFound" + ? Effect.succeed(Option.none()) + : Effect.fail(new BootService.BootServiceInstallError({ cause: error })), + ), + ); if (Option.isNone(recorded)) return false; const presence = decodeServiceLauncherPresence(recorded.value); // Only trust the process id while the record is from this boot. Ids are // recycled across reboots, so an old file can name an unrelated live // process, and waiting on that would never finish. + const recordAge = yield* fs.stat(pidPath).pipe( + Effect.map((info) => Option.map(info.mtime, (mtime) => mtime.getTime())), + Effect.orElseSucceed(() => Option.none()), + ); + const nowMs = yield* DateTime.now.pipe(Effect.map(DateTime.toEpochMillis)); + const refreshedRecently = + Option.isSome(recordAge) && nowMs - recordAge.value <= PID_RECORD_STALE_AFTER_MS; const live = presence !== undefined && serviceLauncherPresenceIsFromThisBoot(presence, yield* currentBootTimeMs) && + refreshedRecently && (yield* processIsAlive(presence.pid)); if (!live || presence === undefined) { // No launcher is listening. Clear the request too: leaving one behind @@ -572,19 +592,20 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( const installed = yield* fs .exists(unitPath) .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + // Before anything is stopped. This guard exists to protect an in-flight + // remote update, and terminating the launcher first would interrupt the + // very thing it is guarding. + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); + if (Option.isSome(previousStateText) && serviceStateHasPendingUpdate(previousStateText.value)) { + return yield* new BootService.BootServiceUpdatePendingError(); + } + // Keyed on the pid file, never on the shortcut. A launcher outlives a // manually deleted shortcut, and starting a second one alongside it would // put two servers on the same database. const stopped = yield* requestStop; yield* Effect.gen(function* () { - const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); - if ( - Option.isSome(previousStateText) && - serviceStateHasPendingUpdate(previousStateText.value) - ) { - return yield* new BootService.BootServiceUpdatePendingError(); - } yield* writeDurably(launcherPath, launcherSource); yield* writeDurably( statePath, diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index a9f94713fc2..b2e1c25f6a4 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -492,3 +492,46 @@ it("round-trips a pid record and rejects a malformed one", () => { // The old format carried a bare pid, and believing it would drop the guard. assert.isUndefined(decodeServiceLauncherPresence("4321\n")); }); + +it.layer(NodeServices.layer)("self-supervising start failures", (it) => { + it.effect("spends another burst attempt when a start fails, instead of dying", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-startfail-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "process.exit(1);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + const running = launcher.run(); + // Remove the runtime mid-flight, so the next start throws rather than + // producing a child that exits. That used to reach #fatal and kill the + // launcher outright, defeating the supervisor over a transient failure. + yield* Effect.sleep("40 millis"); + yield* fs.remove(versionDir, { recursive: true, force: true }); + + // It still ends by exhausting the burst, not by an unhandled start error. + const outcome = yield* Effect.promise(() => + running.then( + () => "completed" as const, + (cause: unknown) => String(cause), + ), + ); + assert.include(outcome, "Active child exited unexpectedly"); + }).pipe(TestClock.withLive), + ); +}); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 95cb13d87c0..d806caccdaf 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -69,6 +69,18 @@ const isSelfSupervising = (): boolean => process.env[SERVICE_SELF_SUPERVISE_ENV] /** Windows only. Milliseconds since the epoch when this machine last booted. */ const currentBootTimeMs = (): number => Date.now() - NodeOS.uptime() * 1_000; +/** + * Windows only. How often a running launcher refreshes its pid record. + * + * The boot stamp catches a record left by an earlier boot, but not one left by + * a launcher that crashed during this boot whose id has since been handed to an + * unrelated process. A record that stops being refreshed is proof of death on + * its own, whoever holds that id now. + */ +const PID_HEARTBEAT_MS = 15_000; +/** Generous against a paused or heavily loaded machine, still far below a reboot. */ +const PID_RECORD_STALE_AFTER_MS = 90_000; + /** * Windows only. Signal 0 asks the kernel whether a process exists without * touching it. A permission error means it exists but is not ours, which still @@ -320,6 +332,16 @@ const stopRequestPath = (baseDir: string) => /** Windows only. Present exactly while a self-supervising launcher is running. */ const pidFilePath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_PID_FILE); +/** Windows only. A record nobody has refreshed lately belongs to a dead launcher. */ +async function pidRecordIsStale(target: string): Promise { + try { + const stats = await NodeFSP.stat(target); + return Date.now() - stats.mtimeMs > PID_RECORD_STALE_AFTER_MS; + } catch { + return true; + } +} + export interface LauncherOptions { /** * Windows only. Whether this launcher must restart its own child and watch @@ -355,6 +377,7 @@ export class Launcher { * child would then read "a replacement server is coming" while none is. */ #recovered = false; + #pidHeartbeat: NodeJS.Timeout | undefined; #stopWatcher: NodeFS.FSWatcher | undefined; #stopPoll: NodeJS.Timeout | undefined; /** Windows only. Lets a stop cut a restart backoff short instead of queueing behind it. */ @@ -392,6 +415,7 @@ export class Launcher { process.stderr.write("[service-launcher] another launcher is already running\n"); return; } + this.#startPidHeartbeat(); this.#watchStopRequest(); } this.#enqueue(() => this.#recover()); @@ -403,6 +427,8 @@ export class Launcher { process.off("SIGTERM", onSigterm); process.off("SIGINT", onSigint); this.#stopWatchingStopRequest(); + clearInterval(this.#pidHeartbeat); + this.#pidHeartbeat = undefined; if (owned) { // Removing these before run() returns is how the CLI learns the stop // finished. Doing it after would race the process exiting. @@ -419,41 +445,74 @@ export class Launcher { */ async #claimPidFile(): Promise { const target = pidFilePath(this.#baseDir); - const bootTimeMs = currentBootTimeMs(); await NodeFSP.mkdir(NodePath.dirname(target), { recursive: true, mode: 0o700 }); - for (let attempt = 0; attempt < 2; attempt += 1) { + if (await this.#writePidFileExclusive(target)) return true; + + if (await this.#pidFileHolderIsAlive()) return false; + + // The record is leftover. Clearing it and creating our own is two steps, so + // two launchers racing here could both delete and both claim. An exclusive + // takeover file decides a single winner; everyone else loses and exits. + const takeover = `${target}.takeover`; + try { + await (await NodeFSP.open(takeover, "wx", 0o600)).close(); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; + return false; + } + try { + // Re-check under the takeover: the previous holder may have revived. + if (await this.#pidFileHolderIsAlive()) return false; + await NodeFSP.rm(target, { force: true }); + return await this.#writePidFileExclusive(target); + } finally { + await NodeFSP.rm(takeover, { force: true }).catch(() => undefined); + } + } + + async #writePidFileExclusive(target: string): Promise { + try { + const handle = await NodeFSP.open(target, "wx", 0o600); try { - const handle = await NodeFSP.open(target, "wx", 0o600); - try { - await handle.writeFile( - encodeServiceLauncherPresence({ pid: process.pid, bootTimeMs }), - "utf8", - ); - await handle.sync(); - } finally { - await handle.close(); - } - return true; - } catch (cause) { - if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; - const owner = await this.#readPidFile(); - // A live owner wins, even when it is this very process. Two Launcher - // instances sharing a process would still be two servers on one - // database, so there is nothing to exempt. But only trust the process - // id while the file is from this boot: ids are recycled across reboots, - // and believing a stranger would keep the service down indefinitely. - if ( - owner !== undefined && - serviceLauncherPresenceIsFromThisBoot(owner, bootTimeMs) && - processIsAlive(owner.pid) - ) { - return false; - } - // Leftover, from a dead owner or an earlier boot. Clear it and retry. - await NodeFSP.rm(target, { force: true }); + await handle.writeFile( + encodeServiceLauncherPresence({ + pid: process.pid, + bootTimeMs: currentBootTimeMs(), + }), + "utf8", + ); + await handle.sync(); + } finally { + await handle.close(); } + return true; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; + return false; } - return false; + } + + /** + * Windows only. True only for a record from this boot, naming a live process, + * and still being refreshed. A live owner wins even when it is this very + * process: two Launcher instances in one process are still two servers on one + * database, so there is nothing to exempt. + */ + async #pidFileHolderIsAlive(): Promise { + const owner = await this.#readPidFile(); + if (owner === undefined) return false; + if (!serviceLauncherPresenceIsFromThisBoot(owner, currentBootTimeMs())) return false; + if (!processIsAlive(owner.pid)) return false; + return !(await pidRecordIsStale(pidFilePath(this.#baseDir))); + } + + /** Windows only. Keeps this launcher's record demonstrably fresh. */ + #startPidHeartbeat(): void { + const target = pidFilePath(this.#baseDir); + this.#pidHeartbeat = setInterval(() => { + void NodeFSP.utimes(target, new Date(), new Date()).catch(() => undefined); + }, PID_HEARTBEAT_MS); + this.#pidHeartbeat.unref(); } async #readPidFile(): Promise { @@ -828,11 +887,17 @@ export class Launcher { // Windows only. Nothing supervises a Startup folder entry, so the launcher // does what Restart=always does on Linux, and gives up on the same terms. // Without the flag this still throws, so the Linux path is unchanged. - if (this.#selfSupervise && this.#recordRestartAttempt()) { + while (this.#selfSupervise && this.#recordRestartAttempt()) { await this.#waitBeforeRestart(); if (this.#stopping || this.#done || this.#stopRequested) return; - await this.#startChild(this.#state.activeVersion, "active", this.#state.update); - return; + try { + await this.#startChild(this.#state.activeVersion, "active", this.#state.update); + return; + } catch { + // A start can fail transiently, and letting that escape would reach + // #fatal and kill the launcher outright. Spend another attempt from the + // same burst instead, so the supervisor survives what it exists for. + } } throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); } From ab09d4d6cc9825671ae14b34967f2aec4765dc0a Mon Sep 17 00:00:00 2001 From: InayaYousfi Date: Sun, 9 Aug 2026 18:47:07 +0200 Subject: [PATCH 6/7] fix(server): recover from a takeover abandoned mid-claim A launcher that died between creating the takeover file and its finally left that file behind forever. Every later start then read it as "somebody is busy", returned false and exited, so the service could never come back and the only cure was deleting a hidden file by hand. A takeover covers a stat, a read, a delete and a write, so one that has been sitting for 30s was abandoned. Recovery renames it away before retrying, because concurrent renames of one source leave exactly one winner and the rest find the source already gone. A fresh takeover still means somebody is genuinely mid-claim, and that case stands down as before. Also fixes a units bug that made three tests lie. A bare number given to utimes is a Unix timestamp in seconds, so passing milliseconds set the modification time to the year 51969. Two staleness tests were therefore backdating into the future and passing without ever reaching the branch they name. Both now use seconds, one had to provoke the watcher to reach the check at all, and the pid test now stops the fake launcher from answering so a live-looking record would actually fail it. --- .../src/cloud/bootServiceWindows.test.ts | 11 +- apps/server/src/serviceLauncher.test.ts | 107 +++++++++++++++++- apps/server/src/serviceLauncher.ts | 58 +++++++++- 3 files changed, 163 insertions(+), 13 deletions(-) diff --git a/apps/server/src/cloud/bootServiceWindows.test.ts b/apps/server/src/cloud/bootServiceWindows.test.ts index 8d60d39a37f..c7868e31831 100644 --- a/apps/server/src/cloud/bootServiceWindows.test.ts +++ b/apps/server/src/cloud/bootServiceWindows.test.ts @@ -450,12 +450,17 @@ it.layer(NodeServices.layer)("windows boot service", (it) => { it.effect("ignores a pid record nobody has refreshed", () => Effect.gen(function* () { - const { service, fs, pidPath } = yield* makeHarness(); + const { service, fs, pidPath, control } = yield* makeHarness(); yield* service.install; + // Nothing answers a stop request now, so if the record were judged live + // this install would wait and then fail. That is what makes the assertion + // below mean something. + control.launcherStopsOnRequest = false; // Same boot and a live process id, but the launcher stopped refreshing // it. Within one boot that is the only way to tell a recycled id apart. - const longAgo = 1_577_836_800_000; // 2020-01-01 - yield* fs.utimes(pidPath, longAgo, longAgo); + // Seconds, not milliseconds: utimes reads a bare number as seconds. + const longAgoSeconds = 1_577_836_800; // 2020-01-01 + yield* fs.utimes(pidPath, longAgoSeconds, longAgoSeconds); yield* service.install; expect((yield* service.status).current).toBe(true); diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index b2e1c25f6a4..53a86129c47 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -413,8 +413,10 @@ process.exit(1); // Backdate it well past the grace window, which is what makes it stale. // A request written just before startup is deliberately treated as real. // A fixed date, so this does not depend on the clock the test runs under. - const staleEpochMs = 1_577_836_800_000; // 2020-01-01 - yield* fs.utimes(requestPath, staleEpochMs, staleEpochMs); + // Seconds, not milliseconds: utimes reads a bare number as seconds, and + // milliseconds would land in the far future and read as brand new. + const staleSeconds = 1_577_836_800; // 2020-01-01 + yield* fs.utimes(requestPath, staleSeconds, staleSeconds); const launcher = new Launcher( root, @@ -422,8 +424,14 @@ process.exit(1); { selfSupervise: true, restartDelayMs: 5 }, ); const running = launcher.run(); - yield* Effect.sleep("100 millis"); - // Still running: the stale request was cleared, not obeyed. + yield* Effect.sleep("150 millis"); + // Provoke the watcher now that recovery has finished. Without this the + // next check is a 2s poll away, and the test would pass without ever + // exercising the staleness branch it exists for. + yield* fs.writeFileString(path.join(root, "runtime", "provoke"), ""); + yield* Effect.sleep("150 millis"); + + // Still running: the stale request was judged old, not obeyed. assert.isFalse(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); yield* Effect.promise(() => launcher.stop("SIGTERM")); @@ -535,3 +543,94 @@ it.layer(NodeServices.layer)("self-supervising start failures", (it) => { }).pipe(TestClock.withLive), ); }); + +it.layer(NodeServices.layer)("abandoned takeover recovery", (it) => { + it.effect("starts anyway when a previous claim died holding the takeover", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-takeover-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + // A launcher that died mid-claim leaves both files behind. Without + // recovery the takeover blocks every future start, and the only cure is + // deleting a hidden file by hand. + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + const takeoverPath = `${pidPath}.takeover`; + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ pid: 999_999, bootTimeMs: 0 }), + ); + yield* fs.writeFileString(takeoverPath, ""); + // Seconds, not milliseconds: a number given to utimes is a Unix + // timestamp in seconds, and passing milliseconds lands in the far future. + const longAgoSeconds = 1_577_836_800; // 2020-01-01 + yield* fs.utimes(takeoverPath, longAgoSeconds, longAgoSeconds); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + const running = launcher.run(); + yield* Effect.sleep("150 millis"); + + // It claimed the record, and cleared the abandoned takeover on its way. + assert.isTrue(yield* fs.exists(pidPath)); + assert.isFalse(yield* fs.exists(takeoverPath)); + + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }).pipe(TestClock.withLive), + ); + + it.effect("stands down while another launcher is genuinely mid-claim", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-takeover-busy-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ pid: 999_999, bootTimeMs: 0 }), + ); + // A fresh takeover means somebody else is actively claiming, so the + // recovery above must not fire and let two launchers through. + yield* fs.writeFileString(`${pidPath}.takeover`, ""); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + yield* Effect.promise(() => launcher.run()); + + // It exited without touching the other launcher's takeover. + assert.isTrue(yield* fs.exists(`${pidPath}.takeover`)); + }).pipe(TestClock.withLive), + ); +}); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index d806caccdaf..bf3e23ef745 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -80,6 +80,12 @@ const currentBootTimeMs = (): number => Date.now() - NodeOS.uptime() * 1_000; const PID_HEARTBEAT_MS = 15_000; /** Generous against a paused or heavily loaded machine, still far below a reboot. */ const PID_RECORD_STALE_AFTER_MS = 90_000; +/** + * Windows only. How long a takeover of a leftover pid record may sit before it + * counts as abandoned. The work it covers is a stat, a read, a delete and a + * write, so this is orders of magnitude more than any real one needs. + */ +const TAKEOVER_STALE_AFTER_MS = 30_000; /** * Windows only. Signal 0 asks the kernel whether a process exists without @@ -454,12 +460,7 @@ export class Launcher { // two launchers racing here could both delete and both claim. An exclusive // takeover file decides a single winner; everyone else loses and exits. const takeover = `${target}.takeover`; - try { - await (await NodeFSP.open(takeover, "wx", 0o600)).close(); - } catch (cause) { - if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; - return false; - } + if (!(await this.#claimTakeover(takeover))) return false; try { // Re-check under the takeover: the previous holder may have revived. if (await this.#pidFileHolderIsAlive()) return false; @@ -470,6 +471,51 @@ export class Launcher { } } + /** + * Windows only. Wins the right to replace a leftover pid record. + * + * A takeover spans a handful of file operations, so one that has been sitting + * around was abandoned by a launcher that died mid-claim, or by the machine + * losing power. Treating that as "somebody is busy" would block every start + * from then on, and the only cure would be deleting a hidden file by hand. + */ + async #claimTakeover(takeover: string): Promise { + if (await this.#createTakeover(takeover)) return true; + + let abandonedAt: number | undefined; + try { + const stats = await NodeFSP.stat(takeover); + abandonedAt = + Date.now() - stats.mtimeMs > TAKEOVER_STALE_AFTER_MS ? stats.mtimeMs : undefined; + } catch { + // It vanished, so the launcher holding it finished. Let that one win. + return false; + } + if (abandonedAt === undefined) return false; + + // Renaming picks a single winner without a read-then-delete window: + // concurrent renames of one source leave exactly one success, and everyone + // else finds the source already gone. + const claimed = `${takeover}.${process.pid}.${NodeCrypto.randomUUID()}`; + try { + await NodeFSP.rename(takeover, claimed); + } catch { + return false; + } + await NodeFSP.rm(claimed, { force: true }).catch(() => undefined); + return await this.#createTakeover(takeover); + } + + async #createTakeover(takeover: string): Promise { + try { + await (await NodeFSP.open(takeover, "wx", 0o600)).close(); + return true; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; + return false; + } + } + async #writePidFileExclusive(target: string): Promise { try { const handle = await NodeFSP.open(target, "wx", 0o600); From d3fcaccb6bad6e9f7382e32da518227f8eedd6d4 Mon Sep 17 00:00:00 2001 From: InayaYousfi Date: Mon, 10 Aug 2026 13:34:17 +0200 Subject: [PATCH 7/7] fix(server): an old heartbeat no longer counts as a dead launcher Four review findings, three of which shared a root cause: I let a timestamp decide something a timestamp cannot decide. A laptop that sleeps for an hour wakes with a perfectly live launcher whose last heartbeat is ancient. Reading that as death cleared its record and started a second server on one SQLite database. An old timestamp now only makes the answer unknown, and the unknown case is settled by watching for the next heartbeat instead of guessing. The CLI gets that for free: its stop wait is already longer than several heartbeats, so a record that never moves during the wait belongs to a recycled process id, and one that does move is a live launcher that will not stop and must not be written over. That also settles the boot tolerance concern. A reboot inside the tolerance window can still look like the same boot, but the record is then quiet and the probe reaches the right answer anyway. The first start had no burst retry, only restarts did, so a transient spawn failure at sign-in left the service down until the next one. Both paths now share one retry. Windows has no KillMode=mixed, so a launcher killed outright leaves its server behind. The record now names the child, and whoever takes a leftover record over puts that orphan down before starting its own. --- .../src/cloud/bootServiceWindows.test.ts | 28 +++- apps/server/src/cloud/bootServiceWindows.ts | 36 +++-- apps/server/src/cloud/serviceProtocol.ts | 22 ++- apps/server/src/serviceLauncher.test.ts | 100 +++++++++++++ apps/server/src/serviceLauncher.ts | 140 ++++++++++++++---- 5 files changed, 273 insertions(+), 53 deletions(-) diff --git a/apps/server/src/cloud/bootServiceWindows.test.ts b/apps/server/src/cloud/bootServiceWindows.test.ts index c7868e31831..fa597273fbe 100644 --- a/apps/server/src/cloud/bootServiceWindows.test.ts +++ b/apps/server/src/cloud/bootServiceWindows.test.ts @@ -161,6 +161,10 @@ const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function failCommand: undefined as string | undefined, /** Set to false to model a launcher that never answers the stop request. */ launcherStopsOnRequest: true, + /** Set to false to model a record whose process id was recycled. */ + launcherRefreshesRecord: true, + /** Rewritten by the stand-in launcher to prove it is still alive. */ + presence: "", /** What the fake shortcut currently points at, so drift can be modelled. */ shortcutTarget: powershell, shortcutArguments: expectedArguments, @@ -179,10 +183,8 @@ const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function // boot stamp matters, since a record from an earlier boot is ignored. const bootTimeMs = DateTime.toEpochMillis(yield* DateTime.now) - NodeOS.uptime() * 1_000; yield* fs.makeDirectory(runtimeDir, { recursive: true }); - yield* fs.writeFileString( - pidPath, - encodeServiceLauncherPresence({ pid: process.pid, bootTimeMs }), - ); + control.presence = encodeServiceLauncherPresence({ pid: process.pid, bootTimeMs }); + yield* fs.writeFileString(pidPath, control.presence); } if (command.includes("service-shortcut")) { if (action === "Install") { @@ -237,8 +239,17 @@ const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function Effect.gen(function* () { while (true) { yield* Effect.sleep(Duration.millis(5)); - if (!control.launcherStopsOnRequest) continue; const asked = yield* fs.exists(stopRequestPath).pipe(Effect.orElseSucceed(() => false)); + if (!control.launcherStopsOnRequest) { + // A launcher that will not stop is still alive, and a live one keeps + // refreshing its record. Without that it would look like a process id + // the system recycled, which is a different case entirely. + const present = yield* fs.exists(pidPath).pipe(Effect.orElseSucceed(() => false)); + if (control.launcherRefreshesRecord && present) { + yield* fs.writeFileString(pidPath, control.presence).pipe(Effect.ignore); + } + continue; + } if (!asked) continue; yield* fs.remove(stopRequestPath, { force: true }).pipe(Effect.ignore); yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore); @@ -452,10 +463,11 @@ it.layer(NodeServices.layer)("windows boot service", (it) => { Effect.gen(function* () { const { service, fs, pidPath, control } = yield* makeHarness(); yield* service.install; - // Nothing answers a stop request now, so if the record were judged live - // this install would wait and then fail. That is what makes the assertion - // below mean something. + // Nothing answers a stop request now, and nothing refreshes the record + // either. That pair is what a recycled process id looks like, and it is + // what makes the assertion below mean something. control.launcherStopsOnRequest = false; + control.launcherRefreshesRecord = false; // Same boot and a live process id, but the launcher stopped refreshing // it. Within one boot that is the only way to tell a recycled id apart. // Seconds, not milliseconds: utimes reads a bare number as seconds. diff --git a/apps/server/src/cloud/bootServiceWindows.ts b/apps/server/src/cloud/bootServiceWindows.ts index 630f5c97d35..d9694a63c5b 100644 --- a/apps/server/src/cloud/bootServiceWindows.ts +++ b/apps/server/src/cloud/bootServiceWindows.ts @@ -68,8 +68,6 @@ const STOP_REQUEST_TIMEOUT = Duration.seconds(30); /** Windows only. How often the CLI re-checks whether the launcher has gone. */ const STOP_REQUEST_ACK_POLL = Duration.millis(250); const POWERSHELL_TIMEOUT = Duration.seconds(30); -/** Windows only. Must match the launcher's own staleness rule for its pid record. */ -const PID_RECORD_STALE_AFTER_MS = 90_000; /** * Windows only. The absolute interpreter path, rather than trusting PATH. @@ -434,6 +432,11 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( * stopped, and the install would then rewrite the runtime underneath it and * start a second server on the same database. */ + const pidRecordMtime = fs.stat(pidPath).pipe( + Effect.map((info) => Option.map(info.mtime, (mtime) => mtime.getTime())), + Effect.orElseSucceed(() => Option.none()), + ); + const requestStop = Effect.gen(function* () { // Only a confirmed absence means no launcher. A transient read error, say a // sharing violation while the launcher rewrites its record, must not be @@ -451,17 +454,14 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( // Only trust the process id while the record is from this boot. Ids are // recycled across reboots, so an old file can name an unrelated live // process, and waiting on that would never finish. - const recordAge = yield* fs.stat(pidPath).pipe( - Effect.map((info) => Option.map(info.mtime, (mtime) => mtime.getTime())), - Effect.orElseSucceed(() => Option.none()), - ); - const nowMs = yield* DateTime.now.pipe(Effect.map(DateTime.toEpochMillis)); - const refreshedRecently = - Option.isSome(recordAge) && nowMs - recordAge.value <= PID_RECORD_STALE_AFTER_MS; + // An old timestamp is not evidence of death. A laptop that slept wakes with + // a live launcher whose last heartbeat is ancient, and clearing its record + // would start a second server on one database. So this only asks whether the + // holder could be ours; the wait below settles whether it really is. + const recordMtimeBefore = yield* pidRecordMtime; const live = presence !== undefined && serviceLauncherPresenceIsFromThisBoot(presence, yield* currentBootTimeMs) && - refreshedRecently && (yield* processIsAlive(presence.pid)); if (!live || presence === undefined) { // No launcher is listening. Clear the request too: leaving one behind @@ -487,8 +487,20 @@ export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( const stillRunning = yield* fs.exists(pidPath).pipe(Effect.orElseSucceed(() => true)); if (!stillRunning) return true; } - // Refusing here is the whole point. Carrying on would write over a launcher - // we could not confirm dead. + // Nothing stopped. The wait was longer than several heartbeats, so a record + // that never moved belongs to a process that is not our launcher at all, + // which happens when the operating system recycles the id. Clear it and let + // the caller proceed. A record that did move is a launcher that is alive and + // simply will not stop, and writing over that one must not happen. + const recordMtimeAfter = yield* pidRecordMtime; + const neverRefreshed = + Option.isNone(recordMtimeAfter) || + (Option.isSome(recordMtimeBefore) && recordMtimeBefore.value === recordMtimeAfter.value); + if (neverRefreshed) { + yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore); + yield* fs.remove(stopRequestPath, { force: true }).pipe(Effect.ignore); + return false; + } return yield* new BootService.BootServiceCommandError({ step: "stopping the running service", pid, diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 2b9ac00ae7e..09aa9dc37fa 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -36,22 +36,34 @@ export const SERVICE_PID_FILE = ".service-pid"; export interface ServiceLauncherPresence { readonly pid: number; readonly bootTimeMs: number; + /** + * The server the launcher is currently running, when it has one. + * + * Windows has no equivalent of `KillMode=mixed`, so a launcher killed + * outright leaves its server behind. Recording the child means whoever takes + * the record over can put that orphan down before starting its own, instead + * of quietly ending up with two servers on one database. + */ + readonly childPid?: number; } /** Boot time is derived from uptime, which drifts by a little between reads. */ export const SAME_BOOT_TOLERANCE_MS = 60_000; export function encodeServiceLauncherPresence(presence: ServiceLauncherPresence): string { - return `${presence.pid} ${presence.bootTimeMs}\n`; + const child = presence.childPid === undefined ? "" : ` ${presence.childPid}`; + return `${presence.pid} ${presence.bootTimeMs}${child}\n`; } export function decodeServiceLauncherPresence(text: string): ServiceLauncherPresence | undefined { - const [rawPid, rawBoot] = text.trim().split(/\s+/); + const [rawPid, rawBoot, rawChild] = text.trim().split(/\s+/); const pid = Number(rawPid); const bootTimeMs = Number(rawBoot); - return Number.isInteger(pid) && pid > 0 && Number.isFinite(bootTimeMs) - ? { pid, bootTimeMs } - : undefined; + if (!Number.isInteger(pid) || pid <= 0 || !Number.isFinite(bootTimeMs)) return undefined; + const childPid = rawChild === undefined ? Number.NaN : Number(rawChild); + return Number.isInteger(childPid) && childPid > 0 + ? { pid, bootTimeMs, childPid } + : { pid, bootTimeMs }; } /** False when the file predates this boot, whatever process now holds that id. */ diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 53a86129c47..6b8c12f5530 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -2,7 +2,10 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as NodeOS from "node:os"; + import * as Path from "effect/Path"; +import * as DateTime from "effect/DateTime"; import * as TestClock from "effect/testing/TestClock"; import { Launcher, readServiceState, writeServiceState } from "./serviceLauncher.ts"; @@ -634,3 +637,100 @@ it.layer(NodeServices.layer)("abandoned takeover recovery", (it) => { }).pipe(TestClock.withLive), ); }); + +const nowMillis = DateTime.now.pipe(Effect.map(DateTime.toEpochMillis)); + +it.layer(NodeServices.layer)("liveness of a quiet pid record", (it) => { + const seed = Effect.fn("test.seed_quiet_runtime")(function* (root: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + return statePath; + }); + + it.effect("stands down when an old record starts moving again", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-sleepy-" }); + const statePath = yield* seed(root); + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + // A live launcher that slept through its heartbeat: this process id is + // real and from this boot, but the record looks ancient. Treating that as + // dead would put a second server on one database. + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ + pid: process.pid, + bootTimeMs: (yield* nowMillis) - NodeOS.uptime() * 1_000, + }), + ); + const longAgoSeconds = 1_577_836_800; // 2020-01-01, in seconds + yield* fs.utimes(pidPath, longAgoSeconds, longAgoSeconds); + + // It wakes and refreshes while the probe is watching. + yield* Effect.forkScoped( + Effect.sleep("60 millis").pipe( + Effect.flatMap(() => fs.writeFileString(pidPath, "refreshed by the live launcher\n")), + ), + ); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5, heartbeatProbeMs: 200 }, + ); + yield* Effect.promise(() => launcher.run()); + + // It exited rather than claiming, and left the record alone. + assert.equal(yield* fs.readFileString(pidPath), "refreshed by the live launcher\n"); + }).pipe(TestClock.withLive), + ); + + it.effect("takes over a record that never moves again", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-recycled-" }); + const statePath = yield* seed(root); + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + // Same shape, but nothing ever refreshes it. That is a process id the + // system handed to somebody else, so the record is not a launcher at all. + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ + pid: process.pid, + bootTimeMs: (yield* nowMillis) - NodeOS.uptime() * 1_000, + }), + ); + const longAgoSeconds = 1_577_836_800; // 2020-01-01, in seconds + yield* fs.utimes(pidPath, longAgoSeconds, longAgoSeconds); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5, heartbeatProbeMs: 200 }, + ); + const running = launcher.run(); + yield* Effect.sleep("600 millis"); + + const claimed = decodeServiceLauncherPresence(yield* fs.readFileString(pidPath)); + assert.equal(claimed?.pid, process.pid); + assert.isDefined(claimed?.childPid); + + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }).pipe(TestClock.withLive), + ); +}); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index bf3e23ef745..c1dbb24eaac 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -338,14 +338,34 @@ const stopRequestPath = (baseDir: string) => /** Windows only. Present exactly while a self-supervising launcher is running. */ const pidFilePath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_PID_FILE); -/** Windows only. A record nobody has refreshed lately belongs to a dead launcher. */ -async function pidRecordIsStale(target: string): Promise { +const pidRecordMtime = async (target: string): Promise => { try { - const stats = await NodeFSP.stat(target); - return Date.now() - stats.mtimeMs > PID_RECORD_STALE_AFTER_MS; + return (await NodeFSP.stat(target)).mtimeMs; } catch { - return true; + return undefined; } +}; + +/** + * Windows only. Whether a launcher is still refreshing its record. + * + * An old modification time on its own proves nothing. A laptop that slept for + * an hour wakes with a perfectly live launcher whose last heartbeat is ancient, + * and treating that as dead would start a second server on one database. So an + * old timestamp only makes the answer unknown, and the unknown case is settled + * by watching for the next heartbeat rather than guessing. + */ +export async function pidRecordIsBeingRefreshed( + target: string, + probeMs: number = PID_HEARTBEAT_MS * 2, +): Promise { + const first = await pidRecordMtime(target); + if (first === undefined) return false; + if (Date.now() - first <= PID_RECORD_STALE_AFTER_MS) return true; + + await new Promise((resolve) => setTimeout(resolve, probeMs)); + const second = await pidRecordMtime(target); + return second !== undefined && second !== first; } export interface LauncherOptions { @@ -357,6 +377,8 @@ export interface LauncherOptions { readonly selfSupervise?: boolean; /** Overridable so tests do not sit through the real wait. */ readonly restartDelayMs?: number; + /** Overridable for the same reason: how long to watch for a heartbeat. */ + readonly heartbeatProbeMs?: number; } export class Launcher { @@ -364,6 +386,7 @@ export class Launcher { readonly #statePath: string; readonly #selfSupervise: boolean; readonly #restartDelayMs: number; + readonly #heartbeatProbeMs: number; #state: ServiceState; #child: ManagedChild | null = null; #timer: NodeJS.Timeout | undefined; @@ -396,6 +419,7 @@ export class Launcher { this.#state = state; this.#selfSupervise = options?.selfSupervise ?? isSelfSupervising(); this.#restartDelayMs = options?.restartDelayMs ?? RESTART_DELAY_MS; + this.#heartbeatProbeMs = options?.heartbeatProbeMs ?? PID_HEARTBEAT_MS * 2; } async run(): Promise { @@ -464,6 +488,8 @@ export class Launcher { try { // Re-check under the takeover: the previous holder may have revived. if (await this.#pidFileHolderIsAlive()) return false; + // Whoever left this record may have left its server running too. + await this.#terminateOrphanedChild(await this.#readPidFile()); await NodeFSP.rm(target, { force: true }); return await this.#writePidFileExclusive(target); } finally { @@ -520,13 +546,7 @@ export class Launcher { try { const handle = await NodeFSP.open(target, "wx", 0o600); try { - await handle.writeFile( - encodeServiceLauncherPresence({ - pid: process.pid, - bootTimeMs: currentBootTimeMs(), - }), - "utf8", - ); + await handle.writeFile(this.#presenceRecord(), "utf8"); await handle.sync(); } finally { await handle.close(); @@ -538,6 +558,46 @@ export class Launcher { } } + #presenceRecord(): string { + const childPid = this.#child?.process.pid; + return encodeServiceLauncherPresence({ + pid: process.pid, + bootTimeMs: currentBootTimeMs(), + ...(childPid === undefined ? {} : { childPid }), + }); + } + + /** + * Windows only. Rewrites the record so it names the current child, and so its + * modification time proves this launcher is still here. Doubling as the + * heartbeat keeps one writer for one file. + */ + async #refreshPidFile(): Promise { + if (!this.#selfSupervise) return; + await NodeFSP.writeFile(pidFilePath(this.#baseDir), this.#presenceRecord(), { + mode: 0o600, + }).catch(() => undefined); + } + + /** + * Windows only. Puts down a server left behind by a launcher that died + * without taking it with it. Linux gets this from KillMode=mixed. + */ + async #terminateOrphanedChild(owner: ServiceLauncherPresence | undefined): Promise { + const childPid = owner?.childPid; + if (childPid === undefined || childPid === process.pid) return; + if (owner !== undefined && !serviceLauncherPresenceIsFromThisBoot(owner, currentBootTimeMs())) { + return; + } + if (!processIsAlive(childPid)) return; + try { + process.kill(childPid, "SIGKILL"); + process.stderr.write(`[service-launcher] ended an orphaned server (pid ${childPid})\n`); + } catch { + // It exited between the check and the signal, which is the good outcome. + } + } + /** * Windows only. True only for a record from this boot, naming a live process, * and still being refreshed. A live owner wins even when it is this very @@ -549,14 +609,13 @@ export class Launcher { if (owner === undefined) return false; if (!serviceLauncherPresenceIsFromThisBoot(owner, currentBootTimeMs())) return false; if (!processIsAlive(owner.pid)) return false; - return !(await pidRecordIsStale(pidFilePath(this.#baseDir))); + return await pidRecordIsBeingRefreshed(pidFilePath(this.#baseDir), this.#heartbeatProbeMs); } /** Windows only. Keeps this launcher's record demonstrably fresh. */ #startPidHeartbeat(): void { - const target = pidFilePath(this.#baseDir); this.#pidHeartbeat = setInterval(() => { - void NodeFSP.utimes(target, new Date(), new Date()).catch(() => undefined); + void this.#refreshPidFile(); }, PID_HEARTBEAT_MS); this.#pidHeartbeat.unref(); } @@ -640,6 +699,41 @@ export class Launcher { }); } + /** + * Windows only. Retries the active child until the burst is spent. Returns + * false once it is, so the caller can fail the way Linux already does. + */ + async #restartActiveChild(): Promise { + while (this.#selfSupervise && this.#recordRestartAttempt()) { + await this.#waitBeforeRestart(); + if (this.#stopping || this.#done || this.#stopRequested) return true; + try { + await this.#startChild(this.#state.activeVersion, "active", this.#state.update); + return true; + } catch { + // A start can fail transiently, and letting that escape would reach + // #fatal and kill the launcher outright. Spend another attempt from the + // same burst instead, so the supervisor survives what it exists for. + } + } + return false; + } + + /** + * Windows only. The very first start needs the same treatment. A transient + * failure at sign-in would otherwise leave the service down until the next + * one, which is not what Restart=always does. + */ + async #startActiveChildWithRetry(update?: ServiceUpdateRecord): Promise { + try { + await this.#startChild(this.#state.activeVersion, "active", update); + return; + } catch (cause) { + if (!this.#selfSupervise) throw cause; + if (!(await this.#restartActiveChild())) throw cause; + } + } + /** Windows only. False once the burst limit is reached inside the window. */ #recordRestartAttempt(): boolean { const now = Date.now(); @@ -716,7 +810,7 @@ export class Launcher { if (update !== undefined) { await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); } - await this.#startChild(this.#state.activeVersion, "active", update); + await this.#startActiveChildWithRetry(update); return; } if (await databaseRestorePending(this.#baseDir, update)) { @@ -781,6 +875,7 @@ export class Launcher { process: child, }; this.#child = managed; + await this.#refreshPidFile(); child.on("message", (value) => { const message = decodeServiceLauncherChildMessage(value); if (message !== undefined) this.#enqueue(() => this.#handleMessage(managed, message)); @@ -933,18 +1028,7 @@ export class Launcher { // Windows only. Nothing supervises a Startup folder entry, so the launcher // does what Restart=always does on Linux, and gives up on the same terms. // Without the flag this still throws, so the Linux path is unchanged. - while (this.#selfSupervise && this.#recordRestartAttempt()) { - await this.#waitBeforeRestart(); - if (this.#stopping || this.#done || this.#stopRequested) return; - try { - await this.#startChild(this.#state.activeVersion, "active", this.#state.update); - return; - } catch { - // A start can fail transiently, and letting that escape would reach - // #fatal and kill the launcher outright. Spend another attempt from the - // same burst instead, so the supervisor survives what it exists for. - } - } + if (await this.#restartActiveChild()) return; throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); }