diff --git a/apps/server/src/cli/connect.ts b/apps/server/src/cli/connect.ts index ef15e650a6f..c0d46ac9f01 100644 --- a/apps/server/src/cli/connect.ts +++ b/apps/server/src/cli/connect.ts @@ -5,6 +5,7 @@ import { type RelayClientInstallProgressStage, } from "@t3tools/contracts"; import { RelayOkResponse } from "@t3tools/contracts/relay"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as RelayClient from "@t3tools/shared/relayClient"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import * as Cause from "effect/Cause"; @@ -694,9 +695,13 @@ export const connectCommand = Command.make("connect", { // fail the command, just tell the user what happened and move on. const background = yield* recoverServiceOnboardingOffer(offerServiceDuringOnboarding); if (background) { - yield* Console.log( - "\n✓ Background service ready\n\nT3 Code will stay reachable after you log out.", - ); + // Windows starts the service at sign-in and stops it at sign-out, so + // the Linux promise would contradict the prompt the user just agreed to. + const reach = + (yield* HostProcessPlatform) === "win32" + ? "T3 Code will start again every time you sign in to Windows." + : "T3 Code will stay reachable after you log out."; + yield* Console.log(`\n✓ Background service ready\n\n${reach}`); return; } const serveCommand = yield* resolveCliCommand("serve"); diff --git a/apps/server/src/cli/service.test.ts b/apps/server/src/cli/service.test.ts index e91e10000b6..e1ca7c51945 100644 --- a/apps/server/src/cli/service.test.ts +++ b/apps/server/src/cli/service.test.ts @@ -6,10 +6,18 @@ const status = { supported: true, installed: true, current: true, + kind: "systemd", unitPath: "/home/me/.config/systemd/user/t3code.service", logPath: "/home/me/.t3/userdata/logs/boot-service.log", } as const; +const windowsStatus = { + ...status, + kind: "win32-startup-shortcut", + unitPath: + "C:\\Users\\me\\AppData\\Roaming\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\T3 Code Server.lnk", +} as const; + it("reports the installed service version and host paths", () => { assert.equal( formatServiceStatus(status, "0.0.29"), @@ -32,6 +40,12 @@ it("gives a direct repair command for a stale service", () => { it("explains service availability without systemd", () => { assert.include( formatServiceStatus({ ...status, supported: false, installed: false }, "0.0.29"), - "Supported on: Linux with systemd", + "Supported on: Linux with systemd, or Windows", ); }); + +it("calls the Windows definition a shortcut, not a unit", () => { + const output = formatServiceStatus(windowsStatus, "0.0.29"); + assert.include(output, ` Shortcut: ${windowsStatus.unitPath}`); + assert.notInclude(output, "Unit:"); +}); diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index d55b270f183..ca01274ae4e 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -1,3 +1,4 @@ +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Console from "effect/Console"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -10,12 +11,23 @@ import type * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts"; -export const bootServiceLayer = (config: ServerConfig.ServerConfig["Service"]) => - BootService.layer({ +export const bootServiceLayer = (config: ServerConfig.ServerConfig["Service"]) => { + const input = { baseDir: config.baseDir, logsDir: config.logsDir, cliVersion: packageJson.version, - }).pipe(Layer.provide(ProcessRunner.layer)); + }; + // Windows has no systemd, so it gets its own backend. Loading it lazily keeps + // the Linux path free of any Windows import. + return Layer.unwrap( + Effect.gen(function* () { + const platform = yield* HostProcessPlatform; + if (platform !== "win32") return BootService.layer(input); + const windows = yield* Effect.promise(() => import("../cloud/bootServiceWindows.ts")); + return windows.layer(input); + }), + ).pipe(Layer.provide(ProcessRunner.layer)); +}; export type ServiceReconcileResult = | { @@ -48,7 +60,7 @@ export function formatServiceStatus( cliVersion: string, ): string { if (!status.supported) { - return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd"; + return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd, or Windows"; } if (!status.installed) { return "T3 Code service\n Status: not installed\n Next: Run `t3 service install`."; @@ -56,7 +68,7 @@ export function formatServiceStatus( return [ "T3 Code service", ` Status: ${status.current ? `installed · t3@${cliVersion}` : "needs an update or repair"}`, - ` Unit: ${status.unitPath}`, + ` ${status.kind === "systemd" ? "Unit" : "Shortcut"}: ${status.unitPath}`, ` Logs: ${status.logPath}`, ...(status.current ? [] : [" Next: Run `npx t3@latest service update`."]), ].join("\n"); @@ -71,6 +83,30 @@ const runServiceCommand = Effect.fn("cli.service.run")(function* ( return yield* run.pipe(Effect.provide(bootServiceLayer(config))); }); +/** Windows only. The blink at sign-in looks alarming until you know what it is. */ +const WINDOWS_SERVICE_NOTICE = [ + 'A small window named "T3 Code Server" blinks once when you sign in. That is expected.', + "It appears under Startup apps in Windows Settings, where you can switch it off.", + "It starts at sign-in, not at boot, and it stops when you sign out.", +].join("\n"); + +/** Shared by install and update. The Linux output is unchanged. */ +const reportReconcileResult = Effect.fn("cli.service.report")(function* ( + result: ServiceReconcileResult, + unchangedMessage: string, +) { + if (!result.changed) { + yield* Console.log(unchangedMessage); + return; + } + yield* Console.log( + `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`, + ); + if ((yield* HostProcessPlatform) === "win32") { + yield* Console.log(WINDOWS_SERVICE_NOTICE); + } +}); + const serviceInstallCommand = Command.make("install", projectLocationFlags).pipe( Command.withDescription("Install T3 Code as a background service for this user."), Command.withHandler((flags) => @@ -78,14 +114,9 @@ const serviceInstallCommand = Command.make("install", projectLocationFlags).pipe flags, Effect.gen(function* () { const result = yield* reconcileService(); - if (!result.changed) { - yield* Console.log( - `T3 Code service is already installed with t3@${packageJson.version}.`, - ); - return; - } - yield* Console.log( - `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`, + yield* reportReconcileResult( + result, + `T3 Code service is already installed with t3@${packageJson.version}.`, ); }), ), @@ -101,12 +132,9 @@ const serviceUpdateCommand = Command.make("update", projectLocationFlags).pipe( flags, Effect.gen(function* () { const result = yield* reconcileService(); - if (!result.changed) { - yield* Console.log(`T3 Code service is already using t3@${packageJson.version}.`); - return; - } - yield* Console.log( - `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`, + yield* reportReconcileResult( + result, + `T3 Code service is already using t3@${packageJson.version}.`, ); }), ), @@ -152,12 +180,18 @@ export const offerServiceDuringOnboarding = Effect.gen(function* () { yield* Console.log("T3 Code is already set up to run in the background on this machine."); return true; } + // Windows starts the service at sign-in, not at boot, and it stops at + // sign-out. Promising otherwise here would be a lie. + const windows = (yield* HostProcessPlatform) === "win32"; const wanted = yield* Prompt.run( Prompt.confirm({ message: installed ? "The installed T3 Code service needs an update or repair. Update it now?" - : "Run T3 Code in the background whenever this machine boots? " + - "It stays reachable through T3 Connect even after you log out.", + : windows + ? "Run T3 Code in the background whenever you sign in to Windows? " + + "It starts again after every reboot, and stops when you sign out." + : "Run T3 Code in the background whenever this machine boots? " + + "It stays reachable through T3 Connect even after you log out.", initial: true, }), ); @@ -169,6 +203,7 @@ export const offerServiceDuringOnboarding = Effect.gen(function* () { yield* Console.log( `Background service ${result.previouslyInstalled ? "updated" : "installed"}. Logs: ${result.plan.logPath}`, ); + if (windows) yield* Console.log(WINDOWS_SERVICE_NOTICE); } return true; }); @@ -187,6 +222,12 @@ export const recoverServiceOnboardingOffer = ( Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), BootServiceUpdatePendingError: (error) => Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), + // Both carry guidance the user has to act on, so print the message itself + // rather than the generic "did not finish" wrapper. + BootServicePathHasPercentError: (error) => + Console.warn(`Skipping background setup: ${error.message}`).pipe(Effect.as(false)), + BootServiceStartupEntryDisabledError: (error) => + Console.warn(`Skipping background setup: ${error.message}`).pipe(Effect.as(false)), }), ); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 7eef6feba50..a1a13c5708c 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -88,7 +88,7 @@ export class BootServiceUnsupportedError extends Schema.TaggedErrorClass()( + "BootServicePathHasPercentError", + { pathLabel: Schema.String }, +) { + override get message(): string { + // The label carries what to change. Naming T3CODE_HOME here would be wrong + // advice for the Node executable, which that variable cannot move. + return ( + `The path to ${this.pathLabel} contains a percent sign, and the Windows command ` + + "shell would rewrite it before the service could start. Move it to a path without " + + "one, then run this again." + ); + } +} + +/** Windows only. The Startup entry exists but Windows Settings has it switched off. */ +export class BootServiceStartupEntryDisabledError extends Schema.TaggedErrorClass()( + "BootServiceStartupEntryDisabledError", + { shortcutName: Schema.String }, +) { + override get message(): string { + return ( + `"${this.shortcutName}" is switched off under Startup apps in Windows Settings. ` + + "Turn it back on, then run this again." + ); + } +} + export type BootServiceError = | BootServiceUnsupportedError | BootServiceCommandError | BootServiceInstallError - | BootServiceUpdatePendingError; + | BootServiceUpdatePendingError + | BootServicePathHasPercentError + | BootServiceStartupEntryDisabledError; + +/** + * Which backend answered. The name carries the platform on purpose: a bare + * "shortcut" would read as a macOS or Linux desktop entry just as easily. + */ +export type BootServiceKind = "systemd" | "win32-startup-shortcut"; export interface BootServiceStatus { readonly supported: boolean; readonly installed: boolean; readonly current: boolean; + readonly kind: BootServiceKind; + /** The systemd unit on Linux, the Startup folder shortcut on Windows. */ readonly unitPath: string; readonly logPath: string; } @@ -382,11 +433,12 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { }).pipe(Effect.withSpan("cloud.boot_service.uninstall")); const status: BootService["Service"]["status"] = Effect.gen(function* () { + const base = { kind: "systemd", unitPath, logPath } as const; if (platform !== "linux" || homeDir === "") { - return { supported: false, installed: false, current: false, unitPath, logPath }; + return { supported: false, installed: false, current: false, ...base }; } if (!(yield* fs.exists(unitPath))) { - return { supported: true, installed: false, current: false, unitPath, logPath }; + return { supported: true, installed: false, current: false, ...base }; } const [unit, launcherExists, runtimeEntryExists, runtimeSentinel, stateText] = yield* Effect.all([ @@ -408,8 +460,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { runtimeSentinel.value.trim() === input.cliVersion && state?.activeVersion === input.cliVersion && state?.update?.status !== "pending", - unitPath, - logPath, + ...base, }; }).pipe( Effect.mapError((cause) => new BootServiceInstallError({ cause })), diff --git a/apps/server/src/cloud/bootServiceWindows.test.ts b/apps/server/src/cloud/bootServiceWindows.test.ts new file mode 100644 index 00000000000..fa597273fbe --- /dev/null +++ b/apps/server/src/cloud/bootServiceWindows.test.ts @@ -0,0 +1,489 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { + HostProcessArguments, + HostProcessExecutablePath, + HostProcessPlatform, +} from "@t3tools/shared/hostProcess"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; +import * as NodeOS from "node:os"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +import * as ProcessRunner from "../processRunner.ts"; +import * as BootServiceWindows from "./bootServiceWindows.ts"; +import { pinnedRuntimePaths } from "./pinnedRuntime.ts"; +import { + encodeServiceLauncherPresence, + parseServiceState, + SERVICE_LAUNCHER_PROTOCOL, + SERVICE_PID_FILE, +} from "./serviceProtocol.ts"; + +const plan = { + nodePath: "C:\\Program Files\\nodejs\\node.exe", + launcherPath: "C:\\Users\\me\\.t3\\runtime\\service-launcher.mjs", + baseDir: "C:\\Users\\me\\.t3", + logPath: "C:\\Users\\me\\.t3\\userdata\\logs\\boot-service.log", + unitPath: + "C:\\Users\\me\\AppData\\Roaming\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\T3 Code Server.lnk", + startupScriptPath: "C:\\Users\\me\\.t3\\runtime\\service-startup.ps1", + shortcutScriptPath: "C:\\Users\\me\\.t3\\runtime\\service-shortcut.ps1", +} as const; + +it("escapes a quote in a PowerShell literal by doubling it", () => { + expect(BootServiceWindows.quotePowerShellLiteral("C:\\Users\\o'brien\\.t3")).toBe( + "'C:\\Users\\o''brien\\.t3'", + ); +}); + +it("starts the launcher with no window and redirects both streams to the log", () => { + const script = BootServiceWindows.renderStartupScript(plan); + + expect(script).toContain("$startInfo.CreateNoWindow = $true"); + expect(script).toContain("$startInfo.UseShellExecute = $false"); + // Doubling the outer quote is what carries a fully quoted command through cmd. + expect(script).toContain( + `/c ""${plan.nodePath}" "${plan.launcherPath}" >> "${plan.logPath}" 2>&1"`, + ); +}); + +it("tells the launcher to supervise itself, because nothing else will", () => { + expect(BootServiceWindows.renderStartupScript(plan)).toContain( + "$env:T3_SERVICE_SELF_SUPERVISE = '1'", + ); +}); + +it("names the window so the blink at sign-in does not look like malware", () => { + const script = BootServiceWindows.renderStartupScript(plan); + const shortcut = BootServiceWindows.renderShortcutScript(plan); + + expect(script).toContain("$Host.UI.RawUI.WindowTitle = 'T3 Code Server'"); + expect(shortcut).toContain("$shortcut.Description = 'T3 Code Server, started at sign-in'"); +}); + +it("points the shortcut at a hidden PowerShell running the startup script", () => { + const shortcut = BootServiceWindows.renderShortcutScript(plan); + + expect(shortcut).toContain("System32\\WindowsPowerShell\\v1.0\\powershell.exe"); + expect(shortcut).toContain(`-WindowStyle Hidden -File "${plan.startupScriptPath}"`); + expect(shortcut).toContain("$shortcut.WindowStyle = 7"); +}); + +it("has the probe report the shortcut it finds, and the disabled flag only with it", () => { + // The harness fake produces these lines itself, so without this the real + // script could stop emitting them and every test would still pass while + // Windows reported "needs repair" forever. + const shortcut = BootServiceWindows.renderShortcutScript(plan); + + expect(shortcut).toContain('Write-Output "target=$($existing.TargetPath)"'); + expect(shortcut).toContain('Write-Output "arguments=$($existing.Arguments)"'); + // A disabled flag outlives the shortcut, so it is only read alongside one. + expect(shortcut).toContain("$present = Test-Path -LiteralPath $shortcutPath"); + expect(shortcut).toContain("if ($present -and (Test-Path -LiteralPath $key)) {"); +}); + +it("reads both findings out of the probe output", () => { + expect(BootServiceWindows.parseProbeOutput("shell=True\r\ndisabled=False\r\n")).toEqual({ + shellRunning: true, + entryDisabled: false, + }); + expect(BootServiceWindows.parseProbeOutput("shell=False\ndisabled=True\n")).toEqual({ + shellRunning: false, + entryDisabled: true, + }); +}); + +it("refuses to guess when the probe output is unreadable", () => { + // Defaulting to false would read as "the entry is not disabled", so a probe + // that never ran would look exactly like one that passed. + expect(BootServiceWindows.parseProbeOutput("")).toBeUndefined(); + expect(BootServiceWindows.parseProbeOutput("shell=True\n")).toBeUndefined(); +}); + +it("spots a percent sign, which the command shell would rewrite silently", () => { + expect( + BootServiceWindows.findPercentInPaths([ + ["the Node executable", "C:\\node.exe"], + ["the data directory", "C:\\pct %TEMP% dir\\launcher.mjs"], + ]), + ).toBe("the data directory"); + expect( + BootServiceWindows.findPercentInPaths([["the Node executable", "C:\\node.exe"]]), + ).toBeUndefined(); +}); + +const makeHarness = Effect.fn("test.make_windows_boot_service_harness")(function* ( + platform: NodeJS.Platform = "win32", + execPath = "C:\\node.exe", +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boot-service-win-test-" }); + const appData = path.join(home, "AppData", "Roaming"); + const baseDir = path.join(home, ".t3"); + const sourceLauncher = path.join(home, "service-launcher.mjs"); + const runtimeDir = path.join(baseDir, "runtime"); + const statePath = path.join(runtimeDir, "service-state.json"); + const startupScriptPath = path.join(runtimeDir, "service-startup.ps1"); + const shortcutPath = path.join( + appData, + "Microsoft", + "Windows", + "Start Menu", + "Programs", + "Startup", + BootServiceWindows.SHORTCUT_FILE, + ); + yield* fs.writeFileString(sourceLauncher, "export {};\n"); + const runtime = pinnedRuntimePaths(path, baseDir, "1.2.3"); + yield* fs.makeDirectory(path.dirname(runtime.entryPath), { recursive: true }); + yield* fs.writeFileString(runtime.entryPath, "export {};\n"); + yield* fs.writeFileString(runtime.sentinelPath, "1.2.3\n"); + + const pidPath = path.join(runtimeDir, SERVICE_PID_FILE); + const stopRequestPath = path.join(runtimeDir, ".service-stop-request"); + const commands: string[] = []; + const powershell = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"; + const expectedArguments = BootServiceWindows.expectedShortcutArguments({ + ...plan, + startupScriptPath, + }); + const control = { + shell: "True", + disabled: "False", + failCommand: undefined as string | undefined, + /** Set to false to model a launcher that never answers the stop request. */ + launcherStopsOnRequest: true, + /** Set to false to model a record whose process id was recycled. */ + launcherRefreshesRecord: true, + /** Rewritten by the stand-in launcher to prove it is still alive. */ + presence: "", + /** What the fake shortcut currently points at, so drift can be modelled. */ + shortcutTarget: powershell, + shortcutArguments: expectedArguments, + }; + // The fake stands in for PowerShell: creating and removing the shortcut is a + // plain file write here, which is all the module observes. + const runner = ProcessRunner.ProcessRunner.of({ + run: (input) => + Effect.gen(function* () { + const command = `${input.command} ${input.args.join(" ")}`; + commands.push(command); + const action = input.args.at(-1); + if (command.includes("service-startup")) { + // Standing in for the logon script: a launcher starts and claims the + // pid file, exactly as the real one does before anything else. The + // boot stamp matters, since a record from an earlier boot is ignored. + const bootTimeMs = DateTime.toEpochMillis(yield* DateTime.now) - NodeOS.uptime() * 1_000; + yield* fs.makeDirectory(runtimeDir, { recursive: true }); + control.presence = encodeServiceLauncherPresence({ pid: process.pid, bootTimeMs }); + yield* fs.writeFileString(pidPath, control.presence); + } + if (command.includes("service-shortcut")) { + if (action === "Install") { + yield* fs.makeDirectory(path.dirname(shortcutPath), { recursive: true }); + yield* fs.writeFileString(shortcutPath, "shortcut"); + } + if (action === "Remove") { + yield* fs.remove(shortcutPath, { force: true }); + } + } + return { + stdout: + action === "Probe" + ? `shell=${control.shell}\ndisabled=${control.disabled}\n` + + ((yield* fs.exists(shortcutPath).pipe(Effect.orElseSucceed(() => false))) + ? `target=${control.shortcutTarget}\narguments=${control.shortcutArguments}\n` + : "") + : input.args[1] === "--version" + ? "t3 v1.2.3\n" + : "", + stderr: "", + code: ChildProcessSpawner.ExitCode(command === control.failCommand ? 1 : 0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }; + }).pipe(Effect.orDie), + }); + + const service = yield* BootServiceWindows.make({ + baseDir, + logsDir: path.join(baseDir, "userdata", "logs"), + cliVersion: "1.2.3", + host: { execPath, launcherSourcePath: sourceLauncher }, + stopRequestTimeout: Duration.millis(30), + }).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.provide( + Layer.mergeAll( + Layer.succeed(HostProcessPlatform, platform), + Layer.succeed(HostProcessExecutablePath, "C:\\node.exe"), + Layer.succeed(HostProcessArguments, ["C:\\node.exe", path.join(home, "bin.mjs")]), + ConfigProvider.layer( + ConfigProvider.fromEnv({ env: { APPDATA: appData, SystemRoot: "C:\\Windows" } }), + ), + ), + ), + ); + // Stands in for a running launcher. The real one drops its pid file when it + // stops, and that disappearance is the only thing the CLI accepts as proof. + yield* Effect.forkScoped( + Effect.gen(function* () { + while (true) { + yield* Effect.sleep(Duration.millis(5)); + const asked = yield* fs.exists(stopRequestPath).pipe(Effect.orElseSucceed(() => false)); + if (!control.launcherStopsOnRequest) { + // A launcher that will not stop is still alive, and a live one keeps + // refreshing its record. Without that it would look like a process id + // the system recycled, which is a different case entirely. + const present = yield* fs.exists(pidPath).pipe(Effect.orElseSucceed(() => false)); + if (control.launcherRefreshesRecord && present) { + yield* fs.writeFileString(pidPath, control.presence).pipe(Effect.ignore); + } + continue; + } + if (!asked) continue; + yield* fs.remove(stopRequestPath, { force: true }).pipe(Effect.ignore); + yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore); + } + }), + ); + + return { + service, + fs, + stopRequestPath, + statePath, + startupScriptPath, + shortcutPath, + pidPath, + commands, + control, + }; +}); + +it.layer(NodeServices.layer)("windows boot service", (it) => { + it.effect("installs, reports current state, and uninstalls", () => + Effect.gen(function* () { + const { service, fs, statePath, shortcutPath } = yield* makeHarness(); + const installed = yield* service.install; + + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + }); + expect(yield* fs.readFileString(installed.launcherPath)).toBe("export {};\n"); + expect(yield* fs.exists(shortcutPath)).toBe(true); + expect((yield* service.status).current).toBe(true); + expect((yield* service.status).kind).toBe("win32-startup-shortcut"); + + expect(yield* service.uninstall).toBe(true); + expect((yield* service.status).installed).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("creates the shortcut only after the Startup folder passes its probe", () => + Effect.gen(function* () { + const { service, commands } = yield* makeHarness(); + yield* service.install; + + const actions = commands + .filter((command) => command.includes("service-shortcut.ps1")) + .map((command) => command.split(" ").at(-1)); + expect(actions).toEqual(["Probe", "Install"]); + }).pipe(TestClock.withLive), + ); + + it.effect("goes stale when the startup script drifts from what it should be", () => + Effect.gen(function* () { + const { service, fs, startupScriptPath } = yield* makeHarness(); + yield* service.install; + expect((yield* service.status).current).toBe(true); + + yield* fs.writeFileString(startupScriptPath, "# edited by hand\n"); + expect((yield* service.status).current).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("refuses to install over an entry switched off in Windows Settings", () => + Effect.gen(function* () { + const { service, control } = yield* makeHarness(); + control.disabled = "True"; + + const error = yield* service.install.pipe(Effect.flip); + // A dedicated tag, because the generic install error hides its cause and + // the CLI prints only `message`. The guidance has to reach the user. + expect(error._tag).toBe("BootServiceStartupEntryDisabledError"); + expect(error.message).toContain("Startup apps in Windows Settings"); + }).pipe(TestClock.withLive), + ); + + it.effect("reinstalls over a running launcher only after it has actually gone", () => + Effect.gen(function* () { + const { service, fs, pidPath } = yield* makeHarness(); + yield* service.install; + // The stand-in launcher claimed the pid file when the startup script ran. + expect(yield* fs.exists(pidPath)).toBe(true); + + yield* service.install; + expect((yield* service.status).current).toBe(true); + }).pipe(TestClock.withLive), + ); + + it.effect("refuses to reinstall when the running launcher will not stop", () => + Effect.gen(function* () { + const { service, control } = yield* makeHarness(); + yield* service.install; + // Writing over a launcher we cannot confirm dead would leave two servers + // on one database, so this must fail rather than carry on. + control.launcherStopsOnRequest = false; + + const error = yield* service.install.pipe(Effect.flip); + expect(error._tag).toBe("BootServiceCommandError"); + expect(error.message).toContain("did not exit within"); + }).pipe(TestClock.withLive), + ); + + it.effect("removes the shortcut without needing the generated script", () => + Effect.gen(function* () { + const { service, fs, shortcutPath, startupScriptPath } = yield* makeHarness(); + yield* service.install; + // A user who cleared the runtime directory still needs uninstall to work, + // because the shortcut is the thing they asked to be rid of. + yield* fs.remove(startupScriptPath, { force: true }); + + expect(yield* service.uninstall).toBe(true); + expect(yield* fs.exists(shortcutPath)).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("goes stale when the shortcut itself is edited, not just the script", () => + Effect.gen(function* () { + const { service, control } = yield* makeHarness(); + yield* service.install; + expect((yield* service.status).current).toBe(true); + + // The generated scripts still match. Only the .lnk was tampered with, and + // that is the thing sign-in actually runs. + control.shortcutArguments = "-File C:\\somewhere\\else.ps1"; + expect((yield* service.status).current).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("stops a running launcher even when the shortcut was deleted by hand", () => + Effect.gen(function* () { + const { service, fs, shortcutPath, pidPath } = yield* makeHarness(); + yield* service.install; + // A launcher outlives its shortcut, so keying the stop on the shortcut + // would leave a server running with nothing left to manage it. + yield* fs.remove(shortcutPath, { force: true }); + + expect(yield* service.uninstall).toBe(true); + expect(yield* fs.exists(pidPath)).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("names the offending path when it holds a percent sign", () => + Effect.gen(function* () { + const { service } = yield* makeHarness("win32", "C:\\pct %TEMP% dir\\node.exe"); + + const error = yield* service.install.pipe(Effect.flip); + // The generic install error prints fixed text, so a dedicated tag is what + // gets the user told which path to move. + expect(error._tag).toBe("BootServicePathHasPercentError"); + expect(error.message).toContain("percent sign"); + }).pipe(TestClock.withLive), + ); + + it.effect("ignores a pid record left behind by an earlier boot", () => + Effect.gen(function* () { + const { service, fs, pidPath } = yield* makeHarness(); + yield* service.install; + // Process ids are recycled across reboots, so a record from a previous + // boot can name a live stranger. Trusting it would block every install. + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ pid: process.pid, bootTimeMs: 0 }), + ); + + yield* service.install; + expect((yield* service.status).current).toBe(true); + }).pipe(TestClock.withLive), + ); + + it.effect("clears a leftover stop request when no launcher is listening", () => + Effect.gen(function* () { + const { service, fs, pidPath, stopRequestPath } = yield* makeHarness(); + yield* service.install; + // A dead launcher leaves both files. Removing only the pid record would + // let the request stop the very launcher the next install starts. + yield* fs.remove(pidPath, { force: true }); + yield* fs.writeFileString(stopRequestPath, ""); + + yield* service.install; + expect(yield* fs.exists(stopRequestPath)).toBe(false); + }).pipe(TestClock.withLive), + ); + + it.effect("refuses to reinstall while a remote update is still pending", () => + Effect.gen(function* () { + const { service, fs, statePath, pidPath } = yield* makeHarness(); + yield* service.install; + // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document. + const pendingState = JSON.stringify({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + update: { + id: "u", + fromVersion: "1.2.3", + targetVersion: "1.2.4", + dbPath: "C:\\state.sqlite", + status: "pending", + }, + }); + yield* fs.writeFileString(statePath, pendingState); + + const error = yield* service.install.pipe(Effect.flip); + expect(error._tag).toBe("BootServiceUpdatePendingError"); + // The guard protects the in-flight update, so it must refuse before + // anything is terminated. A surviving pid record proves nothing stopped. + expect(yield* fs.exists(pidPath)).toBe(true); + }).pipe(TestClock.withLive), + ); + + it.effect("ignores a pid record nobody has refreshed", () => + Effect.gen(function* () { + const { service, fs, pidPath, control } = yield* makeHarness(); + yield* service.install; + // Nothing answers a stop request now, and nothing refreshes the record + // either. That pair is what a recycled process id looks like, and it is + // what makes the assertion below mean something. + control.launcherStopsOnRequest = false; + control.launcherRefreshesRecord = false; + // Same boot and a live process id, but the launcher stopped refreshing + // it. Within one boot that is the only way to tell a recycled id apart. + // Seconds, not milliseconds: utimes reads a bare number as seconds. + const longAgoSeconds = 1_577_836_800; // 2020-01-01 + yield* fs.utimes(pidPath, longAgoSeconds, longAgoSeconds); + + yield* service.install; + expect((yield* service.status).current).toBe(true); + }).pipe(TestClock.withLive), + ); + + it.effect("fails closed off Windows", () => + Effect.gen(function* () { + const { service } = yield* makeHarness("linux"); + expect((yield* service.status).supported).toBe(false); + expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUnsupportedError"); + }).pipe(TestClock.withLive), + ); +}); diff --git a/apps/server/src/cloud/bootServiceWindows.ts b/apps/server/src/cloud/bootServiceWindows.ts new file mode 100644 index 00000000000..d9694a63c5b --- /dev/null +++ b/apps/server/src/cloud/bootServiceWindows.ts @@ -0,0 +1,741 @@ +/** + * Windows boot service. Everything in this file is the Windows path. + * + * Windows has no systemd, so the equivalent of the user unit is a shortcut in + * the per-user Startup folder. The shell (`explorer.exe`) runs that shortcut at + * every sign-in. The shortcut points at PowerShell, PowerShell spawns the + * launcher with its console hidden, then PowerShell exits. + * + * Three differences from the Linux path are deliberate and load bearing: + * + * - Nothing supervises the launcher, so the launcher restarts its own child. + * The generated logon script sets `SERVICE_SELF_SUPERVISE_ENV` to say so. + * - Nothing captures the launcher's output, so the logon script redirects it + * through the command shell (`cmd.exe`) rather than the launcher opening it. + * - Windows has no SIGTERM, so stopping goes through a request file that the + * launcher watches. That stop is not graceful: the child is terminated + * without running its shutdown finalizer. + */ +import { HostProcessExecutablePath, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Config from "effect/Config"; +import * as Console from "effect/Console"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as NodeOS from "node:os"; + +import * as ProcessRunner from "../processRunner.ts"; +import * as BootService from "./bootService.ts"; +import { + ensurePinnedRuntimeInstalled, + pinnedRuntimePaths, + PinnedRuntimeInstallError, +} from "./pinnedRuntime.ts"; +import { + SERVICE_LAUNCHER_FILE, + SERVICE_LAUNCHER_PROTOCOL, + SERVICE_PID_FILE, + SERVICE_STATE_FILE, + decodeServiceLauncherPresence, + serviceLauncherPresenceIsFromThisBoot, + SERVICE_STOP_REQUEST_FILE, + parseServiceState, + serviceStateHasPendingUpdate, + type ServiceState, +} from "./serviceProtocol.ts"; + +/** Windows only. The shortcut basename doubles as the title of the window that + blinks once at sign-in, because a console launched from a shortcut takes its + title from the shortcut. "Server" rather than "Code" because the desktop app + is a separate thing the user may also start. */ +export const SHORTCUT_NAME = "T3 Code Server"; +export const SHORTCUT_FILE = `${SHORTCUT_NAME}.lnk`; +const STARTUP_SCRIPT_FILE = "service-startup.ps1"; +const SHORTCUT_SCRIPT_FILE = "service-shortcut.ps1"; + +/** + * Windows only. How long the CLI waits for a running launcher to shut down. + * + * It must comfortably exceed the launcher's own worst case. That is a restart + * backoff (5s, now interruptible) plus one watch interval (2s), so 30s leaves + * room for a slow database backup holding the launcher's transition queue. + */ +const STOP_REQUEST_TIMEOUT = Duration.seconds(30); +/** Windows only. How often the CLI re-checks whether the launcher has gone. */ +const STOP_REQUEST_ACK_POLL = Duration.millis(250); +const POWERSHELL_TIMEOUT = Duration.seconds(30); + +/** + * Windows only. The absolute interpreter path, rather than trusting PATH. + * The shortcut has to spell it out anyway, so every call site uses the same one. + */ +const powershellPath = (systemRoot: string) => + `${systemRoot}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`; + +/** + * Windows only. The command shell expands `%VAR%` on its command line, even + * inside double quotes, and there is no escape that works there. A path holding + * a percent sign would therefore be rewritten before the launcher ever starts, + * silently, with a successful install and nothing running. The Linux renderer + * guards the same hazard for systemd in `escapeSystemdSpecifiers`; here the only + * honest option is to refuse. + */ +export function findPercentInPaths( + paths: ReadonlyArray, +): string | undefined { + const offender = paths.find(([, value]) => value.includes("%")); + return offender === undefined ? undefined : offender[0]; +} + +/** + * Windows only. Signal 0 asks the kernel whether a process exists without + * touching it. Node implements it on Windows too. A permission error means the + * process is there but not ours, which still counts as running. + */ +const processIsAlive = (pid: number): Effect.Effect => + Effect.sync(() => { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException | undefined)?.code === "EPERM"; + } + }); + +/** Windows only. Milliseconds since the epoch when this machine last booted. */ +const currentBootTimeMs = DateTime.now.pipe( + Effect.map((now) => DateTime.toEpochMillis(now) - NodeOS.uptime() * 1_000), +); + +/** Windows only. PowerShell single-quoted strings escape a quote by doubling it. */ +export function quotePowerShellLiteral(value: string): string { + return `'${value.replaceAll("'", "''")}'`; +} + +export interface WindowsBootServicePlan extends BootService.BootServicePlan { + readonly startupScriptPath: string; + readonly shortcutScriptPath: string; +} + +/** Windows only. What the preflight learned about this machine. */ +export interface WindowsPreflight { + readonly shellRunning: boolean; + readonly entryDisabled: boolean; + /** Absent when no shortcut exists yet. */ + readonly shortcutTarget?: string; + readonly shortcutArguments?: string; +} + +/** Windows only. What the shortcut must point at for the service to work. */ +export function expectedShortcutArguments(plan: WindowsBootServicePlan): string { + return `-NoLogo -NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden -File "${plan.startupScriptPath}"`; +} + +/** + * Windows only. Pure renderer for the script the shortcut runs at sign-in. + * + * The command shell does the redirect because PowerShell cannot append, and + * cannot send both streams to the same file. Doubling the outer quote is the + * documented way to pass a fully quoted command line through `cmd /c`. + */ +export function renderStartupScript(plan: WindowsBootServicePlan): string { + const command = `""${plan.nodePath}" "${plan.launcherPath}" >> "${plan.logPath}" 2>&1"`; + return [ + "# Generated by `t3 service install`. Do not edit; it is rewritten on update.", + "$ErrorActionPreference = 'Stop'", + "# Belt and braces. The blink is usually titled from the shortcut name.", + `try { $Host.UI.RawUI.WindowTitle = ${quotePowerShellLiteral(SHORTCUT_NAME)} } catch { }`, + `$env:T3CODE_HOME = ${quotePowerShellLiteral(plan.baseDir)}`, + "# Nothing supervises a Startup folder entry, so the launcher supervises itself.", + "$env:T3_SERVICE_SELF_SUPERVISE = '1'", + `$logDir = Split-Path -Parent ${quotePowerShellLiteral(plan.logPath)}`, + "if (-not (Test-Path -LiteralPath $logDir)) {", + " New-Item -ItemType Directory -Path $logDir -Force | Out-Null", + "}", + "$startInfo = New-Object System.Diagnostics.ProcessStartInfo", + "$startInfo.FileName = $env:ComSpec", + `$startInfo.Arguments = ${quotePowerShellLiteral(`/c ${command}`)}`, + "$startInfo.UseShellExecute = $false", + "# The whole point: the launcher runs with no console window at all.", + "$startInfo.CreateNoWindow = $true", + `$startInfo.WorkingDirectory = ${quotePowerShellLiteral(plan.baseDir)}`, + "[System.Diagnostics.Process]::Start($startInfo) | Out-Null", + "", + ].join("\n"); +} + +/** + * Windows only. Pure renderer for the script that probes, creates and removes + * the shortcut. One script owns every shortcut interaction so there is a single + * place to read when the Startup entry misbehaves. + */ +export function renderShortcutScript(plan: WindowsBootServicePlan): string { + return [ + "# Generated by `t3 service install`. Do not edit; it is rewritten on update.", + "param([Parameter(Mandatory = $true)][ValidateSet('Probe', 'Install')][string]$Action)", + "$ErrorActionPreference = 'Stop'", + `$shortcutPath = ${quotePowerShellLiteral(plan.unitPath)}`, + `$shortcutFile = ${quotePowerShellLiteral(SHORTCUT_FILE)}`, + "", + "if ($Action -eq 'Probe') {", + " # The shell is what runs Startup folder entries. A different shell is a", + " # deliberate choice, so this is reported and not treated as a failure.", + " $shell = @(Get-Process -Name 'explorer' -ErrorAction SilentlyContinue).Count -gt 0", + " # Windows records entries disabled through Settings here. The low bit of", + " # the first byte is the disabled flag. It only means anything while the", + " # shortcut exists, because the value outlives the .lnk. Reporting a stale", + " # one sends the user hunting for an entry Startup apps no longer shows.", + " $key = 'HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\StartupApproved\\StartupFolder'", + " $disabled = $false", + " $present = Test-Path -LiteralPath $shortcutPath", + " if ($present -and (Test-Path -LiteralPath $key)) {", + " $value = (Get-ItemProperty -LiteralPath $key -Name $shortcutFile -ErrorAction SilentlyContinue).$shortcutFile", + " if ($null -ne $value -and $value.Length -gt 0) { $disabled = ($value[0] -band 1) -eq 1 }", + " }", + ' Write-Output "shell=$shell"', + ' Write-Output "disabled=$disabled"', + " # Read the shortcut back. That the script which wrote it still matches", + " # says nothing about a .lnk somebody edited or replaced since.", + " if ($present) {", + " $existing = (New-Object -ComObject WScript.Shell).CreateShortcut($shortcutPath)", + ' Write-Output "target=$($existing.TargetPath)"', + ' Write-Output "arguments=$($existing.Arguments)"', + " }", + " exit 0", + "}", + "", + "$startupDir = Split-Path -Parent $shortcutPath", + "if (-not (Test-Path -LiteralPath $startupDir)) {", + " New-Item -ItemType Directory -Path $startupDir -Force | Out-Null", + "}", + "$shell = New-Object -ComObject WScript.Shell", + "$shortcut = $shell.CreateShortcut($shortcutPath)", + "$shortcut.TargetPath = Join-Path $env:SystemRoot 'System32\\WindowsPowerShell\\v1.0\\powershell.exe'", + `$shortcut.Arguments = ${quotePowerShellLiteral(expectedShortcutArguments(plan))}`, + "$shortcut.WorkingDirectory = $env:USERPROFILE", + `$shortcut.Description = ${quotePowerShellLiteral(`${SHORTCUT_NAME}, started at sign-in`)}`, + "# Minimized, because PowerShell paints a console before it hides itself.", + "$shortcut.WindowStyle = 7", + "$shortcut.Save()", + "", + ].join("\n"); +} + +/** + * Windows only. Parses the two lines the Probe action writes. + * + * Returns undefined when either line is missing. Defaulting to false would read + * as "the entry is not disabled", so a probe that failed to run would look + * exactly like a probe that passed, and the install would sail past a check it + * never actually performed. + */ +export function parseProbeOutput(stdout: string): WindowsPreflight | undefined { + const read = (key: string) => { + const match = new RegExp(`^${key}=(True|False)\\s*$`, "im").exec(stdout)?.[1]; + return match === undefined ? undefined : match === "True"; + }; + const readText = (key: string) => new RegExp(`^${key}=(.*)$`, "im").exec(stdout)?.[1]?.trim(); + const shellRunning = read("shell"); + const entryDisabled = read("disabled"); + if (shellRunning === undefined || entryDisabled === undefined) return undefined; + const shortcutTarget = readText("target"); + const shortcutArguments = readText("arguments"); + return { + shellRunning, + entryDisabled, + ...(shortcutTarget === undefined ? {} : { shortcutTarget }), + ...(shortcutArguments === undefined ? {} : { shortcutArguments }), + }; +} + +export interface WindowsBootServiceInput { + readonly baseDir: string; + readonly logsDir: string; + readonly cliVersion: string; + readonly host?: BootService.BootServiceHost; + /** Overridable so tests do not sit through the real wait. */ + readonly stopRequestTimeout?: Duration.Duration; +} + +export const make = Effect.fn("cloud.boot_service_windows.make")(function* ( + input: WindowsBootServiceInput, +) { + const hostExecPath = yield* HostProcessExecutablePath; + const platform = yield* HostProcessPlatform; + const appData = yield* Config.string("APPDATA").pipe(Config.withDefault("")); + const systemRoot = yield* Config.string("SystemRoot").pipe(Config.withDefault("C:\\Windows")); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const runner = yield* ProcessRunner.ProcessRunner; + const host = input.host ?? { execPath: hostExecPath }; + + const startupDir = path.join( + appData, + "Microsoft", + "Windows", + "Start Menu", + "Programs", + "Startup", + ); + const unitPath = path.join(startupDir, SHORTCUT_FILE); + const logPath = path.join(input.logsDir, "boot-service.log"); + const runtimeDir = path.join(input.baseDir, "runtime"); + const launcherPath = path.join(runtimeDir, SERVICE_LAUNCHER_FILE); + const statePath = path.join(runtimeDir, SERVICE_STATE_FILE); + const stopRequestPath = path.join(runtimeDir, SERVICE_STOP_REQUEST_FILE); + const pidPath = path.join(runtimeDir, SERVICE_PID_FILE); + const startupScriptPath = path.join(runtimeDir, STARTUP_SCRIPT_FILE); + const shortcutScriptPath = path.join(runtimeDir, SHORTCUT_SCRIPT_FILE); + const runtimePaths = pinnedRuntimePaths(path, input.baseDir, input.cliVersion); + const launcherSourcePath = + host.launcherSourcePath ?? + path.join(path.dirname(runtimePaths.entryPath), SERVICE_LAUNCHER_FILE); + + const plan: WindowsBootServicePlan = { + nodePath: host.execPath, + launcherPath, + baseDir: input.baseDir, + logPath, + unitPath, + startupScriptPath, + shortcutScriptPath, + }; + + /** + * Windows only. A copy of the Linux durable write, minus its final directory + * sync. Windows cannot open a directory as a file, so that sync always fails + * here. Every other step is identical on purpose. + */ + const writeDurably = (filePath: string, contents: string) => + Effect.scoped( + Effect.gen(function* () { + const directory = path.dirname(filePath); + yield* fs.makeDirectory(directory, { recursive: true }); + const tempPath = yield* fs.makeTempFileScoped({ directory, prefix: ".service-write-" }); + yield* fs.writeFileString(tempPath, contents, { mode: 0o600 }); + yield* (yield* fs.open(tempPath, { flag: "r" })).sync; + yield* fs.rename(tempPath, filePath); + }), + ).pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + + const requireWindows = Effect.gen(function* () { + if (platform !== "win32" || appData === "") { + return yield* new BootService.BootServiceUnsupportedError({ platform }); + } + }); + + /** Windows only. A copy of the Linux step runner, duplicated for the same + reason as the pinned-runtime step below: extracting it would restructure + the systemd path. Fix bugs here and in bootService.ts together. */ + const runStep = Effect.fn("cloud.boot_service_windows.run_step")(function* ( + step: string, + command: string, + args: ReadonlyArray, + options?: { readonly timeout?: Duration.Input }, + ) { + return yield* runner.run({ command, args, timeout: options?.timeout }).pipe( + Effect.mapError((cause) => new BootService.BootServiceCommandError({ step, cause })), + Effect.filterOrFail( + (result) => result.code === 0, + (result) => + new BootService.BootServiceCommandError({ + step, + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ), + Effect.tapError((error) => + DateTime.now.pipe( + Effect.flatMap((now) => + fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, { + flag: "a", + }), + ), + Effect.ignore, + ), + ), + ); + }); + + const powershell = powershellPath(systemRoot); + + const runPowerShellScript = ( + step: string, + scriptPath: string, + extraArgs: ReadonlyArray = [], + ) => + runStep( + step, + powershell, + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + scriptPath, + ...extraArgs, + ], + { timeout: POWERSHELL_TIMEOUT }, + ); + + const runShortcutScript = (step: string, action: "Probe" | "Install") => + runPowerShellScript(step, shortcutScriptPath, ["-Action", action]); + + /** + * Windows only. Proves the Startup folder is usable and reports what would + * silently stop the entry from ever running. The shortcut script must already + * be on disk, which is safe: it lives under the data directory, not the + * Startup folder, so nothing outside our own tree has been touched yet. + */ + const preflight: Effect.Effect< + WindowsPreflight, + | BootService.BootServiceUnsupportedError + | BootService.BootServiceInstallError + | BootService.BootServiceCommandError + > = Effect.gen(function* () { + yield* requireWindows; + yield* fs + .makeDirectory(startupDir, { recursive: true }) + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + // Prove it is writable now, rather than failing halfway through an install. + yield* Effect.scoped( + fs.makeTempFileScoped({ directory: startupDir, prefix: ".t3-service-probe-" }), + ).pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + + const probe = yield* runShortcutScript("checking the Windows Startup folder", "Probe"); + const findings = parseProbeOutput(probe.stdout); + if (findings === undefined) { + return yield* new BootService.BootServiceCommandError({ + step: "reading the Windows Startup folder check", + stdoutLength: probe.stdout.length, + stderrLength: probe.stderr.length, + }); + } + return findings; + }); + + /** + * Windows only. Asks a running launcher to stop, because Windows has no + * SIGTERM. The stop is not graceful: the launcher terminates its child + * without letting it run its shutdown finalizer. + * + * The pid file is what makes the answer unambiguous. Its absence means no + * launcher is running, and its disappearance means one stopped. Treating a + * vanished request file as proof instead would let a slow launcher look + * stopped, and the install would then rewrite the runtime underneath it and + * start a second server on the same database. + */ + const pidRecordMtime = fs.stat(pidPath).pipe( + Effect.map((info) => Option.map(info.mtime, (mtime) => mtime.getTime())), + Effect.orElseSucceed(() => Option.none()), + ); + + const requestStop = Effect.gen(function* () { + // Only a confirmed absence means no launcher. A transient read error, say a + // sharing violation while the launcher rewrites its record, must not be + // read as proof the launcher is gone, or the install writes over a live one. + const recorded = yield* fs.readFileString(pidPath).pipe( + Effect.map(Option.some), + Effect.catch((error) => + error.reason._tag === "NotFound" + ? Effect.succeed(Option.none()) + : Effect.fail(new BootService.BootServiceInstallError({ cause: error })), + ), + ); + if (Option.isNone(recorded)) return false; + const presence = decodeServiceLauncherPresence(recorded.value); + // Only trust the process id while the record is from this boot. Ids are + // recycled across reboots, so an old file can name an unrelated live + // process, and waiting on that would never finish. + // An old timestamp is not evidence of death. A laptop that slept wakes with + // a live launcher whose last heartbeat is ancient, and clearing its record + // would start a second server on one database. So this only asks whether the + // holder could be ours; the wait below settles whether it really is. + const recordMtimeBefore = yield* pidRecordMtime; + const live = + presence !== undefined && + serviceLauncherPresenceIsFromThisBoot(presence, yield* currentBootTimeMs) && + (yield* processIsAlive(presence.pid)); + if (!live || presence === undefined) { + // No launcher is listening. Clear the request too: leaving one behind + // would stop the very launcher this install is about to start. + yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore); + yield* fs.remove(stopRequestPath, { force: true }).pipe(Effect.ignore); + return false; + } + const pid = presence.pid; + + // A launcher is confirmed alive, so a failed write must fail the whole + // operation. Swallowing it would let the caller carry on and start a second + // server against the same database. + yield* fs + .writeFileString(stopRequestPath, "") + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + + const timeoutMs = Duration.toMillis(input.stopRequestTimeout ?? STOP_REQUEST_TIMEOUT); + const pollMs = Math.min(timeoutMs, Duration.toMillis(STOP_REQUEST_ACK_POLL)); + const attempts = Math.max(1, Math.ceil(timeoutMs / pollMs)); + for (let attempt = 0; attempt < attempts; attempt += 1) { + yield* Effect.sleep(Duration.millis(pollMs)); + const stillRunning = yield* fs.exists(pidPath).pipe(Effect.orElseSucceed(() => true)); + if (!stillRunning) return true; + } + // Nothing stopped. The wait was longer than several heartbeats, so a record + // that never moved belongs to a process that is not our launcher at all, + // which happens when the operating system recycles the id. Clear it and let + // the caller proceed. A record that did move is a launcher that is alive and + // simply will not stop, and writing over that one must not happen. + const recordMtimeAfter = yield* pidRecordMtime; + const neverRefreshed = + Option.isNone(recordMtimeAfter) || + (Option.isSome(recordMtimeBefore) && recordMtimeBefore.value === recordMtimeAfter.value); + if (neverRefreshed) { + yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore); + yield* fs.remove(stopRequestPath, { force: true }).pipe(Effect.ignore); + return false; + } + return yield* new BootService.BootServiceCommandError({ + step: "stopping the running service", + pid, + timeoutSeconds: Math.round(timeoutMs / 1000), + }); + }); + + /** Windows only. A copy of the Linux install's pinned-runtime step, which + validates the runtime by asking it for its own version. It is identical, + and it is duplicated rather than extracted because pulling it out of the + Linux `make` would restructure the systemd path. That path is deliberately + left alone. Fix bugs here and in bootService.ts together. */ + const installPinnedRuntime = ensurePinnedRuntimeInstalled({ + baseDir: input.baseDir, + version: input.cliVersion, + fs, + path, + runner, + validate: (runtime) => + runner + .run({ + command: host.execPath, + args: [runtime.entryPath, "--version"], + timeout: Duration.seconds(30), + }) + .pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ step: "verifying the pinned t3 runtime", cause }), + ), + Effect.flatMap((result) => { + const reportedVersion = /\bv(\S+)\s*$/.exec(result.stdout)?.[1]; + return result.code === 0 && reportedVersion === input.cliVersion + ? Effect.void + : Effect.fail( + new PinnedRuntimeInstallError({ + step: "verifying the pinned t3 runtime", + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ); + }), + ), + }).pipe( + Effect.mapError((error) => + error._tag === "PinnedRuntimeInstallError" + ? new BootService.BootServiceCommandError({ + step: error.step, + exitCode: error.exitCode, + stdoutLength: error.stdoutLength, + stderrLength: error.stderrLength, + cause: error, + }) + : new BootService.BootServiceInstallError({ cause: error }), + ), + ); + + const install: BootService.BootService["Service"]["install"] = Effect.gen(function* () { + yield* requireWindows; + yield* fs + .makeDirectory(input.logsDir, { recursive: true }) + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + + const percentIn = findPercentInPaths([ + ["the Node executable", plan.nodePath], + // Named with the variable that moves it, because the shared message + // deliberately gives no relocation advice of its own. + ["the T3 Code data directory (T3CODE_HOME)", plan.launcherPath], + ["the T3 Code log directory", plan.logPath], + ]); + if (percentIn !== undefined) { + return yield* new BootService.BootServicePathHasPercentError({ pathLabel: percentIn }); + } + + // Write the shortcut script first so the preflight has something to run. + // It lives under the data directory, so nothing user-visible changes yet. + yield* writeDurably(shortcutScriptPath, renderShortcutScript(plan)); + const checks = yield* preflight; + if (!checks.shellRunning) { + // Not fatal. Running a different shell is a deliberate choice, and it is + // the user's to make. Saying nothing would leave them with a service that + // silently never starts. + yield* Console.warn( + "Windows Explorer is not running, and it is what starts Startup folder entries.\n" + + "If you use a different shell, T3 Code may never start when you sign in.", + ); + } + if (checks.entryDisabled) { + return yield* new BootService.BootServiceStartupEntryDisabledError({ + shortcutName: SHORTCUT_NAME, + }); + } + + // Prepare every immutable artifact before stopping a running launcher. + yield* installPinnedRuntime; + const launcherSource = yield* fs + .readFileString(launcherSourcePath) + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + + const installed = yield* fs + .exists(unitPath) + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + // Before anything is stopped. This guard exists to protect an in-flight + // remote update, and terminating the launcher first would interrupt the + // very thing it is guarding. + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); + if (Option.isSome(previousStateText) && serviceStateHasPendingUpdate(previousStateText.value)) { + return yield* new BootService.BootServiceUpdatePendingError(); + } + + // Keyed on the pid file, never on the shortcut. A launcher outlives a + // manually deleted shortcut, and starting a second one alongside it would + // put two servers on the same database. + const stopped = yield* requestStop; + + yield* Effect.gen(function* () { + yield* writeDurably(launcherPath, launcherSource); + yield* writeDurably( + statePath, + // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned document. + `${JSON.stringify( + { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: input.cliVersion, + } satisfies ServiceState, + null, + 2, + )}\n`, + ); + yield* writeDurably(startupScriptPath, renderStartupScript(plan)); + + yield* runShortcutScript("creating the Windows Startup shortcut", "Install"); + // Start last. No administrative state write occurs after this succeeds. + yield* runPowerShellScript("starting the service", startupScriptPath); + }).pipe( + Effect.tapError(() => + installed || stopped + ? runPowerShellScript( + "restarting the service after a failed update", + startupScriptPath, + ).pipe(Effect.ignore) + : Effect.void, + ), + ); + return plan satisfies BootService.BootServicePlan; + }).pipe(Effect.withSpan("cloud.boot_service_windows.install")); + + const uninstall: BootService.BootService["Service"]["uninstall"] = Effect.gen(function* () { + yield* requireWindows; + const installed = yield* fs + .exists(unitPath) + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + + // Stop first, and decide that on the pid file rather than the shortcut. A + // launcher survives someone deleting the shortcut by hand, and leaving it + // running is exactly what the user asked us not to do. + const stopped = yield* requestStop; + if (!installed && !stopped) return false; + + // A shortcut is an ordinary file, so removing it needs no PowerShell. That + // also means a missing or broken generated script cannot strand the entry. + yield* fs + .remove(unitPath, { force: true }) + .pipe(Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause }))); + yield* fs.remove(startupScriptPath, { force: true }).pipe(Effect.ignore); + yield* fs.remove(shortcutScriptPath, { force: true }).pipe(Effect.ignore); + return true; + }).pipe(Effect.withSpan("cloud.boot_service_windows.uninstall")); + + const status: BootService.BootService["Service"]["status"] = Effect.gen(function* () { + const base = { kind: "win32-startup-shortcut", unitPath, logPath } as const; + if (platform !== "win32" || appData === "") { + return { supported: false, installed: false, current: false, ...base }; + } + if (!(yield* fs.exists(unitPath))) { + return { supported: true, installed: false, current: false, ...base }; + } + const [ + startupScript, + shortcutScript, + launcherExists, + runtimeEntryExists, + runtimeSentinel, + stateText, + ] = yield* Effect.all([ + fs.readFileString(startupScriptPath).pipe(Effect.option), + fs.readFileString(shortcutScriptPath).pipe(Effect.option), + fs.exists(launcherPath), + fs.exists(runtimePaths.entryPath), + fs.readFileString(runtimePaths.sentinelPath).pipe(Effect.option), + fs.readFileString(statePath).pipe(Effect.option), + ]); + const state = Option.isSome(stateText) ? parseServiceState(stateText.value) : undefined; + // Read the shortcut itself back, because the script that wrote it matching + // proves nothing about the .lnk someone may have edited or replaced since. + const shortcut = yield* runShortcutScript( + "checking the Windows Startup shortcut", + "Probe", + ).pipe( + Effect.map((probe) => parseProbeOutput(probe.stdout)), + Effect.orElseSucceed(() => undefined), + ); + const shortcutMatches = + shortcut?.shortcutTarget === powershell && + shortcut.shortcutArguments === expectedShortcutArguments(plan); + // Duplicated from the Linux status. The runtime and state checks are the + // same, but the Linux copy also compares the systemd unit, so it cannot be + // shared without changing the Linux path. + return { + supported: true, + installed: true, + current: + shortcutMatches && + Option.isSome(startupScript) && + startupScript.value === renderStartupScript(plan) && + // The shortcut script embeds the shortcut path, the interpreter and the + // window style, so comparing it is what catches shortcut drift. + Option.isSome(shortcutScript) && + shortcutScript.value === renderShortcutScript(plan) && + launcherExists && + runtimeEntryExists && + Option.isSome(runtimeSentinel) && + runtimeSentinel.value.trim() === input.cliVersion && + state?.activeVersion === input.cliVersion && + state?.update?.status !== "pending", + ...base, + }; + }).pipe( + Effect.mapError((cause) => new BootService.BootServiceInstallError({ cause })), + Effect.withSpan("cloud.boot_service_windows.status"), + ); + + return BootService.BootService.of({ install, uninstall, status }); +}); + +export const layer = (input: WindowsBootServiceInput) => + Layer.effect(BootService.BootService, make(input)); diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 0faf8894837..09aa9dc37fa 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -9,6 +9,70 @@ export const SERVICE_STATE_FILE = "service-state.json"; the child can tell "the service is going away" from "the launcher is about to start my replacement" while a pending update is recorded. */ export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; +/** Windows only. Set by the generated logon script because a Startup folder + shortcut has no supervisor. systemd never sets it, so the Linux launcher + keeps failing fast and letting `Restart=always` bring the whole unit back. */ +export const SERVICE_SELF_SUPERVISE_ENV = "T3_SERVICE_SELF_SUPERVISE"; +/** Windows only. Written by the CLI to ask a self-supervising launcher to stop, + because Windows has no SIGTERM. The launcher deletes it once it has stopped, + which is how the CLI learns the stop finished. */ +export const SERVICE_STOP_REQUEST_FILE = ".service-stop-request"; +/** Windows only. Holds a self-supervising launcher's process id while it runs. + It is what lets the CLI tell "the service stopped" apart from "no service was + running" and from "it is still running and did not answer". Without it, an + install could rewrite the runtime under a live launcher and end up with two + servers on one database. */ +export const SERVICE_PID_FILE = ".service-pid"; + +/** + * Windows only. What the pid file records. + * + * A process id alone is not enough. The file survives a crash or a hard reboot, + * and the operating system reuses process ids freely, so an old file can name a + * live process that has nothing to do with T3 Code. Believing it would leave the + * launcher refusing to start for as long as that stranger runs. Recording when + * the machine booted makes any file written before this boot obviously stale. + */ +export interface ServiceLauncherPresence { + readonly pid: number; + readonly bootTimeMs: number; + /** + * The server the launcher is currently running, when it has one. + * + * Windows has no equivalent of `KillMode=mixed`, so a launcher killed + * outright leaves its server behind. Recording the child means whoever takes + * the record over can put that orphan down before starting its own, instead + * of quietly ending up with two servers on one database. + */ + readonly childPid?: number; +} + +/** Boot time is derived from uptime, which drifts by a little between reads. */ +export const SAME_BOOT_TOLERANCE_MS = 60_000; + +export function encodeServiceLauncherPresence(presence: ServiceLauncherPresence): string { + const child = presence.childPid === undefined ? "" : ` ${presence.childPid}`; + return `${presence.pid} ${presence.bootTimeMs}${child}\n`; +} + +export function decodeServiceLauncherPresence(text: string): ServiceLauncherPresence | undefined { + const [rawPid, rawBoot, rawChild] = text.trim().split(/\s+/); + const pid = Number(rawPid); + const bootTimeMs = Number(rawBoot); + if (!Number.isInteger(pid) || pid <= 0 || !Number.isFinite(bootTimeMs)) return undefined; + const childPid = rawChild === undefined ? Number.NaN : Number(rawChild); + return Number.isInteger(childPid) && childPid > 0 + ? { pid, bootTimeMs, childPid } + : { pid, bootTimeMs }; +} + +/** False when the file predates this boot, whatever process now holds that id. */ +export function serviceLauncherPresenceIsFromThisBoot( + presence: ServiceLauncherPresence, + currentBootTimeMs: number, +): boolean { + return Math.abs(presence.bootTimeMs - currentBootTimeMs) <= SAME_BOOT_TOLERANCE_MS; +} export interface PendingServiceUpdate { readonly id: string; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 45c472af1fc..6b8c12f5530 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -2,15 +2,24 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as NodeOS from "node:os"; + import * as Path from "effect/Path"; +import * as DateTime from "effect/DateTime"; +import * as TestClock from "effect/testing/TestClock"; import { Launcher, readServiceState, writeServiceState } from "./serviceLauncher.ts"; import { compareExactServiceVersions, decodeServiceState, isExactServiceVersion, + decodeServiceLauncherPresence, + encodeServiceLauncherPresence, + serviceLauncherPresenceIsFromThisBoot, SERVICE_LAUNCHER_PROTOCOL, + SERVICE_PID_FILE, SERVICE_STOP_MARKER_FILE, + SERVICE_STOP_REQUEST_FILE, } from "./cloud/serviceProtocol.ts"; it("accepts only exact semantic versions", () => { @@ -290,3 +299,438 @@ if (context.update?.status === "pending") { }), ); }); + +it.layer(NodeServices.layer)("self-supervising launcher", (it) => { + /** A child that records every start, then exits so the launcher sees a crash. */ + const crashingChild = ` +import { appendFileSync } from "node:fs"; +appendFileSync(process.env.T3_TEST_START_LOG, "start\\n"); +process.exit(1); +`; + + const seedRuntime = Effect.fn("test.seed_launcher_runtime")(function* ( + root: string, + source: string, + ) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, source); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + return statePath; + }); + + it.effect("fails fast when something else supervises it, as on Linux", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-supervised-" }); + const startLog = path.join(root, "starts.log"); + process.env.T3_TEST_START_LOG = startLog; + const statePath = yield* seedRuntime(root, crashingChild); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: false }, + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + // One start, then it hands the problem to the init system by exiting. + assert.equal((yield* fs.readFileString(startLog)).trim().split("\n").length, 1); + }), + ); + + it.effect("restarts its own child, then gives up on the systemd burst limit", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-selfsuper-" }); + const startLog = path.join(root, "starts.log"); + process.env.T3_TEST_START_LOG = startLog; + const statePath = yield* seedRuntime(root, crashingChild); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + // Five starts in total, which is what StartLimitBurst=5 permits: the + // first one plus four restarts. + assert.equal((yield* fs.readFileString(startLog)).trim().split("\n").length, 5); + }), + ); + + it.effect("stops when the CLI writes a stop request, then clears it", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-stopreq-" }); + const statePath = yield* seedRuntime(root, "setInterval(() => {}, 1_000);\n"); + const requestPath = path.join(root, "runtime", SERVICE_STOP_REQUEST_FILE); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + const running = launcher.run(); + yield* fs.writeFileString(requestPath, ""); + yield* Effect.promise(() => running); + + // Deleting it is how the CLI learns the stop actually happened. + assert.isFalse(yield* fs.exists(requestPath)); + assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); + }), + ); + + it.effect("ignores a request left over from before it started", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-stalereq-" }); + const statePath = yield* seedRuntime(root, "setInterval(() => {}, 1_000);\n"); + const requestPath = path.join(root, "runtime", SERVICE_STOP_REQUEST_FILE); + yield* fs.writeFileString(requestPath, ""); + // Backdate it well past the grace window, which is what makes it stale. + // A request written just before startup is deliberately treated as real. + // A fixed date, so this does not depend on the clock the test runs under. + // Seconds, not milliseconds: utimes reads a bare number as seconds, and + // milliseconds would land in the far future and read as brand new. + const staleSeconds = 1_577_836_800; // 2020-01-01 + yield* fs.utimes(requestPath, staleSeconds, staleSeconds); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + const running = launcher.run(); + yield* Effect.sleep("150 millis"); + // Provoke the watcher now that recovery has finished. Without this the + // next check is a 2s poll away, and the test would pass without ever + // exercising the staleness branch it exists for. + yield* fs.writeFileString(path.join(root, "runtime", "provoke"), ""); + yield* Effect.sleep("150 millis"); + + // Still running: the stale request was judged old, not obeyed. + assert.isFalse(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); + + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }).pipe(TestClock.withLive), + ); +}); + +it.layer(NodeServices.layer)("self-supervising pid ownership", (it) => { + it.effect("a second launcher exits rather than running a second server", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-pid-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const state = yield* Effect.promise(() => readServiceState(statePath)); + + const first = new Launcher(root, state, { selfSupervise: true, restartDelayMs: 5 }); + const firstRunning = first.run(); + yield* Effect.sleep("100 millis"); + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + assert.isTrue(yield* fs.exists(pidPath)); + + // The Startup shortcut firing twice must not put two servers on one + // database. The second launcher sees a live owner and gives up. + const second = new Launcher(root, state, { selfSupervise: true, restartDelayMs: 5 }); + yield* Effect.promise(() => second.run()); + // It also must not take the first launcher's pid file with it on the way out. + assert.isTrue(yield* fs.exists(pidPath)); + + yield* Effect.promise(() => first.stop("SIGTERM")); + yield* Effect.promise(() => firstRunning); + assert.isFalse(yield* fs.exists(pidPath)); + }).pipe(TestClock.withLive), + ); +}); + +it("treats a pid record from an earlier boot as stale, whoever holds that id now", () => { + const presence = { pid: process.pid, bootTimeMs: 1_000 }; + + assert.isFalse(serviceLauncherPresenceIsFromThisBoot(presence, 9_000_000)); + assert.isTrue(serviceLauncherPresenceIsFromThisBoot(presence, 1_000)); + // Boot time comes from uptime, which drifts a little between reads. + assert.isTrue(serviceLauncherPresenceIsFromThisBoot(presence, 30_000)); +}); + +it("round-trips a pid record and rejects a malformed one", () => { + const presence = { pid: 4321, bootTimeMs: 1_700_000_000_000 }; + + assert.deepEqual( + decodeServiceLauncherPresence(encodeServiceLauncherPresence(presence)), + presence, + ); + assert.isUndefined(decodeServiceLauncherPresence("")); + // The old format carried a bare pid, and believing it would drop the guard. + assert.isUndefined(decodeServiceLauncherPresence("4321\n")); +}); + +it.layer(NodeServices.layer)("self-supervising start failures", (it) => { + it.effect("spends another burst attempt when a start fails, instead of dying", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-startfail-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "process.exit(1);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + const running = launcher.run(); + // Remove the runtime mid-flight, so the next start throws rather than + // producing a child that exits. That used to reach #fatal and kill the + // launcher outright, defeating the supervisor over a transient failure. + yield* Effect.sleep("40 millis"); + yield* fs.remove(versionDir, { recursive: true, force: true }); + + // It still ends by exhausting the burst, not by an unhandled start error. + const outcome = yield* Effect.promise(() => + running.then( + () => "completed" as const, + (cause: unknown) => String(cause), + ), + ); + assert.include(outcome, "Active child exited unexpectedly"); + }).pipe(TestClock.withLive), + ); +}); + +it.layer(NodeServices.layer)("abandoned takeover recovery", (it) => { + it.effect("starts anyway when a previous claim died holding the takeover", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-takeover-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + // A launcher that died mid-claim leaves both files behind. Without + // recovery the takeover blocks every future start, and the only cure is + // deleting a hidden file by hand. + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + const takeoverPath = `${pidPath}.takeover`; + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ pid: 999_999, bootTimeMs: 0 }), + ); + yield* fs.writeFileString(takeoverPath, ""); + // Seconds, not milliseconds: a number given to utimes is a Unix + // timestamp in seconds, and passing milliseconds lands in the far future. + const longAgoSeconds = 1_577_836_800; // 2020-01-01 + yield* fs.utimes(takeoverPath, longAgoSeconds, longAgoSeconds); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + const running = launcher.run(); + yield* Effect.sleep("150 millis"); + + // It claimed the record, and cleared the abandoned takeover on its way. + assert.isTrue(yield* fs.exists(pidPath)); + assert.isFalse(yield* fs.exists(takeoverPath)); + + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }).pipe(TestClock.withLive), + ); + + it.effect("stands down while another launcher is genuinely mid-claim", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-takeover-busy-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ pid: 999_999, bootTimeMs: 0 }), + ); + // A fresh takeover means somebody else is actively claiming, so the + // recovery above must not fire and let two launchers through. + yield* fs.writeFileString(`${pidPath}.takeover`, ""); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5 }, + ); + yield* Effect.promise(() => launcher.run()); + + // It exited without touching the other launcher's takeover. + assert.isTrue(yield* fs.exists(`${pidPath}.takeover`)); + }).pipe(TestClock.withLive), + ); +}); + +const nowMillis = DateTime.now.pipe(Effect.map(DateTime.toEpochMillis)); + +it.layer(NodeServices.layer)("liveness of a quiet pid record", (it) => { + const seed = Effect.fn("test.seed_quiet_runtime")(function* (root: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + return statePath; + }); + + it.effect("stands down when an old record starts moving again", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-sleepy-" }); + const statePath = yield* seed(root); + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + // A live launcher that slept through its heartbeat: this process id is + // real and from this boot, but the record looks ancient. Treating that as + // dead would put a second server on one database. + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ + pid: process.pid, + bootTimeMs: (yield* nowMillis) - NodeOS.uptime() * 1_000, + }), + ); + const longAgoSeconds = 1_577_836_800; // 2020-01-01, in seconds + yield* fs.utimes(pidPath, longAgoSeconds, longAgoSeconds); + + // It wakes and refreshes while the probe is watching. + yield* Effect.forkScoped( + Effect.sleep("60 millis").pipe( + Effect.flatMap(() => fs.writeFileString(pidPath, "refreshed by the live launcher\n")), + ), + ); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5, heartbeatProbeMs: 200 }, + ); + yield* Effect.promise(() => launcher.run()); + + // It exited rather than claiming, and left the record alone. + assert.equal(yield* fs.readFileString(pidPath), "refreshed by the live launcher\n"); + }).pipe(TestClock.withLive), + ); + + it.effect("takes over a record that never moves again", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launcher-recycled-" }); + const statePath = yield* seed(root); + const pidPath = path.join(root, "runtime", SERVICE_PID_FILE); + // Same shape, but nothing ever refreshes it. That is a process id the + // system handed to somebody else, so the record is not a launcher at all. + yield* fs.writeFileString( + pidPath, + encodeServiceLauncherPresence({ + pid: process.pid, + bootTimeMs: (yield* nowMillis) - NodeOS.uptime() * 1_000, + }), + ); + const longAgoSeconds = 1_577_836_800; // 2020-01-01, in seconds + yield* fs.utimes(pidPath, longAgoSeconds, longAgoSeconds); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { selfSupervise: true, restartDelayMs: 5, heartbeatProbeMs: 200 }, + ); + const running = launcher.run(); + yield* Effect.sleep("600 millis"); + + const claimed = decodeServiceLauncherPresence(yield* fs.readFileString(pidPath)); + assert.equal(claimed?.pid, process.pid); + assert.isDefined(claimed?.childPid); + + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }).pipe(TestClock.withLive), + ); +}); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 3641593ecf0..c1dbb24eaac 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -7,6 +7,7 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import type { @@ -14,6 +15,7 @@ import type { ServiceLauncherChildMessage, ServiceLauncherContext, ServiceLauncherParentMessage, + ServiceLauncherPresence, ServiceState, ServiceUpdateRecord, } from "./cloud/serviceProtocol.ts"; @@ -24,14 +26,81 @@ import { parseServiceState, SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_LAUNCHER_PROTOCOL, + SERVICE_PID_FILE, + decodeServiceLauncherPresence, + encodeServiceLauncherPresence, + serviceLauncherPresenceIsFromThisBoot, + SERVICE_SELF_SUPERVISE_ENV, SERVICE_STATE_FILE, SERVICE_STOP_MARKER_FILE, + SERVICE_STOP_REQUEST_FILE, } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; const PREPARED_TIMEOUT_MS = 120_000; const TERMINATE_GRACE_MS = 5_000; +/** + * Windows only. A Startup folder shortcut has no supervisor, so the launcher + * does the job the systemd unit does on Linux. These three mirror RestartSec=5, + * StartLimitBurst=5 and StartLimitIntervalSec=300, so a self-supervising + * launcher gives up at exactly the point systemd would. The burst counts the + * first start as well, the way systemd counts it, so five means four restarts. + */ +const RESTART_DELAY_MS = 5_000; +const RESTART_BURST = 5; +const RESTART_WINDOW_MS = 300_000; +/** Windows only. How often the launcher looks for a stop request. Directory + watching misses events, so this poll is the real delivery guarantee. */ +const STOP_REQUEST_WATCH_INTERVAL_MS = 2_000; +/** + * Windows only. A request older than this when the launcher starts was left by + * a stop nobody completed, so obeying it would stop a launcher the request was + * never meant for. The window is generous on purpose: some filesystems report + * a coarse modification time, and wrongly ignoring a real stop is far worse + * than wrongly obeying an old one. Ignoring one lets an install rewrite the + * runtime under a launcher that is still running. + */ +const STOP_REQUEST_GRACE_MS = 5_000; + +/** Windows only. Set by the generated logon script. systemd never sets it. */ +const isSelfSupervising = (): boolean => process.env[SERVICE_SELF_SUPERVISE_ENV] === "1"; + +/** Windows only. Milliseconds since the epoch when this machine last booted. */ +const currentBootTimeMs = (): number => Date.now() - NodeOS.uptime() * 1_000; + +/** + * Windows only. How often a running launcher refreshes its pid record. + * + * The boot stamp catches a record left by an earlier boot, but not one left by + * a launcher that crashed during this boot whose id has since been handed to an + * unrelated process. A record that stops being refreshed is proof of death on + * its own, whoever holds that id now. + */ +const PID_HEARTBEAT_MS = 15_000; +/** Generous against a paused or heavily loaded machine, still far below a reboot. */ +const PID_RECORD_STALE_AFTER_MS = 90_000; +/** + * Windows only. How long a takeover of a leftover pid record may sit before it + * counts as abandoned. The work it covers is a stat, a read, a delete and a + * write, so this is orders of magnitude more than any real one needs. + */ +const TAKEOVER_STALE_AFTER_MS = 30_000; + +/** + * Windows only. Signal 0 asks the kernel whether a process exists without + * touching it. A permission error means it exists but is not ours, which still + * counts as running. + */ +function processIsAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException | undefined)?.code === "EPERM"; + } +} + type TerminalStatus = "committed" | "rolled-back" | "failed"; type ChildRole = "active" | "trial"; @@ -262,9 +331,62 @@ async function terminateChild( const stopMarkerPath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); +/** Windows only. Written by the CLI to ask for a stop, because there is no SIGTERM. */ +const stopRequestPath = (baseDir: string) => + NodePath.join(baseDir, "runtime", SERVICE_STOP_REQUEST_FILE); + +/** Windows only. Present exactly while a self-supervising launcher is running. */ +const pidFilePath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_PID_FILE); + +const pidRecordMtime = async (target: string): Promise => { + try { + return (await NodeFSP.stat(target)).mtimeMs; + } catch { + return undefined; + } +}; + +/** + * Windows only. Whether a launcher is still refreshing its record. + * + * An old modification time on its own proves nothing. A laptop that slept for + * an hour wakes with a perfectly live launcher whose last heartbeat is ancient, + * and treating that as dead would start a second server on one database. So an + * old timestamp only makes the answer unknown, and the unknown case is settled + * by watching for the next heartbeat rather than guessing. + */ +export async function pidRecordIsBeingRefreshed( + target: string, + probeMs: number = PID_HEARTBEAT_MS * 2, +): Promise { + const first = await pidRecordMtime(target); + if (first === undefined) return false; + if (Date.now() - first <= PID_RECORD_STALE_AFTER_MS) return true; + + await new Promise((resolve) => setTimeout(resolve, probeMs)); + const second = await pidRecordMtime(target); + return second !== undefined && second !== first; +} + +export interface LauncherOptions { + /** + * Windows only. Whether this launcher must restart its own child and watch + * for stop requests. Defaults to the flag the generated logon script sets, + * which systemd never sets. + */ + readonly selfSupervise?: boolean; + /** Overridable so tests do not sit through the real wait. */ + readonly restartDelayMs?: number; + /** Overridable for the same reason: how long to watch for a heartbeat. */ + readonly heartbeatProbeMs?: number; +} + export class Launcher { readonly #baseDir: string; readonly #statePath: string; + readonly #selfSupervise: boolean; + readonly #restartDelayMs: number; + readonly #heartbeatProbeMs: number; #state: ServiceState; #child: ManagedChild | null = null; #timer: NodeJS.Timeout | undefined; @@ -272,12 +394,32 @@ export class Launcher { #stopRequested = false; #stopping = false; #done = false; + /** Windows only. Restart times inside the current window. */ + #restartAttempts: Array = []; + /** Windows only. Used to tell a fresh stop request from a leftover one. */ + #startedAt = 0; + /** Windows only. Modification time of a request already judged stale. */ + #dismissedRequestAt: number | undefined; + /** + * Windows only. Recovery clears a stale stop marker, so acting on a request + * before it runs would let recovery wipe the marker the stop just wrote. The + * child would then read "a replacement server is coming" while none is. + */ + #recovered = false; + #pidHeartbeat: NodeJS.Timeout | undefined; + #stopWatcher: NodeFS.FSWatcher | undefined; + #stopPoll: NodeJS.Timeout | undefined; + /** Windows only. Lets a stop cut a restart backoff short instead of queueing behind it. */ + #restartWait: { readonly timer: NodeJS.Timeout; readonly resolve: () => void } | undefined; readonly #completion = Promise.withResolvers(); - constructor(baseDir: string, state: ServiceState) { + constructor(baseDir: string, state: ServiceState, options?: LauncherOptions) { this.#baseDir = baseDir; this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); this.#state = state; + this.#selfSupervise = options?.selfSupervise ?? isSelfSupervising(); + this.#restartDelayMs = options?.restartDelayMs ?? RESTART_DELAY_MS; + this.#heartbeatProbeMs = options?.heartbeatProbeMs ?? PID_HEARTBEAT_MS * 2; } async run(): Promise { @@ -285,15 +427,322 @@ export class Launcher { const onSigint = () => void this.stop("SIGINT"); process.once("SIGTERM", onSigterm); process.once("SIGINT", onSigint); + const supervising = this.#selfSupervise; + let owned = false; + this.#startedAt = Date.now(); + // The first start counts toward the burst, the way systemd counts it. + this.#restartAttempts = [this.#startedAt]; try { + if (supervising) { + // Claimed before anything else, and never swallowed on failure. The CLI + // reads an absent pid file as proof nothing is running, so a live + // launcher without one would let an install rewrite the runtime + // underneath it and start a second server on the same database. + owned = await this.#claimPidFile(); + if (!owned) { + // Another launcher is already running, which happens when the Startup + // shortcut fires twice. Leave it alone and exit. + process.stderr.write("[service-launcher] another launcher is already running\n"); + return; + } + this.#startPidHeartbeat(); + this.#watchStopRequest(); + } this.#enqueue(() => this.#recover()); + this.#enqueue(async () => { + this.#recovered = true; + }); await this.#completion.promise; } finally { process.off("SIGTERM", onSigterm); process.off("SIGINT", onSigint); + this.#stopWatchingStopRequest(); + clearInterval(this.#pidHeartbeat); + this.#pidHeartbeat = undefined; + if (owned) { + // Removing these before run() returns is how the CLI learns the stop + // finished. Doing it after would race the process exiting. + await NodeFSP.rm(stopRequestPath(this.#baseDir), { force: true }).catch(() => undefined); + await this.#releasePidFile(); + } + } + } + + /** + * Windows only. Claims the pid file exclusively. Returns false when another + * live launcher already owns it, which is what stops a Startup shortcut that + * fires twice from running two servers against one database. + */ + async #claimPidFile(): Promise { + const target = pidFilePath(this.#baseDir); + await NodeFSP.mkdir(NodePath.dirname(target), { recursive: true, mode: 0o700 }); + if (await this.#writePidFileExclusive(target)) return true; + + if (await this.#pidFileHolderIsAlive()) return false; + + // The record is leftover. Clearing it and creating our own is two steps, so + // two launchers racing here could both delete and both claim. An exclusive + // takeover file decides a single winner; everyone else loses and exits. + const takeover = `${target}.takeover`; + if (!(await this.#claimTakeover(takeover))) return false; + try { + // Re-check under the takeover: the previous holder may have revived. + if (await this.#pidFileHolderIsAlive()) return false; + // Whoever left this record may have left its server running too. + await this.#terminateOrphanedChild(await this.#readPidFile()); + await NodeFSP.rm(target, { force: true }); + return await this.#writePidFileExclusive(target); + } finally { + await NodeFSP.rm(takeover, { force: true }).catch(() => undefined); + } + } + + /** + * Windows only. Wins the right to replace a leftover pid record. + * + * A takeover spans a handful of file operations, so one that has been sitting + * around was abandoned by a launcher that died mid-claim, or by the machine + * losing power. Treating that as "somebody is busy" would block every start + * from then on, and the only cure would be deleting a hidden file by hand. + */ + async #claimTakeover(takeover: string): Promise { + if (await this.#createTakeover(takeover)) return true; + + let abandonedAt: number | undefined; + try { + const stats = await NodeFSP.stat(takeover); + abandonedAt = + Date.now() - stats.mtimeMs > TAKEOVER_STALE_AFTER_MS ? stats.mtimeMs : undefined; + } catch { + // It vanished, so the launcher holding it finished. Let that one win. + return false; + } + if (abandonedAt === undefined) return false; + + // Renaming picks a single winner without a read-then-delete window: + // concurrent renames of one source leave exactly one success, and everyone + // else finds the source already gone. + const claimed = `${takeover}.${process.pid}.${NodeCrypto.randomUUID()}`; + try { + await NodeFSP.rename(takeover, claimed); + } catch { + return false; + } + await NodeFSP.rm(claimed, { force: true }).catch(() => undefined); + return await this.#createTakeover(takeover); + } + + async #createTakeover(takeover: string): Promise { + try { + await (await NodeFSP.open(takeover, "wx", 0o600)).close(); + return true; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; + return false; + } + } + + async #writePidFileExclusive(target: string): Promise { + try { + const handle = await NodeFSP.open(target, "wx", 0o600); + try { + await handle.writeFile(this.#presenceRecord(), "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + return true; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; + return false; + } + } + + #presenceRecord(): string { + const childPid = this.#child?.process.pid; + return encodeServiceLauncherPresence({ + pid: process.pid, + bootTimeMs: currentBootTimeMs(), + ...(childPid === undefined ? {} : { childPid }), + }); + } + + /** + * Windows only. Rewrites the record so it names the current child, and so its + * modification time proves this launcher is still here. Doubling as the + * heartbeat keeps one writer for one file. + */ + async #refreshPidFile(): Promise { + if (!this.#selfSupervise) return; + await NodeFSP.writeFile(pidFilePath(this.#baseDir), this.#presenceRecord(), { + mode: 0o600, + }).catch(() => undefined); + } + + /** + * Windows only. Puts down a server left behind by a launcher that died + * without taking it with it. Linux gets this from KillMode=mixed. + */ + async #terminateOrphanedChild(owner: ServiceLauncherPresence | undefined): Promise { + const childPid = owner?.childPid; + if (childPid === undefined || childPid === process.pid) return; + if (owner !== undefined && !serviceLauncherPresenceIsFromThisBoot(owner, currentBootTimeMs())) { + return; + } + if (!processIsAlive(childPid)) return; + try { + process.kill(childPid, "SIGKILL"); + process.stderr.write(`[service-launcher] ended an orphaned server (pid ${childPid})\n`); + } catch { + // It exited between the check and the signal, which is the good outcome. + } + } + + /** + * Windows only. True only for a record from this boot, naming a live process, + * and still being refreshed. A live owner wins even when it is this very + * process: two Launcher instances in one process are still two servers on one + * database, so there is nothing to exempt. + */ + async #pidFileHolderIsAlive(): Promise { + const owner = await this.#readPidFile(); + if (owner === undefined) return false; + if (!serviceLauncherPresenceIsFromThisBoot(owner, currentBootTimeMs())) return false; + if (!processIsAlive(owner.pid)) return false; + return await pidRecordIsBeingRefreshed(pidFilePath(this.#baseDir), this.#heartbeatProbeMs); + } + + /** Windows only. Keeps this launcher's record demonstrably fresh. */ + #startPidHeartbeat(): void { + this.#pidHeartbeat = setInterval(() => { + void this.#refreshPidFile(); + }, PID_HEARTBEAT_MS); + this.#pidHeartbeat.unref(); + } + + async #readPidFile(): Promise { + const contents = await NodeFSP.readFile(pidFilePath(this.#baseDir), "utf8").catch(() => ""); + return decodeServiceLauncherPresence(contents); + } + + /** Windows only. Only clears the file if this process still owns it. */ + async #releasePidFile(): Promise { + if ((await this.#readPidFile())?.pid !== process.pid) return; + await NodeFSP.rm(pidFilePath(this.#baseDir), { force: true }).catch(() => undefined); + } + + /** + * Windows only. Watches for the CLI's stop request, because Windows has no + * SIGTERM. The stop it triggers is not graceful: the child is terminated + * without running its shutdown finalizer. + */ + #watchStopRequest(): void { + const target = stopRequestPath(this.#baseDir); + const check = () => { + if (!this.#recovered || this.#stopRequested || this.#stopping) return; + void NodeFSP.stat(target).then( + async (stats) => { + if (this.#stopRequested || this.#stopping) return; + if (this.#dismissedRequestAt === stats.mtimeMs) return; + if (stats.mtimeMs >= this.#startedAt - STOP_REQUEST_GRACE_MS) { + await this.stop("SIGTERM"); + return; + } + // Left behind by a stop nobody completed. Remember it rather than + // deleting it: the CLI reads a vanished pid file as proof that a + // launcher stopped, and deleting files here must never look like that. + this.#dismissedRequestAt = stats.mtimeMs; + }, + () => undefined, + ); + }; + try { + this.#stopWatcher = NodeFS.watch(NodePath.dirname(target), () => check()); + // An error event with no listener is an uncaught exception, and Windows + // emits EPERM here when the watched directory goes away. The poll below + // is the real delivery guarantee, so losing the watcher costs nothing. + this.#stopWatcher.on("error", () => undefined); + } catch { + // Best effort, for the same reason. + } + this.#stopPoll = setInterval(check, STOP_REQUEST_WATCH_INTERVAL_MS); + this.#stopPoll.unref(); + check(); + } + + #stopWatchingStopRequest(): void { + this.#stopWatcher?.close(); + this.#stopWatcher = undefined; + clearInterval(this.#stopPoll); + this.#stopPoll = undefined; + } + + /** + * Windows only. Waits out the restart backoff, but returns early when a stop + * arrives. The wait runs inside a queued transition, so sleeping through it + * would delay the stop by the full delay and eat the CLI's patience. + */ + #waitBeforeRestart(): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.#restartWait = undefined; + resolve(); + }, this.#restartDelayMs); + this.#restartWait = { + timer, + resolve: () => { + clearTimeout(timer); + this.#restartWait = undefined; + resolve(); + }, + }; + }); + } + + /** + * Windows only. Retries the active child until the burst is spent. Returns + * false once it is, so the caller can fail the way Linux already does. + */ + async #restartActiveChild(): Promise { + while (this.#selfSupervise && this.#recordRestartAttempt()) { + await this.#waitBeforeRestart(); + if (this.#stopping || this.#done || this.#stopRequested) return true; + try { + await this.#startChild(this.#state.activeVersion, "active", this.#state.update); + return true; + } catch { + // A start can fail transiently, and letting that escape would reach + // #fatal and kill the launcher outright. Spend another attempt from the + // same burst instead, so the supervisor survives what it exists for. + } + } + return false; + } + + /** + * Windows only. The very first start needs the same treatment. A transient + * failure at sign-in would otherwise leave the service down until the next + * one, which is not what Restart=always does. + */ + async #startActiveChildWithRetry(update?: ServiceUpdateRecord): Promise { + try { + await this.#startChild(this.#state.activeVersion, "active", update); + return; + } catch (cause) { + if (!this.#selfSupervise) throw cause; + if (!(await this.#restartActiveChild())) throw cause; } } + /** Windows only. False once the burst limit is reached inside the window. */ + #recordRestartAttempt(): boolean { + const now = Date.now(); + this.#restartAttempts = this.#restartAttempts.filter((at) => now - at < RESTART_WINDOW_MS); + if (this.#restartAttempts.length >= RESTART_BURST) return false; + this.#restartAttempts.push(now); + return true; + } + #enqueue(transition: () => Promise): void { this.#transitions = this.#transitions .then(transition, transition) @@ -328,6 +777,9 @@ export class Launcher { return; } this.#stopRequested = true; + // Windows only. A restart backoff holds the transition queue, so a stop + // queued behind it would wait the full delay before it even started. + this.#restartWait?.resolve(); this.#clearTimer(); this.#enqueue(async () => { // Let an update transition already in progress start its replacement @@ -358,7 +810,7 @@ export class Launcher { if (update !== undefined) { await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); } - await this.#startChild(this.#state.activeVersion, "active", update); + await this.#startActiveChildWithRetry(update); return; } if (await databaseRestorePending(this.#baseDir, update)) { @@ -423,6 +875,7 @@ export class Launcher { process: child, }; this.#child = managed; + await this.#refreshPidFile(); child.on("message", (value) => { const message = decodeServiceLauncherChildMessage(value); if (message !== undefined) this.#enqueue(() => this.#handleMessage(managed, message)); @@ -572,6 +1025,10 @@ export class Launcher { await this.#startTrial(pending); return; } + // Windows only. Nothing supervises a Startup folder entry, so the launcher + // does what Restart=always does on Linux, and gives up on the same terms. + // Without the flag this still throws, so the Linux path is unchanged. + if (await this.#restartActiveChild()) return; throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); } diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 299e3b641f1..4252e77da02 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -3,6 +3,9 @@ On a Linux host, T3 Code can run as a background service for your user. It starts when the machine boots and keeps running after you log out. +On Windows it starts when you sign in and stops when you sign out. See +[On Windows](#on-windows) for the full list of differences. + ## Manage the Service Install it with the latest T3 Code release: @@ -46,4 +49,34 @@ out. This is only an onboarding shortcut: the service and T3 Connect are managed Signing out of T3 Connect does not remove the service. Use `t3 service uninstall` when you no longer want T3 Code to start in the background. -The background service currently requires Linux with systemd. +The background service requires Linux with systemd, or Windows. macOS is not supported yet. + +## On Windows + +Windows has no systemd, so `t3 service install` puts a shortcut named `T3 Code Server` in your +personal Startup folder. Windows Explorer runs that shortcut every time you sign in. The shortcut +starts PowerShell, PowerShell starts the T3 Code server with no window, then PowerShell exits. + +It is called `T3 Code Server` rather than `T3 Code` because it only starts the server. The desktop +app is separate, and you start that yourself. + +The same four commands work: `install`, `status`, `update` and `uninstall`. + +Windows differs from Linux in ways worth knowing before you rely on it: + +- **It starts at sign-in, not at boot.** Nothing runs while the machine sits at the sign-in screen. +- **It stops when you sign out.** Windows ends your session's processes, and the server is one. +- **Stopping is not graceful.** `update` and `uninstall` terminate the server rather than asking it + to shut down. In-flight agent work is cut, and the server does not get to release its T3 Connect + link, so the host can look online for a short while after it has gone. +- **A small window blinks once at sign-in.** It is named `T3 Code Server`. That flash is PowerShell + starting, and it is expected. Nothing stays on your taskbar afterwards. +- **Windows Settings can switch it off.** It appears under Startup apps. If you disable it there, + `t3 service status` still reports it as installed, because the shortcut is still on disk, and + `t3 service install` still reports nothing to do. Check Startup apps first if the service stops + coming back after you sign in. An install that does have work to do refuses to write over a + disabled entry and tells you to turn it back on. +- **Windows Explorer must be your shell.** That is the default. If you replaced it, Startup folder + entries may never run, and `t3 service install` warns you about that. + +Let agent work finish before running `update` on Windows, because the stop is not graceful.