[OTEL] Logging setup

ts/config.sample.yaml

@@ -400,14 +400,18 @@ typeagent:
 #   - otlpEndpoint       fallback endpoint for traces + metrics + logs
 #   - logFile            local JSONL log file; enables the logs signal
 #                        independently of any OTLP endpoint. Leading
-#                        `~`/`~/`/`~\` expands to your home dir.
+#                        `~`/`~/`/`~\` expands to your home dir. Supports
+#                        `{service}`, `{process}`, and `{pid}` placeholders.
+#   - debugBridge        copy enabled typeagent:* debug output to OTel logs
 #   - tracesSampler      one of always_on, always_off, traceidratio,
 #                        parentbased_always_on, parentbased_always_off,
 #                        parentbased_traceidratio
 #   - tracesSamplerArg   required with the ratio samplers (0.0 - 1.0)
 
 # telemetry:
 #   otlpEndpoint: http://localhost:4318
-#   logFile: ~/.typeagent/logs/typeagent-{service}-{pid}.jsonl
+#   logFile: ~/.typeagent/logs/typeagent-{service}-{process}-{pid}.jsonl
+#   debugBridge: true
+#   structuredLogs: true
 #   tracesSampler: parentbased_traceidratio
 #   tracesSamplerArg: 0.1

ts/examples/workflow/lsp/test/serverIntegration.spec.ts

@@ -22,6 +22,8 @@ import {
     DefinitionRequest,
     RenameRequest,
     DocumentRangeFormattingRequest,
+    ShutdownRequest,
+    ExitNotification,
     StreamMessageReader,
     StreamMessageWriter,
 } from "vscode-languageserver-protocol/node.js";
@@ -67,16 +69,13 @@ async function startSession(debounceMs = 5) {
         server,
         publishes,
         cleanup: async () => {
-            // Add a small delay to let pending operations and error handlers complete
-            // before disposing connections. This prevents "Connection is disposed" errors
-            // when error handlers try to send notifications after test completion.
-            await new Promise((resolve) => setTimeout(resolve, 50));
-            client.dispose();
-            server.dispose();
-            pipes.serverTransport.input.destroy();
-            pipes.serverTransport.output.destroy();
-            pipes.clientReader.dispose();
-            pipes.clientWriter.dispose();
+            try {
+                await client.sendRequest(ShutdownRequest.type);
+                await client.sendNotification(ExitNotification.type);
+            } finally {
+                server.dispose();
+                client.dispose();
+            }
         },
     };
 }

ts/package.json

@@ -78,6 +78,7 @@
     "start:agent-server:tunnel": "pnpm -C packages/agentServer/server run start:tunnel",
     "start:mcp": "pnpm -C packages/commandExecutor run start",
     "stop:agent-server": "pnpm -C packages/agentServer/server run stop",
+    "telemetry:grafana": "node tools/scripts/startLocalTelemetry.mjs",
     "test": "pnpm run test:local && pnpm run test:live && pnpm run shell:test",
     "test:keys": "npx tsx tools/scripts/testServiceKeys.ts",
     "test:live": "pnpm -r ---no-bail  -no-sort --stream --workspace-concurrency=1 run test:live",

ts/packages/agentServer/server/src/server.ts

@@ -179,7 +179,15 @@ process.once("message", (message) => {
 // Load config from YAML layers + Key Vault (replacing legacy dotenv).
 // vault.shared is auto-discovered from config.local.yaml / config.defaults.yaml.
 await loadConfig({ keyVault: {}, strict: false });
-const telemetryInit = otel.initTelemetry();
+const telemetryConfig = otel.resolveTelemetryConfig();
+const telemetryInit = otel.initTelemetry({
+    config: telemetryConfig,
+    processName: "agent-server",
+    debugModules: [registerDebug],
+    debugBridge: {
+        includedNamespacePrefixes: ["typeagent:", "agent-server:"],
+    },
+});
 
 // Snapshot whether this server's local config differs from the shared Key
 // Vault, so clients can be warned on connect (same delivery path as the
@@ -342,6 +350,9 @@ async function main() {
                 dblogging: true,
                 developerMode,
                 traceId,
+                telemetry: {
+                    structuredLogs: telemetryConfig.structuredLogs === true,
+                },
                 indexingServiceRegistry: await getIndexingServiceRegistry(
                     instanceDir,
                     configName,

ts/packages/api/src/index.ts

@@ -4,6 +4,7 @@
 import { TypeAgentServer } from "./typeAgentServer.js";
 import { loadConfig } from "@typeagent/config";
 import { otel } from "@typeagent/telemetry";
+import registerDebug from "debug";
 
 let typeAgentServer: TypeAgentServer | undefined;
 let shutdownPromise: Promise<void> | undefined;
@@ -40,14 +41,22 @@ process.once("SIGTERM", () => {
 async function main(): Promise<void> {
     // Load config from YAML layers + Key Vault (replacing legacy dotenv).
     await loadConfig({ keyVault: {}, strict: false });
-    await otel.initTelemetry();
+    const telemetryConfig = otel.resolveTelemetryConfig();
+    await otel.initTelemetry({
+        config: telemetryConfig,
+        processName: "api-server",
+        debugModules: [registerDebug],
+        debugBridge: {
+            includedNamespacePrefixes: ["typeagent:", "agent-server:"],
+        },
+    });
     if (shutdownRequested) {
         return;
     }
 
     typeAgentServer = new TypeAgentServer((exitCode) => {
         void shutdownHost(exitCode);
-    });
+    }, telemetryConfig.structuredLogs === true);
 
     await typeAgentServer.start();
 }

ts/packages/api/src/typeAgentServer.ts

@@ -40,7 +40,10 @@ export class TypeAgentServer {
     private storageProvider: TypeAgentStorageProvider | undefined;
     private config: TypeAgentAPIServerConfig;
 
-    constructor(private readonly requestExit: (exitCode: number) => void) {
+    constructor(
+        private readonly requestExit: (exitCode: number) => void,
+        private readonly structuredLogs: boolean,
+    ) {
         // Build typed runtime Config from process.env (already populated
         // by loadConfig in the entry point) so aiclient consumers can
         // use the typed accessor; legacy callers still see the same
@@ -83,7 +86,7 @@ export class TypeAgentServer {
             sw.stop("Downloaded Session Backup");
         }
 
-        this.webDispatcher = await createWebDispatcher();
+        this.webDispatcher = await createWebDispatcher(this.structuredLogs);
         debug("Web Dispatcher created.");
 
         // web server

ts/packages/api/src/webDispatcher.ts

@@ -35,7 +35,9 @@ export interface WebDispatcher {
     handleAction(action: FullAction): Promise<CommandResult>;
 }
 
-export async function createWebDispatcher(): Promise<WebDispatcher> {
+export async function createWebDispatcher(
+    structuredLogs: boolean,
+): Promise<WebDispatcher> {
     let ws: WebSocket | null = null;
     const clientIOChannel = createChannelAdapter((message: any) =>
         ws?.send(
@@ -64,6 +66,9 @@ export async function createWebDispatcher(): Promise<WebDispatcher> {
         metrics: true,
         dblogging: true,
         traceId: getTraceId(),
+        telemetry: {
+            structuredLogs,
+        },
         clientIO: clientIO,
         constructionProvider: getDefaultConstructionProvider(),
         indexingServiceRegistry: await getIndexingServiceRegistry(instanceDir),

ts/packages/cache/src/cache/cache.ts

@@ -498,11 +498,15 @@ export class AgentCache {
                         message: `Grammar generation error: ${error.message}`,
                     };
 
-                    this.logger?.logEvent("grammarGeneration", {
-                        request: requestAction.request,
-                        success: false,
-                        error: error.message,
-                    });
+                    this.logger?.logEvent(
+                        "grammarGeneration",
+                        {
+                            request: requestAction.request,
+                            success: false,
+                            error: error.message,
+                        },
+                        "error",
+                    );
                 }
             }
 
@@ -518,15 +522,19 @@ export class AgentCache {
                 ...(grammarResult !== undefined && { grammarResult }),
             };
         } catch (e: any) {
-            this.logger?.logEvent("error", {
-                request: requestAction.request,
-                actions: requestAction.actions,
-                history: requestAction.history,
-                cache,
-                options,
-                message: e.message,
-                stack: e.stack,
-            });
+            this.logger?.logEvent(
+                "error",
+                {
+                    request: requestAction.request,
+                    actions: requestAction.actions,
+                    history: requestAction.history,
+                    cache,
+                    options,
+                    message: e.message,
+                    stack: e.stack,
+                },
+                "error",
+            );
             throw e;
         }
     }

ts/packages/cli/bin/dev.js

@@ -4,6 +4,7 @@
 
 import { loadConfigSync } from "@typeagent/config";
 import { otel } from "@typeagent/telemetry";
+import registerDebug from "debug";
 import { registerEarlyTelemetrySignalHandlers } from "../src/telemetry.ts";
 loadConfigSync();
 
@@ -15,7 +16,10 @@ async function main() {
     process.env.NODE_ENV = "development";
     settings.debug = true;
     try {
-        await otel.initTelemetry();
+        await otel.initTelemetry({
+            processName: "cli",
+            debugModules: [registerDebug],
+        });
         await run(process.argv.slice(2), import.meta.url);
         await flush();
         await otel.shutdownTelemetry();

ts/packages/cli/bin/run.js

@@ -4,6 +4,7 @@
 
 import { loadConfigSync } from "@typeagent/config";
 import { otel } from "@typeagent/telemetry";
+import registerDebug from "debug";
 import { registerEarlyTelemetrySignalHandlers } from "../dist/telemetry.js";
 loadConfigSync();
 
@@ -12,7 +13,10 @@ registerEarlyTelemetrySignalHandlers();
 async function main() {
     const { flush, handle, run } = await import("@oclif/core");
     try {
-        await otel.initTelemetry();
+        await otel.initTelemetry({
+            processName: "cli",
+            debugModules: [registerDebug],
+        });
         await run(process.argv.slice(2), import.meta.url);
         await flush();
         await otel.shutdownTelemetry();

ts/packages/dispatcher/dispatcher/src/command/command.ts

@@ -392,11 +392,15 @@ export async function processCommandNoLock(
         );
         debugCommandError(e.stack);
 
-        context?.logger?.logEvent("command:exception", {
-            request: originalInput,
-            message: e.message,
-            stack: e.stack,
-        });
+        context?.logger?.logEvent(
+            "command:exception",
+            {
+                request: originalInput,
+                message: e.message,
+                stack: e.stack,
+            },
+            "error",
+        );
     }
 }
 

ts/packages/dispatcher/dispatcher/src/context/commandHandlerContext.ts

@@ -51,6 +51,7 @@ import {
     ensureDirectory,
     lockInstanceDir,
 } from "../utils/fsUtils.js";
+import { createDispatcherOtelLoggerSink } from "../otel/structuredLogSink.js";
 import {
     ActionContext,
     AppAgentEvent,
@@ -599,6 +600,11 @@ export type DispatcherOptions = DeepPartialUndefined<DispatcherConfig> & {
          * Default false: each request starts a new trace.
          */
         joinActiveTrace?: boolean;
+        /**
+         * Export structured dispatcher events through the global OTel logs
+         * provider. Default false because event payloads may contain user data.
+         */
+        structuredLogs?: boolean;
     };
 
     // Additional integration options
@@ -727,7 +733,11 @@ function getCosmosFactories(): PromptLoggerOptions {
     return result;
 }
 
-function getLoggerSink(isDbEnabled: () => boolean, clientIO: ClientIO) {
+function getLoggerSink(
+    isDbEnabled: () => boolean,
+    clientIO: ClientIO,
+    structuredLogs: boolean,
+) {
     const debugLoggerSink = createDebugLoggerSink();
     let dbLoggerSink: LoggerSink | undefined;
 
@@ -759,11 +769,14 @@ function getLoggerSink(isDbEnabled: () => boolean, clientIO: ClientIO) {
         );
     }
 
-    return new MultiSinkLogger(
+    const sinks =
         dbLoggerSink === undefined
             ? [debugLoggerSink]
-            : [debugLoggerSink, dbLoggerSink],
-    );
+            : [debugLoggerSink, dbLoggerSink];
+    if (structuredLogs) {
+        sinks.push(createDispatcherOtelLoggerSink());
+    }
+    return new MultiSinkLogger(sinks);
 }
 
 async function lockEmbeddingCacheDir(context: CommandHandlerContext) {
@@ -1206,7 +1219,11 @@ export async function initializeCommandHandlerContext(
         const sessionDirPath = session.getSessionDirPath();
         debug(`Session directory: ${sessionDirPath}`);
         const clientIO = options?.clientIO ?? nullClientIO;
-        const loggerSink = getLoggerSink(() => context.dblogging, clientIO);
+        const loggerSink = getLoggerSink(
+            () => context.dblogging,
+            clientIO,
+            options?.telemetry?.structuredLogs === true,
+        );
         const activationId = randomUUID();
         const traceId = options?.traceId;
         const logger = new ChildLogger(loggerSink, DispatcherName, {
@@ -1415,8 +1432,8 @@ export async function initializeCommandHandlerContext(
             },
             context.logger
                 ? {
-                      logEvent: (name, data) =>
-                          context.logger?.logEvent(name, data as any),
+                      logEvent: (name, data, severity) =>
+                          context.logger?.logEvent(name, data as any, severity),
                   }
                 : undefined,
         );

ts/packages/dispatcher/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts

@@ -284,13 +284,17 @@ async function canTranslateWithoutContext(
             newActions,
         });
     } catch (e: any) {
-        logger?.logEvent("contextlessTranslation", {
-            requestAction,
-            actions: oldActions,
-            history: requestAction.history,
-            newActions,
-            error: e.message,
-        });
+        logger?.logEvent(
+            "contextlessTranslation",
+            {
+                requestAction,
+                actions: oldActions,
+                history: requestAction.history,
+                newActions,
+                error: e.message,
+            },
+            "error",
+        );
         throw e;
     }
 }
@@ -824,11 +828,15 @@ export class RequestCommandHandler implements CommandHandler {
                         DispatcherName,
                     );
                 }
-                systemContext?.logger?.logEvent("request:exception", {
-                    request,
-                    message: e.message,
-                    stack: e.stack,
-                });
+                systemContext?.logger?.logEvent(
+                    "request:exception",
+                    {
+                        request,
+                        message: e.message,
+                        stack: e.stack,
+                    },
+                    "error",
+                );
                 throw e;
             }