From b655fa4be2d3c50d568262ab3f9ec0c017ee4879 Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Wed, 23 Sep 2026 02:41:52 -0700 Subject: [PATCH 1/6] Add opt-in synthetic engine product telemetry Introduce isolated, default-off engine telemetry with explicit validation gates, bounded aggregation, and Azure Monitor export. Cover startup, configuration, request and transport completion with synthetic regression tests. Honor the umbrella opt-out and preserve customer telemetry independence. Keep internal design drafts local and ignored, outside the published commit history. --- .gitignore | 6 + docs/telemetry.md | 114 ++ .../Core/McpEndpointRouteBuilderExtensions.cs | 34 +- .../Core/McpServerConfiguration.cs | 3 +- .../Core/McpStdioServer.cs | 14 +- .../Utils/McpProductResponseCompletion.cs | 126 ++ .../Utils/McpProductResponseStream.cs | 87 + .../Utils/McpTelemetryHelper.cs | 307 +++ src/Config/FileSystemRuntimeConfigLoader.cs | 21 + src/Config/ObjectModel/RuntimeConfig.cs | 9 + src/Config/Properties/AssemblyInfo.cs | 2 + src/Config/RuntimeConfigLoader.cs | 64 +- .../Telemetry/ApplicationNameTelemetry.cs | 90 +- .../Telemetry/ProductTelemetryPolicy.cs | 37 + .../TelemetryConfigurationPresence.cs | 242 +++ src/Core/Azure.DataApiBuilder.Core.csproj | 5 + .../Configurations/RuntimeConfigProvider.cs | 57 +- src/Core/Resolvers/CosmosClientProvider.cs | 20 +- src/Core/Resolvers/QueryExecutor.cs | 101 +- .../Services/Embeddings/EmbeddingService.cs | 93 +- src/Core/Services/ExecutionHelper.cs | 81 + src/Core/Services/RestService.cs | 84 + .../Product/EngineTelemetryAggregator.cs | 364 ++++ .../Product/EngineTelemetryCacheObserver.cs | 91 + .../EngineTelemetryConfigurationSnapshot.cs | 751 ++++++++ .../Product/EngineTelemetryContext.cs | 65 + .../Product/EngineTelemetryDelivery.cs | 409 ++++ .../Telemetry/Product/EngineTelemetryEvent.cs | 24 + .../Product/EngineTelemetryIdentityStore.cs | 515 +++++ .../EngineTelemetryMeasurementScope.cs | 47 + .../Product/EngineTelemetryMeasurements.cs | 136 ++ .../Product/EngineTelemetryOptions.cs | 41 + .../Product/EngineTelemetryRequestScope.cs | 105 + .../Product/EngineTelemetrySession.cs | 594 ++++++ .../Product/EngineTelemetryValueFormatter.cs | 71 + .../Product/IEngineTelemetryExporter.cs | 15 + .../Product/IProductTelemetryControl.cs | 20 + src/Directory.Packages.props | 11 +- .../Azure.DataApiBuilder.Service.Tests.csproj | 2 + ...ApplicationInsightsEventAttributesTests.cs | 107 ++ ...cationInsightsTelemetryDestinationTests.cs | 94 + .../EngineTelemetryAggregatorTests.cs | 563 ++++++ .../Telemetry/EngineTelemetryCacheTests.cs | 679 +++++++ .../Telemetry/EngineTelemetryCloudTests.cs | 315 +++ .../Telemetry/EngineTelemetryDeliveryTests.cs | 1013 ++++++++++ .../EngineTelemetryEmbeddingTests.cs | 420 ++++ .../Telemetry/EngineTelemetryExporterTests.cs | 706 +++++++ .../EngineTelemetryGraphQLIntegrationTests.cs | 532 ++++++ .../Telemetry/EngineTelemetryIdentityTests.cs | 703 +++++++ .../Telemetry/EngineTelemetryLocalDbTests.cs | 1050 ++++++++++ .../Telemetry/EngineTelemetryMcpHttpTests.cs | 338 ++++ .../EngineTelemetryMcpIntegrationTests.cs | 801 ++++++++ .../Telemetry/EngineTelemetryPresenceTests.cs | 588 ++++++ .../Telemetry/EngineTelemetryProtocolTests.cs | 729 +++++++ .../EngineTelemetryQueryExecutorTests.cs | 165 ++ .../Telemetry/EngineTelemetryReloadTests.cs | 603 ++++++ .../Telemetry/EngineTelemetrySessionTests.cs | 1683 +++++++++++++++++ .../Telemetry/EngineTelemetrySnapshotTests.cs | 878 +++++++++ .../Telemetry/ProductTelemetryOptOutTests.cs | 442 +++++ .../UnitTests/McpStdioHelperTests.cs | 67 +- .../PostgreSqlQueryExecutorUnitTests.cs | 46 +- src/Service.Tests/telemetry.runsettings | 10 + .../Azure.DataApiBuilder.Service.csproj | 1 + .../Controllers/ConfigurationController.cs | 8 + src/Service/Program.cs | 48 +- src/Service/Startup.cs | 70 +- .../ApplicationInsightsEventAttributes.cs | 66 + ...ApplicationInsightsTelemetryDestination.cs | 132 ++ ...ineTelemetryApplicationInsightsExporter.cs | 279 +++ ...ngineTelemetryEmbeddingEndpointMetadata.cs | 16 + .../Telemetry/EngineTelemetryExportAttempt.cs | 19 + .../EngineTelemetryGraphQLListener.cs | 332 ++++ .../Telemetry/EngineTelemetryHealthProbe.cs | 27 + .../Telemetry/EngineTelemetryHosting.cs | 121 ++ .../EngineTelemetryHttpCompletion.cs | 112 ++ .../EngineTelemetryHttpMiddleware.cs | 145 ++ .../EngineTelemetrySdkTransportHandler.cs | 114 ++ src/Service/Utilities/McpStdioHelper.cs | 24 +- 78 files changed, 18811 insertions(+), 93 deletions(-) create mode 100644 docs/telemetry.md create mode 100644 src/Azure.DataApiBuilder.Mcp/Utils/McpProductResponseCompletion.cs create mode 100644 src/Azure.DataApiBuilder.Mcp/Utils/McpProductResponseStream.cs create mode 100644 src/Config/Telemetry/ProductTelemetryPolicy.cs create mode 100644 src/Config/Telemetry/TelemetryConfigurationPresence.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryAggregator.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryCacheObserver.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryConfigurationSnapshot.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryContext.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryDelivery.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryEvent.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryIdentityStore.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryMeasurementScope.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryMeasurements.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryOptions.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryRequestScope.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetrySession.cs create mode 100644 src/Core/Telemetry/Product/EngineTelemetryValueFormatter.cs create mode 100644 src/Core/Telemetry/Product/IEngineTelemetryExporter.cs create mode 100644 src/Core/Telemetry/Product/IProductTelemetryControl.cs create mode 100644 src/Service.Tests/Telemetry/ApplicationInsightsEventAttributesTests.cs create mode 100644 src/Service.Tests/Telemetry/ApplicationInsightsTelemetryDestinationTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryAggregatorTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryCacheTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryCloudTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryDeliveryTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryEmbeddingTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryExporterTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryGraphQLIntegrationTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryIdentityTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryLocalDbTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryMcpHttpTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryMcpIntegrationTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryPresenceTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryProtocolTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryQueryExecutorTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetrySessionTests.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetrySnapshotTests.cs create mode 100644 src/Service.Tests/Telemetry/ProductTelemetryOptOutTests.cs create mode 100644 src/Service.Tests/telemetry.runsettings create mode 100644 src/Service/Telemetry/ApplicationInsightsEventAttributes.cs create mode 100644 src/Service/Telemetry/ApplicationInsightsTelemetryDestination.cs create mode 100644 src/Service/Telemetry/EngineTelemetryApplicationInsightsExporter.cs create mode 100644 src/Service/Telemetry/EngineTelemetryEmbeddingEndpointMetadata.cs create mode 100644 src/Service/Telemetry/EngineTelemetryExportAttempt.cs create mode 100644 src/Service/Telemetry/EngineTelemetryGraphQLListener.cs create mode 100644 src/Service/Telemetry/EngineTelemetryHealthProbe.cs create mode 100644 src/Service/Telemetry/EngineTelemetryHosting.cs create mode 100644 src/Service/Telemetry/EngineTelemetryHttpCompletion.cs create mode 100644 src/Service/Telemetry/EngineTelemetryHttpMiddleware.cs create mode 100644 src/Service/Telemetry/EngineTelemetrySdkTransportHandler.cs diff --git a/.gitignore b/.gitignore index 4de9ba1a81..728caefd82 100644 --- a/.gitignore +++ b/.gitignore @@ -37,5 +37,11 @@ dab-config*.json # Local-Only files .env +# Internal telemetry design drafts (keep local copies out of publication) +/docs/design/engine-telemetry-functional-spec.md +/docs/design/engine-telemetry-implementation-handoff.md +/docs/design/engine-telemetry-requirements-review.md +/docs/design/engine-telemetry-technical-design.md + # Verify test files *.received.* \ No newline at end of file diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000000..256f431cb3 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,114 @@ +# Engine product telemetry validation + +Engine product telemetry is **off by default in every build**. This implementation is for explicit synthetic validation; production enablement requires privacy/security review. Customer-configured logs, traces, metrics and telemetry destinations are separate. + +## What "synthetic" means + +Synthetic means **deliberately generated test activity**, not real customer usage. A live smoke test creates made-up rows in a temporary database, starts the actual DAB engine, makes real requests and checks the resulting events in Azure. The database calls, measured timings and delivery are real; the workload is artificial. Runtime and OS categories still describe the actual test process. + +| Use of the term | Meaning | +| --- | --- | +| Test fixtures | Made-up configuration, rows, queries and errors. Recognizable sentinel values help detect accidental collection. | +| Product test-mode switch | `DAB_PRODUCT_TELEMETRY_TEST_MODE` permits the current validation-only path. It does not generate test data or change application requests. | +| Internal controls | `EnableSyntheticCollection` / `enableSyntheticCollection` enable internal collectors and sessions for validation; their defaults are off. | +| Event marker | Current event/window objects have `IsSynthetic=true`. The exporter writes `dab_is_synthetic=true` and rejects nonsynthetic events. Queries must explicitly exclude this custom marker from real-usage reporting; Azure does not do that automatically. | +| Local exporter tests | Fake HTTP/exporters and names such as `synthetic.invalid` exercise serialization without contacting Azure. They are distinct from live-cloud tests. | +| Live test-runner opt-in | `DAB_TELEMETRY_CLOUD_TEST=1` allows the `EngineTelemetryCloud` test to send to an explicitly supplied destination. This is a test-harness safeguard, not another customer-facing product setting. | + +**The switch and marker do not anonymize or sanitize data, enforce a test-only database, or grant privacy approval.** Enabling test mode against real customer traffic would mislabel it as test traffic. Use an isolated fixture and destination; all field restrictions still apply. The marker is not a Microsoft-internal classification or an Azure availability-test marker. Likewise, a resource's synthetic-data-only tag is an administrative label, not an ingestion filter. + +This temporary validation opt-in is not the intended production enablement policy. Production requires reviewed enablement and event classification, not asking customers to set a test-mode variable. + +## Selecting a validation destination + +Set these variables in a dedicated synthetic test process, then start DAB normally: + +| Variable | Value / purpose | +| --- | --- | +| `DAB_PRODUCT_TELEMETRY_TEST_MODE` | `1` or `true` explicitly enables synthetic validation. A destination alone never enables collection. | +| `DAB_PRODUCT_TELEMETRY_CONNECTION_STRING` | The complete connection string copied from the chosen Application Insights resource. This is independent of `APPLICATIONINSIGHTS_CONNECTION_STRING` and the customer's runtime telemetry configuration. | +| `APPLICATIONINSIGHTS_STATSBEAT_DISABLED` | Exactly `true` (case-insensitive, no surrounding whitespace), set before the dedicated validation process starts. Suppresses Microsoft's SDK-health/usage stream for this validation phase. | +| `APPLICATIONINSIGHTS_SDKSTATS_DISABLED` | Exactly `true` (case-insensitive, no surrounding whitespace), set before process startup. Separately suppresses the SDK's customer-facing delivery counters and their metadata detection. | +| `DAB_TELEMETRY_OPT_OUT` | `1` or `true` overrides enablement. Values are trimmed and `true` is case-insensitive. | + +Changing only `DAB_PRODUCT_TELEMETRY_CONNECTION_STRING` and restarting the process selects another instance. There is no compiled-in test or production destination, no automatic fallback to customer configuration, and no required AME credential in DAB. The resource must permit connection-string/key-based ingestion; the current adapter does not obtain Entra tokens. + +The connection string needs a nonempty GUID instrumentation key and an explicit HTTPS ingestion endpoint (or a supported Azure endpoint suffix). Credential-bearing extensions, malformed endpoints and redirects are rejected. Do not commit connection strings or paste credentials into diagnostics. Azure management access is needed to create/manage/query the destination, not for every engine installation to send telemetry. + +The product adapter uses **OpenTelemetry .NET with `Azure.Monitor.OpenTelemetry.Exporter` 1.9.0**. A private logger factory emits explicit custom events to `AzureMonitorLogExporter`; the SDK owns Application Insights serialization and response interpretation. No automatic request/dependency collectors, host logging providers, SDK batch processor or event disk spool are registered. The existing DAB delivery worker calls the synchronous SDK exporter; no task is created per event. + +**The two SDK statistics opt-outs are temporary validation prerequisites, not requirements for sending custom events to Application Insights.** Statsbeat normally reports SDK health/usage to Microsoft; customer-facing SDK stats report delivery counters to the configured resource. Both can perform metadata detection outside DAB's current allowlist. DAB checks that both are disabled before initializing the product SDK and otherwise leaves product collection off. It never sets environment variables or `AppContext` switches. Do not apply these opt-outs to an unrelated customer host: SDKs in the same process can observe them too. + +SDK 1.9.0 shares transmitters by connection string and caches some process configuration. Separate adapter objects do not guarantee independent transport, settings or lifetime when another exporter uses the same routing. This phase supports a dedicated synthetic standalone process, not arbitrary embedded coexistence. Public embedded collection and production collection remain off pending their explicit policy and SDK-isolation review. Switching the Application Insights connection string changes the destination, not the event contract or identities; no OneCollector/Aria migration is implied. + +## Enablement, notice and reset + +- Disabled collection does not initialize identities, counters, timers, notices or sender resources. Saved identity state is untouched. +- An enabled run writes a noninteractive notice to stderr before sending, never to MCP stdout. Notice persistence is currently unavailable, so the notice repeats per enabled run. +- The API ID is a random UUID, stored best-effort beside an existing root configuration in a sidecar ending in `.dab-telemetry.json`. It is not a configuration hash, a person, a machine or a guarantee of replica grouping. The sidecar contains only a format version and the random ID. +- Safe persistence is currently supported on Windows and Linux x64/arm64. Missing, read-only, unsafe or unsupported storage uses a flagged per-run ephemeral API ID. Direct engine runs do not allocate CLI installation IDs. +- To reset, stop all affected processes and delete that configuration's identity sidecar. The next enabled run generates an unrelated ID. Opt-out is not reset. Reset does not delete earlier ingested telemetry. +- Hosts can resolve `IProductTelemetryControl` from the engine service provider and call `Disable()` to discard counters and pending delivery without a final send. Already transmitted data cannot be retracted. Environment changes require a new process. +- The umbrella opt-out also prevents the DAB-added database Application Name segment, including its version marker. Recognizable existing DAB segments are removed at configuration load while customer prefixes are retained. Ambiguous/truncated customer text is not destructively guessed. The legacy `DAB_TELEMETRY_APPNAME_OPT_OUT` used alone retains its previous version-only behavior. +- Cosmos DB clients also omit DAB's additional Application Name/user-agent suffix under the umbrella opt-out; the Cosmos SDK's own identification is unchanged. + +## Collected fields + +All events include schema version, random event ID, engine session ID, sequence, UTC occurrence time, configuration epoch and `dab_is_synthetic=true`. Event retries preserve those values. Available API identity includes `dab_api_id` and `dab_api_id_stability` (`newly_saved`, `reused`, `ephemeral`). No CLI identity or fabricated launch linkage is added. + +Context includes DAB version; coarse OS family/version; process architecture; normalized .NET version; execution mode; categorical launcher/hosting/container detection; and distribution/channel/packaging labels. Known CLI/service entry assemblies select a categorical launcher; other entry points remain unknown. Distribution/channel/packaging remain `unknown` without reliable build provenance; synthetic opt-in does not prove source packaging. No network discovery or raw environment value is recorded. + +Configuration events contain `snapshot_schema=configuration-v1`, configuration delivery, source-provider categories and bucketed counts/limits. Fixed feature families are: + +| Family | Settings | +| --- | --- | +| API/runtime | `runtime.rest`, `runtime.graphql`, `runtime.mcp`, `runtime.health`, `runtime.cache`, `runtime.cache.l2`, `runtime.rest.strict_body`, `runtime.graphql.multiple_create` | +| Integrations | `integrations.key_vault`, `integrations.autoentities`, `integrations.multiple_source_files`, `runtime.embeddings`, `runtime.embeddings.endpoint`, endpoint presence | +| Authentication | `authentication.provider`, `host.mode`, `data_sources.obo`, `data_sources.session_context` | +| Customer observability enablement only | `customer_telemetry.open_telemetry`, `customer_telemetry.application_insights`, `customer_telemetry.log_analytics`, `customer_telemetry.file` | +| Entity capabilities | Any table/view/procedure/document, cache, REST/GraphQL/MCP exposure, custom roles, policies, descriptions, relationships and parameter embeddings | +| Scale/limits | Source/entity count, distinct provider count, page sizes, response bytes, cache TTL and modeled query timeouts | + +Where applicable, `.configured` and `.effective` are separate. States are `enabled`, `disabled`, `missing`, `unsupported`, `unknown` and `not_applicable`. Original-input presence is captured as fixed-size, value-free metadata only during loads authorized by an enabled engine session, including nested source files. Ambient test mode alone does not enable capture in a CLI or disabled embedded host. Lost provenance is not reconstructed by serializing defaults. No customer entity/role names or values are retained in that metadata. Unsupported capabilities remain unsupported even when there are zero entities. + +## Events and usage + +Events are `dab.engine.process_started`, `ready`, `startup_failed`, `configuration_changed`, `configuration_change_failed`, `first_request_served`, `first_successful_request`, `heartbeat`, `usage_summary` and `stopped`, all under the `dab.engine.` prefix. + +The mapped embedding HTTP endpoint is included as REST request traffic even when its path is outside the entity REST prefix. Its embedding-service invocations form the separate embedding measurement family; cache-served invocations count without inventing a database attempt. Cache lookups include the dedicated embedding cache when enabled, without double-counting the default-cache fallback. This identification uses endpoint metadata, not request/response contents. Valid REST entity routes are not excluded merely because their names resemble documentation or static assets. + +Readiness requires accepted usable configuration and host/tool readiness. First-served and first-success are independently once per run. Discovery, health, documentation, introspection-only GraphQL, and MCP protocol-control/metadata traffic are excluded. HTTP 200 with GraphQL errors or an MCP tool error is not logical success. Variable-batch GraphQL results count independently. The current incremental/streaming GraphQL path reports `unknown` rather than inventing success from an unfinished stream. + +The dedicated internal health client marks its self-probes with an in-memory per-session value so REST/GraphQL health queries cannot establish usage milestones or add usage counts. The marker is neither stored nor exported and does not grant authorization. Ordinary requests, including callers supplying an unrelated marker, remain eligible. System roles are classified case-insensitively, matching authentication behavior. + +MCP HTTP completion is observed per tool response, including legacy SSE sessions. The SDK's nonserialized message context carries only an opaque completion holder; an outgoing filter and byte-opaque stream observer check write/flush completion without inspecting payloads or storing request IDs. A completed tool response does not wait for session disconnection. A send with no observable write/flush remains `unknown`, and observed write failures remain failures even if the SDK absorbs the exception. This proves server-side flush, not client receipt. + +Usage summaries separate requests, logical operations, database attempts, each cache layer, embedding calls and HTTP outcome classes. No HTTP denominator applies to stdio. Outcomes partition request/operation counts into success, failure, partial failure, cancellation and unknown. Latency is a noncumulative histogram with inclusive millisecond bounds **1, 5, 10, 50, 100, 500, 1000, 5000, 30000**, plus an unbounded final bucket. Timed counts, completeness and capping are explicit. + +Six-hour windows align to UTC 00:00/06:00/12:00/18:00. Work retains its captured configuration epoch across reloads and completes in its completion-time segment. SQL attempt attribution first uses the captured source metadata, then matching current metadata; if neither still describes the executing source, the attempt remains counted with an `unknown` provider. Final shutdown includes only the remaining delta. Memory limits are 256 occupied series per open window, four pending windows and 256 queued/in-flight immutable events. Excess observations/events are dropped with measurable loss counters. Counter saturation and backward-clock loss are explicit. + +### Coverage limits + +- Actual SQL command executions, including observed retries, are counted. Cosmos SDK-internal physical retries are not available through the supported public hooks used here; `database_attempt_coverage=sql_commands_only` must not be interpreted as zero Cosmos attempts. +- Cache measurements observe actual L1/L2 events with a live eligible request context. Background/unattributed activity is not invented; coverage is labeled accordingly. +- Full backend ingestion, deployed destination policy and sender isolation in arbitrary embedded hosts are separate acceptance gates. Local tests do not prove cloud receipt. + +## Privacy and failure behavior + +No request/response contents, commands, SQL, URLs, connection strings, customer/host/database/entity/role names, claims, tokens, observed IP addresses, exception text, hardware identifiers or request/trace IDs are product event fields. The private logger excludes scopes/resources and clears diagnostic and ambient trace fields before SDK export. A transport guard rejects unexpected SDK envelope context rather than rewriting it. SDK diagnostic `EventSource` output can still be observed by a host that deliberately subscribes; general host isolation is not claimed. Identity is pseudonymous, not anonymous. + +The SDK maps the constant `microsoft.client.ip=0.0.0.0` attribute to the required `ai.location.ip=0.0.0.0` tag. The guard permits the SDK/runtime version tag and null resource tags, but rejects populated host/user/trace context or missing IP suppression. Omitting the IP tag allows the receiver to derive geolocation from the connection before masking the stored IP. Destination validation must also check that stored location fields are empty; IP masking alone is insufficient. + +Delivery is bounded and memory-only: no event spool or restart replay. Export happens only on the worker, with at most three attempts per immutable event, a one-second HTTP attempt budget and a two-second graceful drain budget. Disable cancels in-flight work and discards pending data. Outages, process termination, caps and transport ambiguity can lose or duplicate received events; deduplicate by `dab_event_id`. A missing stop is not proof of a crash. + +Identity resolution runs outside the session lock; an in-flight filesystem operation cannot hold up disable/shutdown or re-enable collection when it finishes. An older configuration acceptance finishing late cannot overwrite a newer acceptance. An already-cancelled stop discards without constructing final events, and cancellation from a later stop caller also cancels the shared pending drain. Filesystem calls themselves are best-effort and are not forcibly interrupted. + +Each attempt sends one event through the SDK. Attribute lengths/counts are bounded before SDK conversion; the transport rejects payloads above 64 KiB and response bodies above 16 KiB. SDK offline storage and pipeline retries are disabled, so the DAB worker owns the finite retry budget. The temporary endpoint guard refuses redirects. SDK success is not a guarantee of later storage or exactly-once delivery. It does not mean data was merely queued: this path has no SDK batch queue or offline backlog. Supported production handling of throttling/redirects and independent SDK lifetime remains part of the integration review. + +## Local validation + +The `EngineTelemetry` tests use synthetic input and fake HTTP/exporters. The test project's default [run settings](../src/Service.Tests/telemetry.runsettings) set both SDK statistics opt-outs before testhost startup; an explicitly supplied replacement settings file must do the same. `EngineTelemetryLocalDb` additionally executes actual REST, GraphQL and MCP stdio against a unique LocalDB database and cleans up after host disposal. Exporter tests inspect actual SDK-serialized envelopes, cancellation/response handling, unchanged environment settings and the shared-transmitter limitation. OS-permission and symlink cases may skip when prerequisites are unavailable. + +The separately gated `EngineTelemetryCloud` test launches the exact standalone engine selected by `DAB_TELEMETRY_CLOUD_TEST_ENGINE`, using the approved `DAB_PRODUCT_TELEMETRY_CONNECTION_STRING` supplied privately in the test environment. It sets both SDK statistics opt-outs only in that child process. Without `DAB_TELEMETRY_CLOUD_TEST=1`, it skips before database/cloud activity. It creates and removes its own LocalDB fixture, performs two successful reads (including an empty result) and one rejected read, exercises excluded discovery/control calls, then closes stdin for graceful shutdown. Its receipt contains the engine hash, time window, random API identity and expected counts, not the connection string or fixture contents. + +A passing serving test alone does not prove delivery. Correlate its API identity to the engine session and query `customEvents` in Application Insights and `AppEvents` in the linked workspace. Compare event IDs, deduplicate by `dab_event_id`, reconcile usage counts and inspect privacy indicators. These are two query views of the same workspace-backed data, not separate DAB uploads. Allow for ingestion delay and out-of-order arrival; do not rerun the workload just because the first query is incomplete. Application Insights' default Overview request/availability charts do not display these custom events; use **Monitoring > Logs**. diff --git a/src/Azure.DataApiBuilder.Mcp/Core/McpEndpointRouteBuilderExtensions.cs b/src/Azure.DataApiBuilder.Mcp/Core/McpEndpointRouteBuilderExtensions.cs index bb3a1e5dba..b697f3fb87 100644 --- a/src/Azure.DataApiBuilder.Mcp/Core/McpEndpointRouteBuilderExtensions.cs +++ b/src/Azure.DataApiBuilder.Mcp/Core/McpEndpointRouteBuilderExtensions.cs @@ -4,8 +4,12 @@ using System.Diagnostics.CodeAnalysis; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Core.Configurations; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using Azure.DataApiBuilder.Mcp.Utils; using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; namespace Azure.DataApiBuilder.Mcp.Core { @@ -37,7 +41,35 @@ public static IEndpointRouteBuilder MapDabMcp( string mcpPath = mcpOptions.Path ?? McpRuntimeOptions.DEFAULT_PATH; // Map the MCP endpoint - endpoints.MapMcp(mcpPath); + endpoints.MapMcp(mcpPath).Add(builder => + { + RequestDelegate? next = builder.RequestDelegate; + if (next is null) + { + return; + } + + builder.RequestDelegate = async context => + { + if (context.RequestServices.GetService()?.IsEnabled != true) + { + await next(context); + return; + } + + Stream original = context.Response.Body; + using McpProductResponseStream observer = new(original); + context.Response.Body = observer; + try + { + await next(context); + } + finally + { + context.Response.Body = original; + } + }; + }); return endpoints; } diff --git a/src/Azure.DataApiBuilder.Mcp/Core/McpServerConfiguration.cs b/src/Azure.DataApiBuilder.Mcp/Core/McpServerConfiguration.cs index 20040588fe..8fa315c6a3 100644 --- a/src/Azure.DataApiBuilder.Mcp/Core/McpServerConfiguration.cs +++ b/src/Azure.DataApiBuilder.Mcp/Core/McpServerConfiguration.cs @@ -82,13 +82,14 @@ internal static IServiceCollection ConfigureMcpServer(this IServiceCollection se } return await McpTelemetryHelper.ExecuteWithTelemetryAsync( - tool, toolName, arguments, request.Services, ct); + tool, toolName, arguments, request.Services, ct, sdkResponseItems: request.Items); } finally { arguments?.Dispose(); } }) + .WithMessageFilters(filters => filters.AddOutgoingFilter(McpProductResponseCompletion.Filter)) .WithHttpTransport(); // Configure underlying MCP server options diff --git a/src/Azure.DataApiBuilder.Mcp/Core/McpStdioServer.cs b/src/Azure.DataApiBuilder.Mcp/Core/McpStdioServer.cs index aef3b63dcc..d2edd1f904 100644 --- a/src/Azure.DataApiBuilder.Mcp/Core/McpStdioServer.cs +++ b/src/Azure.DataApiBuilder.Mcp/Core/McpStdioServer.cs @@ -475,7 +475,6 @@ private async Task HandleCallToolAsync(JsonElement? id, JsonElement root, Cancel // helpers that read IHttpContextAccessor will see the role. We also ensure the // Simulator authentication handler can authenticate the user by flowing the // Authorization header commonly used in tests/simulator scenarios. - CallToolResult callResult; IConfiguration? configuration = _serviceProvider.GetService(); string? stdioRole = configuration?.GetValue("MCP:Role"); if (!string.IsNullOrWhiteSpace(stdioRole)) @@ -504,8 +503,10 @@ private async Task HandleCallToolAsync(JsonElement? id, JsonElement root, Cancel try { // Execute the tool with the scoped service provider so any scoped services resolve correctly. - callResult = await McpTelemetryHelper.ExecuteWithTelemetryAsync( - tool, toolName!, argsDoc, scopedProvider, ct); + // Product completion must follow the successful stdout write, not just tool execution. + await McpTelemetryHelper.ExecuteWithTelemetryAsync( + tool, toolName!, argsDoc, scopedProvider, ct, + writeStdioResponse: result => HandleCallToolAsync(id ?? default, result)); } finally { @@ -518,11 +519,10 @@ private async Task HandleCallToolAsync(JsonElement? id, JsonElement root, Cancel } else { - callResult = await McpTelemetryHelper.ExecuteWithTelemetryAsync( - tool, toolName!, argsDoc, _serviceProvider, ct); + await McpTelemetryHelper.ExecuteWithTelemetryAsync( + tool, toolName!, argsDoc, _serviceProvider, ct, + writeStdioResponse: result => HandleCallToolAsync(id ?? default, result)); } - - await HandleCallToolAsync(id ?? default, callResult); } finally { diff --git a/src/Azure.DataApiBuilder.Mcp/Utils/McpProductResponseCompletion.cs b/src/Azure.DataApiBuilder.Mcp/Utils/McpProductResponseCompletion.cs new file mode 100644 index 0000000000..8979d03a4b --- /dev/null +++ b/src/Azure.DataApiBuilder.Mcp/Utils/McpProductResponseCompletion.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.DataApiBuilder.Core.Telemetry.Product; +using Microsoft.AspNetCore.Http; +using ModelContextProtocol.Protocol; +using ModelContextProtocol.Server; + +namespace Azure.DataApiBuilder.Mcp.Utils; + +/// +/// Correlates one tool response through the SDK's nonserialized Items, without retaining an +/// ID or payload. A byte-opaque stream observer confirms that this response actually flushed. +/// +internal sealed class McpProductResponseCompletion +{ + private const string ITEM_KEY = "DAB.ProductTelemetry.ResponseCompletion"; + private static readonly AsyncLocal _sending = new(); + private readonly EngineTelemetryRequestScope _request; + private CancellationTokenRegistration _cancellation; + private int _completed; + private int _wrote; + private int _flushed; + + internal McpProductResponseCompletion(EngineTelemetryRequestScope request, CancellationToken cancellationToken) + { + _request = request; + _cancellation = cancellationToken.UnsafeRegister(static value => + ((McpProductResponseCompletion)value!).Complete(EngineTelemetryOutcome.Canceled), this); + if (Volatile.Read(ref _completed) != 0) + { + _cancellation.Unregister(); + } + } + + internal static McpProductResponseCompletion Attach(IDictionary items, EngineTelemetryRequestScope request, + CancellationToken cancellationToken = default) + { + McpProductResponseCompletion completion = new(request, cancellationToken); + items[ITEM_KEY] = completion; + return completion; + } + + internal void Abandon(IDictionary items) + { + _cancellation.Unregister(); + items.Remove(ITEM_KEY); + } + + internal static void Wrote() => _sending.Value?.RecordWrite(); + + internal static void Flushed() => _sending.Value?.RecordFlush(); + + internal static void WriteFailed(bool canceled) => _sending.Value?.Complete(canceled + ? EngineTelemetryOutcome.Canceled : EngineTelemetryOutcome.Failure); + + internal static McpMessageHandler Filter(McpMessageHandler next) => async (context, cancellationToken) => + { + // The SDK copies the request context onto ordinary responses. SDK-generated error + // messages may not preserve Items; the tool wrapper already records thrown failures. + if (context.JsonRpcMessage is not JsonRpcResponse || + context.JsonRpcMessage.Context?.Items is not { } items || + !items.TryGetValue(ITEM_KEY, out object? value) || value is not McpProductResponseCompletion completion) + { + await next(context, cancellationToken).ConfigureAwait(false); + return; + } + + McpProductResponseCompletion? previous = _sending.Value; + _sending.Value = completion; + try + { + await next(context, cancellationToken).ConfigureAwait(false); + if (!completion._request.IsCompleted) + { + // Disposed transports can silently drop a send; an SDK return alone is not + // delivery. Preserve an unknown outcome if no write+flush was observed. + completion.Complete(Volatile.Read(ref completion._flushed) != 0 + ? completion._request.Outcome : EngineTelemetryOutcome.Unknown); + } + } + catch (OperationCanceledException) + { + completion.Complete(EngineTelemetryOutcome.Canceled); + throw; + } + catch (Exception) + { + completion.Complete(EngineTelemetryOutcome.Failure); + throw; + } + finally + { + _sending.Value = previous; + items.Remove(ITEM_KEY); + } + }; + + private void RecordWrite() => Volatile.Write(ref _wrote, 1); + + private void RecordFlush() + { + if (Volatile.Read(ref _wrote) != 0) + { + Volatile.Write(ref _flushed, 1); + } + } + + private void Complete(EngineTelemetryOutcome outcome) + { + if (Interlocked.Exchange(ref _completed, 1) != 0) + { + return; + } + + _cancellation.Unregister(); + try + { + _request.Complete(outcome, Volatile.Read(ref _flushed) != 0 ? StatusCodes.Status200OK : null); + } + catch (Exception) + { + // Optional product recording must never change SDK response delivery. + } + } +} diff --git a/src/Azure.DataApiBuilder.Mcp/Utils/McpProductResponseStream.cs b/src/Azure.DataApiBuilder.Mcp/Utils/McpProductResponseStream.cs new file mode 100644 index 0000000000..7c6e08826a --- /dev/null +++ b/src/Azure.DataApiBuilder.Mcp/Utils/McpProductResponseStream.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Azure.DataApiBuilder.Mcp.Utils; + +/// +/// Delegates every byte unchanged. Only successful write/flush boundaries and closed failure +/// categories are observed; no buffer, message text, request ID or exception is retained. +/// +internal sealed class McpProductResponseStream(Stream inner) : Stream +{ + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => inner.CanWrite; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + + public override void Flush() + { + try + { + inner.Flush(); + McpProductResponseCompletion.Flushed(); + } + catch (Exception exception) + { + McpProductResponseCompletion.WriteFailed(exception is OperationCanceledException); + throw; + } + } + + public override async Task FlushAsync(CancellationToken cancellationToken) + { + try + { + await inner.FlushAsync(cancellationToken).ConfigureAwait(false); + McpProductResponseCompletion.Flushed(); + } + catch (Exception exception) + { + McpProductResponseCompletion.WriteFailed(exception is OperationCanceledException || cancellationToken.IsCancellationRequested); + throw; + } + } + + public override void Write(byte[] buffer, int offset, int count) + { + try + { + inner.Write(buffer, offset, count); + if (count > 0) + { + McpProductResponseCompletion.Wrote(); + } + } + catch (Exception exception) + { + McpProductResponseCompletion.WriteFailed(exception is OperationCanceledException); + throw; + } + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + try + { + await inner.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + if (!buffer.IsEmpty) + { + McpProductResponseCompletion.Wrote(); + } + } + catch (Exception exception) + { + McpProductResponseCompletion.WriteFailed(exception is OperationCanceledException || cancellationToken.IsCancellationRequested); + throw; + } + } + + public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + public override void SetLength(long value) => inner.SetLength(value); + // The ASP.NET response owns the inner stream; this observer must never dispose it. +} diff --git a/src/Azure.DataApiBuilder.Mcp/Utils/McpTelemetryHelper.cs b/src/Azure.DataApiBuilder.Mcp/Utils/McpTelemetryHelper.cs index c423534816..070e01cb1a 100644 --- a/src/Azure.DataApiBuilder.Mcp/Utils/McpTelemetryHelper.cs +++ b/src/Azure.DataApiBuilder.Mcp/Utils/McpTelemetryHelper.cs @@ -4,11 +4,15 @@ using System.Diagnostics; using System.Text.Json; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Core.Authorization; using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Telemetry; +using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Mcp.Core; using Azure.DataApiBuilder.Mcp.Model; using Azure.DataApiBuilder.Service.Exceptions; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using ModelContextProtocol.Protocol; using static Azure.DataApiBuilder.Mcp.Model.McpEnums; @@ -30,8 +34,86 @@ internal static class McpTelemetryHelper /// The parsed JSON arguments for the tool (may be null). /// The service provider for resolving dependencies. /// Cancellation token. + /// For stdio, writes and flushes the response before product request completion. + /// SDK-owned nonserialized context for per-message HTTP response completion. /// The result of the tool execution. public static async Task ExecuteWithTelemetryAsync( + IMcpTool tool, + string toolName, + JsonDocument? arguments, + IServiceProvider serviceProvider, + CancellationToken cancellationToken, + Func? writeStdioResponse = null, + IDictionary? sdkResponseItems = null) + { + using EngineTelemetryRequestScope? request = BeginProductRequest( + tool, toolName, serviceProvider, writeStdioResponse is not null, + out EngineTelemetrySession? session, out HttpContext? httpContext, out bool isStdio); + ProductRequestCompletion? completion = request is null ? null : new(request, sdkResponseItems is null ? httpContext : null); + McpProductResponseCompletion? sdkCompletion = null; + if (request is not null && sdkResponseItems is not null) + { + sdkCompletion = McpProductResponseCompletion.Attach(sdkResponseItems, request, cancellationToken); + } + + try + { + // Keep the existing customer span and its attributes scoped to tool execution. + // Product collection does not consume that activity, its errors or its content. + CallToolResult result; + using (EngineTelemetryMeasurementScope? operation = request is null ? null : BeginProductOperation(session!, tool, toolName, arguments)) + { + try + { + result = await ExecuteWithCustomerTelemetryAsync( + tool, toolName, arguments, serviceProvider, cancellationToken); + EngineTelemetryOutcome outcome = ClassifyProductResult(result, cancellationToken); + request?.SetOutcome(outcome); + operation?.Complete(outcome); + } + catch (OperationCanceledException) + { + operation?.Complete(EngineTelemetryOutcome.Canceled); + throw; + } + } + + // Logical operation completion precedes transport completion. A failed write + // must not retroactively turn a successfully executed operation into a failure. + + if (writeStdioResponse is not null) + { + await writeStdioResponse(result); + completion?.Complete(); + } + else if (!isStdio && httpContext is null && sdkResponseItems is null) + { + // An embedded call has no response transport to wait for. + completion?.Complete(); + } + + // SDK HTTP completes per message after write/flush; other HTTP callers use + // OnCompleted. A stdio invocation without its writer must + // not be counted as served just because tool execution returned a value. + return result; + } + catch (OperationCanceledException) + { + sdkCompletion?.Abandon(sdkResponseItems!); + completion?.Fail(EngineTelemetryOutcome.Canceled); + throw; + } + catch (Exception) + { + sdkCompletion?.Abandon(sdkResponseItems!); + completion?.Fail(cancellationToken.IsCancellationRequested || httpContext?.RequestAborted.IsCancellationRequested == true + ? EngineTelemetryOutcome.Canceled + : EngineTelemetryOutcome.Failure); + throw; + } + } + + private static async Task ExecuteWithCustomerTelemetryAsync( IMcpTool tool, string toolName, JsonDocument? arguments, @@ -98,6 +180,231 @@ public static async Task ExecuteWithTelemetryAsync( } } + internal static bool IsProductDataTool(IMcpTool tool, string toolName) + => ClassifyProductOperation(tool, toolName) != EngineTelemetryOperation.Unknown; + + internal static EngineTelemetryOperation ClassifyProductOperation(IMcpTool tool, string toolName) + { + if (tool.ToolType == ToolType.Custom) + { + return EngineTelemetryOperation.Execute; + } + + // Unlike the customer operation label, an unknown built-in is not an execute + // request. Discovery and JSON-RPC protocol/control messages are not usage. + if (tool.ToolType != ToolType.BuiltIn) + { + return EngineTelemetryOperation.Unknown; + } + + return toolName.ToLowerInvariant() switch + { + "read_records" or "aggregate_records" => EngineTelemetryOperation.Read, + "create_record" or "update_record" or "delete_record" => EngineTelemetryOperation.Write, + "execute_entity" => EngineTelemetryOperation.Execute, + _ => EngineTelemetryOperation.Unknown + }; + } + + private static EngineTelemetryOutcome ClassifyProductResult(CallToolResult result, CancellationToken cancellationToken) + { + if (result.IsError != true) + { + return EngineTelemetryOutcome.Success; + } + + return cancellationToken.IsCancellationRequested + ? EngineTelemetryOutcome.Canceled + : EngineTelemetryOutcome.Failure; + } + + private static EngineTelemetryMeasurementScope? BeginProductOperation( + EngineTelemetrySession session, IMcpTool tool, string toolName, JsonDocument? arguments) + { + try + { + // Names are used only by the session's local accepted-config lookup. No name, + // argument, stored procedure identifier or result is passed to the aggregator. + string? entityName = null; + if (tool is DynamicCustomTool customTool) + { + entityName = customTool.EntityName; + } + else if (arguments?.RootElement.ValueKind == JsonValueKind.Object) + { + entityName = ExtractEntityNameFromArguments(arguments); + } + + return session.BeginOperation(entityName, ClassifyProductOperation(tool, toolName)); + } + catch (Exception) + { + // Product enrichment must not change tool validation or execution behavior. + return null; + } + } + + private static EngineTelemetryRequestScope? BeginProductRequest( + IMcpTool tool, + string toolName, + IServiceProvider services, + bool hasStdioWriter, + out EngineTelemetrySession? session, + out HttpContext? httpContext, + out bool isStdio) + { + try + { + return BeginProductRequestCore(tool, toolName, services, hasStdioWriter, + out session, out httpContext, out isStdio); + } + catch (Exception) + { + // Optional product dependencies/configuration must not change tool execution + // or emit failures to the customer's existing telemetry pipeline. + session = null; + httpContext = null; + isStdio = hasStdioWriter; + return null; + } + } + + private static EngineTelemetryRequestScope? BeginProductRequestCore( + IMcpTool tool, + string toolName, + IServiceProvider services, + bool hasStdioWriter, + out EngineTelemetrySession? session, + out HttpContext? httpContext, + out bool isStdio) + { + session = null; + httpContext = null; + isStdio = hasStdioWriter; + if (!IsProductDataTool(tool, toolName)) + { + return null; + } + + session = services.GetService() + ?? services.GetService()?.ProductTelemetry; + if (session?.IsEnabled != true) + { + return null; + } + + IConfiguration? configuration = services.GetService(); + isStdio |= configuration?.GetValue("MCP:StdioMode") == true; + // Stdio's authorization shim is a DefaultHttpContext, NOT a response socket. + // Do not read its status or register HTTP response callbacks on it. + if (!isStdio) + { + httpContext = services.GetService()?.HttpContext; + } + + string? role = isStdio + ? configuration?.GetValue("MCP:Role") + : httpContext?.Request.Headers[AuthorizationResolver.CLIENT_ROLE_HEADER].ToString(); + EngineTelemetryRole roleClass = EngineTelemetrySession.ClassifyRole( + role, httpContext?.User.Identity?.IsAuthenticated == true); + EngineTelemetryTransport transport = EngineTelemetryTransport.InProcess; + if (isStdio) + { + transport = EngineTelemetryTransport.Stdio; + } + else if (httpContext is not null) + { + transport = EngineTelemetryTransport.Http; + } + + return session.BeginRequest(EngineTelemetryApi.Mcp, transport, roleClass); + } + + /// + /// Owns one request's transport completion, without a per-request event queue or any + /// names, arguments, error messages or serialized tool results in product telemetry. + /// + private sealed class ProductRequestCompletion + { + private readonly EngineTelemetryRequestScope _request; + private readonly HttpContext? _httpContext; + private CancellationTokenRegistration _abortRegistration; + private int _completed; + + internal ProductRequestCompletion(EngineTelemetryRequestScope request, HttpContext? httpContext) + { + _request = request; + _httpContext = httpContext; + if (httpContext is not null) + { + lock (httpContext) + { + httpContext.Response.OnCompleted(static state => ((ProductRequestCompletion)state).ResponseCompletedAsync(), this); + } + + _abortRegistration = httpContext.RequestAborted.UnsafeRegister( + static state => ((ProductRequestCompletion)state!).Fail(EngineTelemetryOutcome.Canceled), this); + if (Volatile.Read(ref _completed) != 0) + { + _abortRegistration.Unregister(); + } + } + } + + internal void Complete() => Complete(_request.Outcome, httpStatusCode: null); + + internal void Fail(EngineTelemetryOutcome outcome) => Complete(outcome, httpStatusCode: null); + + private Task ResponseCompletedAsync() + { + if (Volatile.Read(ref _completed) != 0) + { + return Task.CompletedTask; + } + + if (_httpContext!.RequestAborted.IsCancellationRequested) + { + Fail(EngineTelemetryOutcome.Canceled); + } + else + { + int status = _httpContext.Response.StatusCode; + EngineTelemetryOutcome outcome = _request.Outcome; + if (outcome is EngineTelemetryOutcome.Success or EngineTelemetryOutcome.Unknown) + { + if (status >= StatusCodes.Status400BadRequest) + { + outcome = EngineTelemetryOutcome.Failure; + } + else if (status >= StatusCodes.Status300MultipleChoices || status < StatusCodes.Status200OK) + { + outcome = EngineTelemetryOutcome.Unknown; + } + } + + Complete(outcome, status); + } + + return Task.CompletedTask; + } + + private void Complete(EngineTelemetryOutcome outcome, int? httpStatusCode) + { + if (Interlocked.Exchange(ref _completed, 1) == 0) + { + _abortRegistration.Unregister(); + try + { + _request.Complete(outcome, httpStatusCode); + } + catch (Exception) + { + // Never affect the tool/response or forward product failures to customer telemetry. + } + } + } + } + /// /// Infers the operation type from the tool instance and name. /// For built-in tools, maps tool name directly to operation. diff --git a/src/Config/FileSystemRuntimeConfigLoader.cs b/src/Config/FileSystemRuntimeConfigLoader.cs index e3529c696f..3a32c6e9d0 100644 --- a/src/Config/FileSystemRuntimeConfigLoader.cs +++ b/src/Config/FileSystemRuntimeConfigLoader.cs @@ -8,6 +8,7 @@ using System.Text.Json; using Azure.DataApiBuilder.Config.Converters; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Config.Utilities; using Azure.DataApiBuilder.Service.Exceptions; using Microsoft.Extensions.Logging; @@ -207,6 +208,7 @@ private void OnNewFileContentsDetected(object? sender, EventArgs e) } catch (Exception ex) { + NotifyTelemetryReload(accepted: false); // Need to remove the dependencies in startup on the RuntimeConfigProvider // before we can have an ILogger here. Console.WriteLine("Unable to hot reload configuration file due to " + ex.Message); @@ -267,6 +269,7 @@ public bool TryLoadConfig( replacementSettings ??= new DeserializationVariableReplacementSettings(); string? parseError = null; + using IDisposable? capture = TelemetryConfigurationPresence.BeginCapture(TelemetryCaptureEnabled); if (!string.IsNullOrEmpty(json) && TryParseConfig( json, out RuntimeConfig, @@ -360,6 +363,8 @@ private void HotReloadConfig(bool isDevMode, ILogger? logger = null) IsNewConfigValidated = false; SignalConfigChanged(); + NotifyTelemetryReload(accepted: true); + // Telemetry (and any other) logs buffered during the reload parse are otherwise only // drained once at startup. Flush them now so hot-reload logs are actually emitted and the // shared static buffer does not accumulate entries across successive reloads. @@ -368,6 +373,22 @@ private void HotReloadConfig(bool isDevMode, ILogger? logger = null) logger?.LogInformation("Hot-reload process finished."); } + // Lifecycle observers cannot interrupt loading, expose exception contents or run before + // the existing validation/metadata/schema subscribers finish accepting the replacement. + internal Action? TelemetryReloadCompleted { get; set; } + + private void NotifyTelemetryReload(bool accepted) + { + try + { + TelemetryReloadCompleted?.Invoke(accepted ? RuntimeConfig : null, accepted); + } + catch (Exception) + { + // Product telemetry is never a configuration dependency. + } + } + /// /// Precedence of environments is /// 1) Value of DAB_ENVIRONMENT. diff --git a/src/Config/ObjectModel/RuntimeConfig.cs b/src/Config/ObjectModel/RuntimeConfig.cs index a8b71d10c9..41b271292c 100644 --- a/src/Config/ObjectModel/RuntimeConfig.cs +++ b/src/Config/ObjectModel/RuntimeConfig.cs @@ -7,6 +7,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Azure.DataApiBuilder.Config.Converters; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Service.Exceptions; using Microsoft.Extensions.Logging; @@ -32,6 +33,14 @@ public record RuntimeConfig public DataSourceFiles? DataSourceFiles { get; init; } + /// + /// Bounded, value-free input provenance captured only for opted-in synthetic product telemetry. + /// Preserved by record clones; never serialized or populated from customer configuration. + /// Recapture or clear it when replacing entity definitions programmatically. + /// + [JsonIgnore] + public TelemetryConfigurationPresence? TelemetryPresence { get; init; } + /// /// Indicates whether this config was loaded as a child via another config's data-source-files. /// diff --git a/src/Config/Properties/AssemblyInfo.cs b/src/Config/Properties/AssemblyInfo.cs index a2e622a838..ff886c401d 100644 --- a/src/Config/Properties/AssemblyInfo.cs +++ b/src/Config/Properties/AssemblyInfo.cs @@ -4,3 +4,5 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Azure.DataApiBuilder.Service.Tests")] +[assembly: InternalsVisibleTo("Azure.DataApiBuilder.Service")] +[assembly: InternalsVisibleTo("Microsoft.DataApiBuilder.Core")] diff --git a/src/Config/RuntimeConfigLoader.cs b/src/Config/RuntimeConfigLoader.cs index 1c0c9c9ac8..0c8bca7d09 100644 --- a/src/Config/RuntimeConfigLoader.cs +++ b/src/Config/RuntimeConfigLoader.cs @@ -61,6 +61,8 @@ public void FlushLogBuffer() public bool IsNewConfigValidated; + internal Func? TelemetryCaptureEnabled { get; set; } + public RuntimeConfigLoader(HotReloadEventHandler? handler = null, string? connectionString = null) { _changeToken = new DabChangeToken(); @@ -265,6 +267,17 @@ public static bool TryParseConfig(string json, return false; } + // Capture original presence before any model clone/rewriting. No raw JSON escapes + // capture, and the default-off gate avoids allocating metadata in ordinary loads. + // Child loads run this same path and retain only their own original declarations. + if (TelemetryConfigurationPresence.IsCaptureEnabled()) + { + config = config with + { + TelemetryPresence = TelemetryConfigurationPresence.TryCapture(json, config, enabled: true) + }; + } + // Embed the DAB Application Name (with anonymous usage telemetry) into the connection // string of every MSSQL / DWSQL / PostgreSQL data source. // @@ -419,7 +432,8 @@ public static string GetConnectionStringWithApplicationName(string connectionStr /// /// Connection string for connecting to database. /// When provided, anonymous DAB telemetry is embedded into the `Application Name` - /// (honoring the `DAB_TELEMETRY_APPNAME_OPT_OUT` opt-out). When null, only the plain user agent is used. + /// (honoring both product telemetry opt-outs). When null, only the plain user agent is used, + /// unless the global product telemetry veto is set. /// The data source whose connection is being opened, used to encode per-pool /// fields (Source, OBO). Ignored when is null. /// Updated connection string with `Application Name` property. @@ -446,6 +460,28 @@ internal static string GetMsSqlConnectionStringWithApplicationName(string connec innerException: ex); } + // Check the shared veto before the legacy idempotency guard, including config-null paths + // and connection strings already decorated by an earlier load or embedding host. + if (ProductTelemetryPolicy.IsOptedOut()) + { + string? optedOutApplicationName = ApplicationNameTelemetry.RemoveApplicationNameSegments(connectionStringBuilder.ApplicationName); + if (string.Equals(optedOutApplicationName, connectionStringBuilder.ApplicationName, StringComparison.Ordinal)) + { + return connectionString; + } + + if (string.IsNullOrEmpty(optedOutApplicationName)) + { + connectionStringBuilder.Remove("Application Name"); + } + else + { + connectionStringBuilder.ApplicationName = optedOutApplicationName; + } + + return connectionStringBuilder.ConnectionString; + } + // Idempotency guard: both OSS and hosted telemetry share the dab_ prefix, so do not append a // second telemetry block if either form is already present. if (connectionStringBuilder.ApplicationName?.Contains(ProductInfo.DAB_MARKER_PREFIX, StringComparison.Ordinal) == true) @@ -492,7 +528,9 @@ internal static string GetMsSqlConnectionStringWithApplicationName(string connec /// else add the Application Name property with DataApiBuilder Application Name based on hosted/oss platform. /// /// Connection string for connecting to database. - /// When provided, anonymous DAB usage telemetry is embedded in the Application Name (honoring the opt-out switch); otherwise the plain user agent is used. + /// When provided, anonymous DAB usage telemetry is embedded in the Application Name + /// (honoring both product telemetry opt-outs); otherwise the plain user agent is used, unless + /// the global product telemetry veto is set. /// The data source whose connection is being opened, used to encode per-pool /// fields (Source, OBO). Ignored when is null. /// Updated connection string with `Application Name` property. @@ -519,6 +557,28 @@ internal static string GetPgSqlConnectionStringWithApplicationName(string connec innerException: ex); } + // Check the shared veto before the legacy idempotency guard, including config-null paths + // and connection strings already decorated by an earlier load or embedding host. + if (ProductTelemetryPolicy.IsOptedOut()) + { + string? optedOutApplicationName = ApplicationNameTelemetry.RemoveApplicationNameSegments(connectionStringBuilder.ApplicationName); + if (string.Equals(optedOutApplicationName, connectionStringBuilder.ApplicationName, StringComparison.Ordinal)) + { + return connectionString; + } + + if (string.IsNullOrEmpty(optedOutApplicationName)) + { + connectionStringBuilder.Remove("Application Name"); + } + else + { + connectionStringBuilder.ApplicationName = optedOutApplicationName; + } + + return connectionStringBuilder.ConnectionString; + } + // Idempotency guard: both OSS and hosted telemetry share the dab_ prefix, so do not append a // second telemetry block if either form is already present. if (connectionStringBuilder.ApplicationName?.Contains(ProductInfo.DAB_MARKER_PREFIX, StringComparison.Ordinal) == true) diff --git a/src/Config/Telemetry/ApplicationNameTelemetry.cs b/src/Config/Telemetry/ApplicationNameTelemetry.cs index d0b3d3bb48..2fbff5827c 100644 --- a/src/Config/Telemetry/ApplicationNameTelemetry.cs +++ b/src/Config/Telemetry/ApplicationNameTelemetry.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System.Text; +using System.Text.RegularExpressions; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Product; @@ -62,6 +63,16 @@ public static class ApplicationNameTelemetry private const char SECTION_SEPARATOR = '|'; private const char PAYLOAD_DELIMITER = '+'; + // ProductInfo emits a three-component numeric version, without a prerelease/commit suffix. + // Require a complete payload with the four positional sections and the known minimum widths + // (context 4, runtime 20, entity 14), rather than treating any dab_ substring as telemetry. + // Allow additive flags and the reserved general section to be populated. + // NonBacktracking keeps recognition linear even for an unusually long customer Application Name. + private static readonly Regex _versionAndPayloadPattern = new( + @"\A(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)" + + @"(?:\+[RGMX][TVSPX][SDPMCX][NACX][A-Z0-9?]*\|[A-Z0-9?]*\|[A-Z0-9?]{20,}\|[A-Z0-9?]{14,}\+)?\z", + RegexOptions.CultureInvariant | RegexOptions.NonBacktracking); + /// Inputs available to a setting encoder. private readonly record struct EncodeInputs(RuntimeConfig Config, DataSource? LiveDataSource); @@ -71,7 +82,7 @@ private sealed record Setting(string Name, Func Encode, Func /// /// Produces the pure telemetry string (<marker><version>+<context>||<runtime>|<entity>+), /// where the marker is dab_oss_ for open source or dab_hosted_ when DAB_APP_NAME_ENV - /// is set. Independent of the opt-out switch. Used by the CLI and as the telemetry-bearing portion of + /// is set. Independent of both opt-out switches. Used by the CLI and as the telemetry-bearing portion of /// the connection-string segment. The empty section after context reserves the general-settings /// position for future use. /// @@ -108,7 +119,9 @@ public static string EncodeTelemetryString(RuntimeConfig config, DataSource? liv /// /// Open source uses the dab_oss_ marker; the hosted scenario (DAB_APP_NAME_ENV set) /// uses dab_hosted_ instead — the dab_oss_ marker is not present in that case. - /// When opted out, only <marker><version> is returned (no payload). + /// The global DAB_TELEMETRY_OPT_OUT veto returns an empty segment. + /// With only the legacy DAB_TELEMETRY_APPNAME_OPT_OUT opt-out, only + /// <marker><version> is returned (no payload). /// Telemetry is always based on the product version, so it is never suppressed by /// DAB_APP_NAME_ENV. /// @@ -117,13 +130,84 @@ public static string EncodeTelemetryString(RuntimeConfig config, DataSource? liv /// The data source whose connection is being opened. public static string BuildApplicationNameSegment(RuntimeConfig config, DataSource? liveDataSource) { + if (ProductTelemetryPolicy.IsOptedOut()) + { + return string.Empty; + } + // The marker itself reflects the hosting scenario (dab_oss_ vs dab_hosted_), so no separate - // label prefix is needed. When opted out, only the marker+version is emitted (no payload). + // label prefix is needed. The legacy opt-out emits only the marker+version (no payload). return IsOptedOut() ? ProductInfo.GetTelemetryApplicationNameBase() : EncodeTelemetryString(config, liveDataSource); } + /// + /// Removes complete trailing DAB Application Name segments, including their marker/version, + /// without trimming or rewriting the retained prefix. DAB appends with a comma; an OBO pipe + /// prefix is retained verbatim because it is not part of this injection's segment. + /// + /// + /// Recognizes OSS/hosted markers across product versions and a versioned current + /// DAB_APP_NAME_ENV label. The reserved, unversioned hosted marker is also recognized for + /// the config-null fallback. Unknown layouts (including shorter payload sections), truncated + /// blocks and opaque custom legacy host labels are retained: their ownership cannot be + /// established from the string alone. A custom label from + /// an earlier environment likewise requires the original undecorated connection string. + /// Customer text identical to a complete recognized terminal segment is indistinguishable from + /// telemetry and is treated as telemetry. Replaced empty/default provider names cannot be + /// reconstructed; callers restore the provider default when no prefix remains. + /// + internal static string? RemoveApplicationNameSegments(string? applicationName) + { + if (string.IsNullOrEmpty(applicationName)) + { + return applicationName; + } + + string? hostLabel = Environment.GetEnvironmentVariable(ProductInfo.DAB_APP_NAME_ENV); + string? hostMarker = string.IsNullOrWhiteSpace(hostLabel) + ? null + : hostLabel.EndsWith("_", StringComparison.Ordinal) ? hostLabel : hostLabel + "_"; + + int retainedLength = applicationName.Length; + bool segmentRemoved; + do + { + segmentRemoved = false; + // Prefer the longest recognized suffix: a configured host label can itself contain + // commas/pipes or a standard marker, and its whole label belongs to the DAB segment. + for (int start = 0; start < retainedLength; start++) + { + if (start != 0 && applicationName[start - 1] is not (',' or SECTION_SEPARATOR)) + { + continue; + } + + ReadOnlySpan segment = applicationName.AsSpan(start, retainedLength - start); + if (IsVersionedSegment(segment, ProductInfo.DAB_USER_AGENT_MARKER) + || IsVersionedSegment(segment, HOSTED_USER_AGENT_MARKER) + || (hostMarker is not null && IsVersionedSegment(segment, hostMarker)) + || segment.SequenceEqual("dab_hosted".AsSpan()) + || segment.SequenceEqual(HOSTED_USER_AGENT_MARKER.AsSpan())) + { + // Remove only the comma introduced when appending. Preserve any preexisting + // commas, whitespace or OBO prefix, even when another DAB segment precedes this one. + retainedLength = start > 0 && applicationName[start - 1] == ',' ? start - 1 : start; + segmentRemoved = true; + break; + } + } + } + while (segmentRemoved); + + return retainedLength == applicationName.Length ? applicationName : applicationName[..retainedLength]; + } + + private static bool IsVersionedSegment(ReadOnlySpan segment, string marker) => + segment.StartsWith(marker.AsSpan(), StringComparison.Ordinal) + && _versionAndPayloadPattern.IsMatch(segment[marker.Length..]); + /// /// Decodes a telemetry-bearing Application Name into human-readable lines. The input may be a /// raw telemetry string or a full Application Name with a user prefix and/or OBO hash. Decoding diff --git a/src/Config/Telemetry/ProductTelemetryPolicy.cs b/src/Config/Telemetry/ProductTelemetryPolicy.cs new file mode 100644 index 0000000000..d569bcf1bd --- /dev/null +++ b/src/Config/Telemetry/ProductTelemetryPolicy.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Azure.DataApiBuilder.Config.Telemetry; + +/// +/// Shared startup switches for DAB-owned product telemetry. Production collection remains off; +/// these switches do not change customer-configured logging, tracing, metrics or destinations. +/// +public static class ProductTelemetryPolicy +{ + /// Environment variable that overrides all DAB product telemetry enablement. + public const string OPT_OUT_ENV_VAR = "DAB_TELEMETRY_OPT_OUT"; + + /// Explicit validation-only collection; never a production enablement switch. + public const string TEST_MODE_ENV_VAR = "DAB_PRODUCT_TELEMETRY_TEST_MODE"; + + /// + /// Interprets an opt-out value without reading the environment. Only trimmed 1 and + /// true (case-insensitive) opt out; missing and unrecognized values do not enable or veto anything. + /// + public static bool IsOptedOut(string? value) => IsTrue(value); + + /// Pure startup policy shared by configuration capture and standalone hosting. + public static bool IsSyntheticCollectionEnabled(string? testMode, string? optOut) + => IsTrue(testMode) && !IsOptedOut(optOut); + + private static bool IsTrue(string? value) + { + string? normalizedValue = value?.Trim(); + return string.Equals(normalizedValue, "1", StringComparison.Ordinal) + || string.Equals(normalizedValue, "true", StringComparison.OrdinalIgnoreCase); + } + + /// Reads the process environment's product telemetry opt-out. + public static bool IsOptedOut() => IsOptedOut(Environment.GetEnvironmentVariable(OPT_OUT_ENV_VAR)); +} diff --git a/src/Config/Telemetry/TelemetryConfigurationPresence.cs b/src/Config/Telemetry/TelemetryConfigurationPresence.cs new file mode 100644 index 0000000000..7fbd77bc5f --- /dev/null +++ b/src/Config/Telemetry/TelemetryConfigurationPresence.cs @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Immutable; +using System.Text.Json; +using Azure.DataApiBuilder.Config.ObjectModel; + +namespace Azure.DataApiBuilder.Config.Telemetry; + +/// +/// Value-free provenance for the original input of one configuration file. The retained shape +/// is fixed: 26 built-in setting paths and five entity presence summaries. No JSON, configuration +/// reference, customer property/entity name, string value, or per-entity collection is retained. +/// Child files keep their own provenance; their merged entities are not counted again here. +/// +public sealed class TelemetryConfigurationPresence +{ + private static readonly AsyncLocal?> _capturePermission = new(); + public const string TEST_MODE_ENV_VAR = ProductTelemetryPolicy.TEST_MODE_ENV_VAR; + public const int SETTING_COUNT = 26; + public const int ENTITY_FEATURE_COUNT = 5; + + public enum Presence { Unavailable, Missing, ExplicitNull, Present, Indeterminate } + + public enum EntityFeature { Cache, Rest, GraphQL, McpDml, McpCustomTool } + + /// + /// Counts structural presence only, never enabled/disabled values. Fixed-width counters keep + /// retained space constant regardless of the number or length of customer entity names. + /// Custom-tool counts cover stored procedures only, matching that feature's applicability. + /// + public readonly record struct EntityPresence(long Missing, long ExplicitNull, long Present, long Indeterminate) + { + public long Total => Missing + ExplicitNull + Present + Indeterminate; + + public EntityPresence Combine(EntityPresence other) => new( + Missing + other.Missing, ExplicitNull + other.ExplicitNull, + Present + other.Present, Indeterminate + other.Indeterminate); + + internal EntityPresence Include(Presence presence) => presence switch + { + Presence.Missing => this with { Missing = Missing + 1 }, + Presence.ExplicitNull => this with { ExplicitNull = ExplicitNull + 1 }, + Presence.Present => this with { Present = Present + 1 }, + _ => this with { Indeterminate = Indeterminate + 1 } + }; + } + + public ImmutableDictionary Settings { get; } + + public ImmutableDictionary Entities { get; } + + private TelemetryConfigurationPresence( + ImmutableDictionary settings, + ImmutableDictionary entities) + { + Settings = settings; + Entities = entities; + } + + /// Pure policy evaluation; an explicit synthetic opt-in is required and the veto wins. + public static bool IsCaptureEnabled(string? testMode, string? productOptOut) + { + return ProductTelemetryPolicy.IsSyntheticCollectionEnabled(testMode, productOptOut); + } + + // Parsing is shared with CLI and embedded hosts. Ambient test mode alone is not a host's + // permission to collect; its fully evaluated session policy must authorize this load. + public static bool IsCaptureEnabled() => _capturePermission.Value?.Invoke() == true && !ProductTelemetryPolicy.IsOptedOut(); + + internal static IDisposable? BeginCapture(Func? permission) + { + if (permission is null) + { + return null; // Nested child loaders inherit the root's evaluated permission. + } + + CaptureScope scope = new(_capturePermission.Value); + _capturePermission.Value = permission; + return scope; + } + + private sealed class CaptureScope(Func? previous) : IDisposable + { + public void Dispose() => _capturePermission.Value = previous; + } + + /// + /// Pure, default-off capture after successful product deserialization. The caller supplies + /// the matching accepted model only to identify original entities and custom-tool applicability; + /// none of its values or references are retained. Never pass a reserialized/defaulted config. + /// The enabled argument is the caller's evaluated policy; capture never reads the environment. + /// Disabled capture does not parse JSON or allocate metadata. Bad JSON returns no provenance + /// and does not replace the loader's validation or error handling. + /// + public static TelemetryConfigurationPresence? TryCapture(string json, RuntimeConfig config, bool enabled = false) + { + if (!enabled) + { + return null; + } + + ArgumentNullException.ThrowIfNull(config); + try + { + // Match the loader's comment/depth/trailing-comma policy. Dispose the only raw document + // before returning; even unrecognized properties never become retained metadata keys. + using JsonDocument document = JsonDocument.Parse(json, new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip }); + if (document.RootElement.ValueKind != JsonValueKind.Object) + { + return null; + } + + Cursor root = new(Presence.Present, document.RootElement); + Cursor runtime = root.Child("runtime"); + Cursor telemetry = runtime.Child("telemetry"); + Cursor source = root.Child("data-source"); + ImmutableDictionary.Builder settings = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + settings.Add("runtime.rest.enabled", runtime.Child("rest").Enablement().State); + settings.Add("runtime.graphql.enabled", runtime.Child("graphql").Enablement().State); + settings.Add("runtime.mcp.enabled", runtime.Child("mcp").Enablement().State); + settings.Add("runtime.health.enabled", runtime.Child("health").Child("enabled").State); + settings.Add("runtime.cache.enabled", runtime.Child("cache").Child("enabled").State); + settings.Add("runtime.cache.level-2.enabled", runtime.Child("cache").Child("level-2").Child("enabled").State); + settings.Add("runtime.rest.request-body-strict", runtime.Child("rest").Child("request-body-strict").State); + settings.Add("runtime.graphql.multiple-mutations.create.enabled", runtime.Child("graphql").Child("multiple-mutations").Child("create").Child("enabled").State); + settings.Add("azure-key-vault", root.Child("azure-key-vault").State); + settings.Add("autoentities", root.Child("autoentities").State); + settings.Add("data-source-files", root.Child("data-source-files").State); + settings.Add("runtime.embeddings.enabled", runtime.Child("embeddings").Child("enabled", ignoreCase: true).State); + settings.Add("runtime.embeddings.endpoint.enabled", runtime.Child("embeddings").Child("endpoint", ignoreCase: true).Child("enabled", ignoreCase: true).State); + settings.Add("runtime.telemetry.open-telemetry.enabled", telemetry.Child("open-telemetry").Child("enabled").State); + settings.Add("runtime.telemetry.application-insights.enabled", telemetry.Child("application-insights").Child("enabled").State); + settings.Add("runtime.telemetry.azure-log-analytics.enabled", telemetry.Child("azure-log-analytics").Child("enabled").State); + settings.Add("runtime.telemetry.file.enabled", telemetry.Child("file").Child("enabled").State); + settings.Add("runtime.host.authentication.provider", runtime.Child("host").Child("authentication").Child("provider").State); + settings.Add("runtime.host.mode", runtime.Child("host").Child("mode").State); + settings.Add("data-source.user-delegated-auth.enabled", source.Child("user-delegated-auth").Child("enabled").State); + settings.Add("data-source.options.set-session-context", source.Child("options").Child("set-session-context").State); + settings.Add("runtime.pagination.default-page-size", runtime.Child("pagination").Child("default-page-size").State); + settings.Add("runtime.pagination.max-page-size", runtime.Child("pagination").Child("max-page-size").State); + settings.Add("runtime.host.max-response-size-mb", runtime.Child("host").Child("max-response-size-mb").State); + settings.Add("runtime.cache.ttl-seconds", runtime.Child("cache").Child("ttl-seconds").State); + settings.Add("runtime.mcp.dml-tools.aggregate-records.query-timeout", runtime.Child("mcp").Child("dml-tools").Child("aggregate-records", ignoreCase: true).Child("query-timeout", ignoreCase: true).State); + + ImmutableDictionary.Builder entities = ImmutableDictionary.CreateBuilder(); + foreach (EntityFeature feature in Enum.GetValues()) + { + entities.Add(feature, default); + } + + Cursor originalEntities = root.Child("entities"); + if (originalEntities.IsObject) + { + foreach ((string name, Entity entity) in config.Entities) + { + // Look up the last JSON property, as the product dictionary deserializer does. + // Iterate the accepted dictionary so duplicate JSON names cannot inflate counts. + // An absent definition can belong to a child or be generated; it proves no omission. + Cursor original = originalEntities.Child(name); + if (!original.IsObject) + { + continue; + } + + entities[EntityFeature.Cache] = entities[EntityFeature.Cache].Include(original.Child("cache").Child("enabled").State); + entities[EntityFeature.Rest] = entities[EntityFeature.Rest].Include(original.Child("rest").Enablement(allowString: true).State); + entities[EntityFeature.GraphQL] = entities[EntityFeature.GraphQL].Include(original.Child("graphql").Enablement(allowString: true).State); + entities[EntityFeature.McpDml] = entities[EntityFeature.McpDml].Include(original.Child("mcp").ShorthandOrChild("dml-tools").State); + if (entity.Source.Type == EntitySourceType.StoredProcedure) + { + entities[EntityFeature.McpCustomTool] = entities[EntityFeature.McpCustomTool].Include(original.Child("mcp").Child("custom-tool").State); + } + } + } + + return new(settings.ToImmutable(), entities.ToImmutable()); + } + catch (JsonException) + { + return null; + } + } + + /// A stack-local reader; no cursor escapes capture or enters the retained object. + private readonly struct Cursor + { + private readonly JsonElement _value; + public Presence State { get; } + public bool IsObject => State == Presence.Present && _value.ValueKind == JsonValueKind.Object; + + public Cursor(Presence state, JsonElement value = default) + { + State = state; + _value = value; + } + + public Cursor Child(string property, bool ignoreCase = false) + { + if (State != Presence.Present) + { + return this; + } + + if (_value.ValueKind is JsonValueKind.True or JsonValueKind.False or JsonValueKind.String) + { + return new(Presence.Missing); + } + + if (!IsObject) + { + return new(Presence.Indeterminate); + } + + if (!ignoreCase) + { + return _value.TryGetProperty(property, out JsonElement child) ? FromValue(child) : new(Presence.Missing); + } + + Cursor found = new(Presence.Missing); + foreach (JsonProperty candidate in _value.EnumerateObject()) + { + if (string.Equals(candidate.Name, property, StringComparison.OrdinalIgnoreCase)) + { + found = FromValue(candidate.Value); + } + } + + return found; + } + + private static Cursor FromValue(JsonElement value) => new( + value.ValueKind == JsonValueKind.Null ? Presence.ExplicitNull : Presence.Present, value); + + public Cursor Enablement(bool allowString = false) => State == Presence.Present + && (_value.ValueKind is JsonValueKind.True or JsonValueKind.False || (allowString && _value.ValueKind == JsonValueKind.String)) + ? this : Child("enabled"); + + public Cursor ShorthandOrChild(string property) => State == Presence.Present + && _value.ValueKind is JsonValueKind.True or JsonValueKind.False ? this : Child(property); + } +} diff --git a/src/Core/Azure.DataApiBuilder.Core.csproj b/src/Core/Azure.DataApiBuilder.Core.csproj index 0082d2e9b2..859c441f72 100644 --- a/src/Core/Azure.DataApiBuilder.Core.csproj +++ b/src/Core/Azure.DataApiBuilder.Core.csproj @@ -25,6 +25,11 @@ $(TargetsForTfmSpecificContentInPackage);IncludeInternalDependenciesInPackage + + + + + diff --git a/src/Core/Configurations/RuntimeConfigProvider.cs b/src/Core/Configurations/RuntimeConfigProvider.cs index c38f666d5b..38af21a309 100644 --- a/src/Core/Configurations/RuntimeConfigProvider.cs +++ b/src/Core/Configurations/RuntimeConfigProvider.cs @@ -8,6 +8,8 @@ using Azure.DataApiBuilder.Config; using Azure.DataApiBuilder.Config.NamingPolicies; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Telemetry; +using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Service.Exceptions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Primitives; @@ -31,6 +33,17 @@ public class RuntimeConfigProvider : IDisposable public List RuntimeConfigLoadedHandlers { get; } = new List(); + // One engine-owned instance; never a process-global or customer telemetry provider. + internal EngineTelemetrySession? ProductTelemetry + { + get; + set + { + field = value; + _configLoader.TelemetryCaptureEnabled = () => value?.IsEnabled == true; + } + } + /// /// Indicates whether the config was loaded after the runtime was initialized. /// @@ -182,11 +195,17 @@ public bool TryGetLoadedConfig([NotNullWhen(true)] out RuntimeConfig? runtimeCon /// The GraphQL Schema. Can be left null for SQL configurations. /// The string representation of a managed identity access token /// true if the initialization succeeded, false otherwise. - public async Task Initialize( + public Task Initialize( string configuration, string? schema, string? accessToken) + => ProductTelemetry?.IsEnabled == true + ? ObserveInitializationAsync(() => InitializeCoreAsync(configuration, schema, accessToken)) + : InitializeCoreAsync(configuration, schema, accessToken); + + private async Task InitializeCoreAsync(string configuration, string? schema, string? accessToken) { + using IDisposable? capture = TelemetryConfigurationPresence.BeginCapture(_configLoader.TelemetryCaptureEnabled); if (string.IsNullOrEmpty(configuration)) { throw new ArgumentException($"'{nameof(configuration)}' cannot be null or empty.", nameof(configuration)); @@ -267,13 +286,20 @@ public bool TrySetAccesstoken( /// The connection string to the database. /// The string representation of a managed identity access token /// true if the initialization succeeded, false otherwise. - public async Task Initialize( + public Task Initialize( string jsonConfig, string? graphQLSchema, string connectionString, string? accessToken, DeserializationVariableReplacementSettings? replacementSettings) + => ProductTelemetry?.IsEnabled == true + ? ObserveInitializationAsync(() => InitializeCoreAsync(jsonConfig, graphQLSchema, connectionString, accessToken, replacementSettings)) + : InitializeCoreAsync(jsonConfig, graphQLSchema, connectionString, accessToken, replacementSettings); + + private async Task InitializeCoreAsync(string jsonConfig, string? graphQLSchema, string connectionString, + string? accessToken, DeserializationVariableReplacementSettings? replacementSettings) { + using IDisposable? capture = TelemetryConfigurationPresence.BeginCapture(_configLoader.TelemetryCaptureEnabled); if (string.IsNullOrEmpty(connectionString)) { throw new ArgumentException($"'{nameof(connectionString)}' cannot be null or empty.", nameof(connectionString)); @@ -326,6 +352,33 @@ public async Task Initialize( return false; } + private async Task ObserveInitializationAsync(Func> initialize) + { + bool accepted = false; + try + { + bool initialized = await initialize(); + // The existing V2 API can report true with no parsed model (no handlers ran). + // Match the controller's acceptance condition without changing that public API. + if (initialized && TryGetLoadedConfig(out RuntimeConfig? acceptedConfig)) + { + // Publish telemetry only after every loaded-config handler has accepted it. + // Startup's own success is not sufficient while another handler is pending. + accepted = true; + ProductTelemetry?.AcceptConfiguration(acceptedConfig, "late_configuration", ConfigFilePath); + } + + return initialized; + } + finally + { + if (!accepted) + { + ProductTelemetry?.ConfigurationChangeFailed(); + } + } + } + /// /// Embeds anonymous usage telemetry into the Application Name of each data source's /// connection string. Hosted / late-config initialization parses with env-var replacement disabled, diff --git a/src/Core/Resolvers/CosmosClientProvider.cs b/src/Core/Resolvers/CosmosClientProvider.cs index 82e2b5e9c7..10b9604d6a 100644 --- a/src/Core/Resolvers/CosmosClientProvider.cs +++ b/src/Core/Resolvers/CosmosClientProvider.cs @@ -5,6 +5,7 @@ using System.IdentityModel.Tokens.Jwt; using Azure.Core; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Product; using Azure.Identity; @@ -63,19 +64,24 @@ private void InitializeClient(RuntimeConfig? configuration) if (!Clients.ContainsKey(dataSourceName)) { CosmosClient client; - string userAgent = ProductInfo.GetDataApiBuilderUserAgent(); - CosmosClientOptions options = new() + string? userAgent = ProductTelemetryPolicy.IsOptedOut() ? null : ProductInfo.GetDataApiBuilderUserAgent(); + CosmosClientOptions options = new(); + if (userAgent is not null) { - ApplicationName = userAgent - }; + options.ApplicationName = userAgent; + } (string? accountEndPoint, string? accountKey) = ParseCosmosConnectionString(dataSource.ConnectionString); if (!string.IsNullOrEmpty(accountKey)) { - client = new CosmosClientBuilder(dataSource.ConnectionString).WithContentResponseOnWrite(true) - .WithApplicationName(userAgent) - .Build(); + CosmosClientBuilder builder = new CosmosClientBuilder(dataSource.ConnectionString).WithContentResponseOnWrite(true); + if (userAgent is not null) + { + builder.WithApplicationName(userAgent); + } + + client = builder.Build(); } else if (!_accessToken.ContainsKey(dataSourceName)) { diff --git a/src/Core/Resolvers/QueryExecutor.cs b/src/Core/Resolvers/QueryExecutor.cs index 98917ed2c9..ebd608821f 100644 --- a/src/Core/Resolvers/QueryExecutor.cs +++ b/src/Core/Resolvers/QueryExecutor.cs @@ -9,8 +9,10 @@ using System.Text.Json; using System.Text.Json.Nodes; using Azure.DataApiBuilder.Config; +using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Models; +using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Service.Exceptions; using Microsoft.AspNetCore.Http; using Microsoft.Data.SqlTypes; @@ -292,21 +294,33 @@ public virtual TConnection CreateConnection(string dataSourceName) await conn.OpenAsync(); DbCommand cmd = PrepareDbCommand(conn, sqltext, parameters, httpContext, dataSourceName); TResult? result = default(TResult); + CommandBehavior commandBehavior = ConfigProvider.GetConfig().MaxResponseSizeLogicEnabled() ? CommandBehavior.SequentialAccess : CommandBehavior.CloseConnection; + // CancellationToken is passed to ExecuteReaderAsync to ensure that if the client times out while the query is executing, the execution will be cancelled and resources will be freed up. + CancellationToken cancellationToken = httpContext?.RequestAborted ?? CancellationToken.None; + + // Start only at provider execution, not connection setup or the retry-policy boundary. + // Each execution gets its own scope, including executions in a retry attempt. + using EngineTelemetryMeasurementScope? databaseAttempt = BeginDatabaseAttempt(dataSourceName); try { - CommandBehavior commandBehavior = ConfigProvider.GetConfig().MaxResponseSizeLogicEnabled() ? CommandBehavior.SequentialAccess : CommandBehavior.CloseConnection; - // CancellationToken is passed to ExecuteReaderAsync to ensure that if the client times out while the query is executing, the execution will be cancelled and resources will be freed up. - CancellationToken cancellationToken = httpContext?.RequestAborted ?? CancellationToken.None; - using DbDataReader dbDataReader = await cmd.ExecuteReaderAsync(commandBehavior, cancellationToken); - - if (dataReaderHandler is not null && dbDataReader is not null) - { - result = await dataReaderHandler(dbDataReader, args); - } - else + using (DbDataReader dbDataReader = await cmd.ExecuteReaderAsync(commandBehavior, cancellationToken)) { - result = default(TResult); + if (dataReaderHandler is not null && dbDataReader is not null) + { + result = await dataReaderHandler(dbDataReader, args); + } + else + { + result = default(TResult); + } } + + databaseAttempt?.Complete(EngineTelemetryOutcome.Success); + } + catch (OperationCanceledException) + { + databaseAttempt?.Complete(EngineTelemetryOutcome.Canceled); + throw; } catch (DbException e) { @@ -384,18 +398,30 @@ public virtual DbCommand PrepareDbCommand( conn.Open(); DbCommand cmd = PrepareDbCommand(conn, sqltext, parameters, httpContext, dataSourceName); + CommandBehavior commandBehavior = ConfigProvider.GetConfig().MaxResponseSizeLogicEnabled() ? CommandBehavior.SequentialAccess : CommandBehavior.CloseConnection; + using EngineTelemetryMeasurementScope? databaseAttempt = BeginDatabaseAttempt(dataSourceName); try { - using DbDataReader dbDataReader = ConfigProvider.GetConfig().MaxResponseSizeLogicEnabled() ? - cmd.ExecuteReader(CommandBehavior.SequentialAccess) : cmd.ExecuteReader(CommandBehavior.CloseConnection); - if (dataReaderHandler is not null && dbDataReader is not null) - { - return dataReaderHandler(dbDataReader, args); - } - else + TResult? result; + using (DbDataReader dbDataReader = cmd.ExecuteReader(commandBehavior)) { - return default(TResult); + if (dataReaderHandler is not null && dbDataReader is not null) + { + result = dataReaderHandler(dbDataReader, args); + } + else + { + result = default(TResult); + } } + + databaseAttempt?.Complete(EngineTelemetryOutcome.Success); + return result; + } + catch (OperationCanceledException) + { + databaseAttempt?.Complete(EngineTelemetryOutcome.Canceled); + throw; } catch (DbException e) { @@ -415,6 +441,43 @@ public virtual DbCommand PrepareDbCommand( } } + /// + /// Resolves only the provider category, preserving the request's captured source IDs + /// across reload. The session excludes work without an eligible request. + /// + private EngineTelemetryMeasurementScope? BeginDatabaseAttempt(string dataSourceName) + { + EngineTelemetrySession? session = ConfigProvider.ProductTelemetry; + RuntimeConfig? config = session?.CurrentRequest?.Config; + if (session?.IsEnabled != true || config is null) + { + return null; + } + + try + { + string resolvedDataSourceName = string.IsNullOrEmpty(dataSourceName) ? config.DefaultDataSourceName : dataSourceName; + // A later executor selection may use the replacement model, while an already + // prepared command still uses the captured one. Match the actual source ID. + if (!config.CheckDataSourceExists(resolvedDataSourceName) && + ConfigProvider.TryGetLoadedConfig(out RuntimeConfig? currentConfig)) + { + config = currentConfig; + } + + DatabaseType? provider = config.CheckDataSourceExists(resolvedDataSourceName) + ? config.GetDataSourceFromDataSourceName(resolvedDataSourceName).DatabaseType + : null; + return session.BeginDatabaseAttempt(provider); + } + catch (DataApiBuilderException) + { + // Multiple replacements can remove both models' attribution. Preserve the + // observed attempt as unknown rather than discard it or guess its provider. + return session.BeginDatabaseAttempt(null); + } + } + /// public virtual string GetSessionParamsQuery(HttpContext? httpContext, IDictionary parameters, string dataSourceName = "") { diff --git a/src/Core/Services/Embeddings/EmbeddingService.cs b/src/Core/Services/Embeddings/EmbeddingService.cs index b4bc4dd25d..ce3a5d7259 100644 --- a/src/Core/Services/Embeddings/EmbeddingService.cs +++ b/src/Core/Services/Embeddings/EmbeddingService.cs @@ -8,6 +8,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Azure.DataApiBuilder.Config.ObjectModel.Embeddings; +using Azure.DataApiBuilder.Core.Telemetry.Product; using Microsoft.Extensions.Logging; using ZiggyCreatures.Caching.Fusion; @@ -27,6 +28,12 @@ public class EmbeddingService : IEmbeddingService private readonly IFusionCache _cache; private readonly string _providerName; + /// + /// Optional product telemetry assigned by the host without changing the public constructor. + /// Only top-level service calls create measurements; internal calls use the core methods. + /// + internal EngineTelemetrySession? ProductTelemetry { get; set; } + // Constants private const char KEY_DELIMITER = ':'; private const string CACHE_KEY_PREFIX = "embedding"; @@ -114,11 +121,28 @@ private void ConfigureHttpClient() /// public async Task TryEmbedAsync(string text, CancellationToken cancellationToken = default) + { + using EngineTelemetryMeasurementScope? measurement = ProductTelemetry?.BeginEmbedding(); + try + { + (EmbeddingResult result, EngineTelemetryOutcome outcome) = await TryEmbedCoreAsync(text, cancellationToken); + measurement?.Complete(outcome); + return result; + } + catch (OperationCanceledException) + { + measurement?.Complete(EngineTelemetryOutcome.Canceled); + throw; + } + } + + private async Task<(EmbeddingResult Result, EngineTelemetryOutcome Outcome)> TryEmbedCoreAsync( + string text, CancellationToken cancellationToken) { EmbeddingResult? validationResult = ValidateTryEmbedRequest(text); if (validationResult != null) { - return validationResult; + return (validationResult, EngineTelemetryOutcome.Failure); } Stopwatch stopwatch = Stopwatch.StartNew(); @@ -145,7 +169,7 @@ public async Task TryEmbedAsync(string text, CancellationToken EmbeddingTelemetryHelper.TrackCacheMiss(_providerName); } - return new EmbeddingResult(true, embedding); + return (new EmbeddingResult(true, embedding), EngineTelemetryOutcome.Success); } catch (Exception ex) { @@ -154,17 +178,37 @@ public async Task TryEmbedAsync(string text, CancellationToken activity?.SetEmbeddingActivityError(ex); EmbeddingTelemetryHelper.TrackError(_providerName, ex.GetType().Name); - return new EmbeddingResult(false, null, "Failed to generate embedding."); + EngineTelemetryOutcome outcome = ex is OperationCanceledException + ? EngineTelemetryOutcome.Canceled + : EngineTelemetryOutcome.Failure; + return (new EmbeddingResult(false, null, "Failed to generate embedding."), outcome); } } /// public async Task TryEmbedBatchAsync(string[] texts, CancellationToken cancellationToken = default) + { + using EngineTelemetryMeasurementScope? measurement = ProductTelemetry?.BeginEmbedding(); + try + { + (EmbeddingBatchResult result, EngineTelemetryOutcome outcome) = await TryEmbedBatchCoreAsync(texts, cancellationToken); + measurement?.Complete(outcome); + return result; + } + catch (OperationCanceledException) + { + measurement?.Complete(EngineTelemetryOutcome.Canceled); + throw; + } + } + + private async Task<(EmbeddingBatchResult Result, EngineTelemetryOutcome Outcome)> TryEmbedBatchCoreAsync( + string[] texts, CancellationToken cancellationToken) { EmbeddingBatchResult? validationResult = ValidateTryEmbedBatchRequest(texts); if (validationResult != null) { - return validationResult; + return (validationResult, EngineTelemetryOutcome.Failure); } Stopwatch stopwatch = Stopwatch.StartNew(); @@ -175,7 +219,7 @@ public async Task TryEmbedBatchAsync(string[] texts, Cance { EmbeddingTelemetryHelper.TrackEmbeddingRequest(_providerName, texts.Length); - float[][] embeddings = await EmbedBatchAsync(texts, cancellationToken); + float[][] embeddings = await EmbedBatchCoreAsync(texts, cancellationToken); stopwatch.Stop(); int dimensions = embeddings.Length > 0 ? embeddings[0].Length : 0; @@ -186,7 +230,7 @@ public async Task TryEmbedBatchAsync(string[] texts, Cance EmbeddingTelemetryHelper.TrackDimensions(_providerName, dimensions); } - return new EmbeddingBatchResult(true, embeddings); + return (new EmbeddingBatchResult(true, embeddings), EngineTelemetryOutcome.Success); } catch (Exception ex) { @@ -195,12 +239,31 @@ public async Task TryEmbedBatchAsync(string[] texts, Cance activity?.SetEmbeddingActivityError(ex); EmbeddingTelemetryHelper.TrackError(_providerName, ex.GetType().Name); - return new EmbeddingBatchResult(false, null, "Failed to generate embeddings."); + EngineTelemetryOutcome outcome = ex is OperationCanceledException + ? EngineTelemetryOutcome.Canceled + : EngineTelemetryOutcome.Failure; + return (new EmbeddingBatchResult(false, null, "Failed to generate embeddings."), outcome); } } /// public async Task EmbedAsync(string text, CancellationToken cancellationToken = default) + { + using EngineTelemetryMeasurementScope? measurement = ProductTelemetry?.BeginEmbedding(); + try + { + float[] embedding = await EmbedCoreAsync(text, cancellationToken); + measurement?.Complete(EngineTelemetryOutcome.Success); + return embedding; + } + catch (OperationCanceledException) + { + measurement?.Complete(EngineTelemetryOutcome.Canceled); + throw; + } + } + + private async Task EmbedCoreAsync(string text, CancellationToken cancellationToken) { ValidateEmbedRequest(text); @@ -322,6 +385,22 @@ private void ValidateEmbedBatchRequest(string[] texts) /// public async Task EmbedBatchAsync(string[] texts, CancellationToken cancellationToken = default) + { + using EngineTelemetryMeasurementScope? measurement = ProductTelemetry?.BeginEmbedding(); + try + { + float[][] embeddings = await EmbedBatchCoreAsync(texts, cancellationToken); + measurement?.Complete(EngineTelemetryOutcome.Success); + return embeddings; + } + catch (OperationCanceledException) + { + measurement?.Complete(EngineTelemetryOutcome.Canceled); + throw; + } + } + + private async Task EmbedBatchCoreAsync(string[] texts, CancellationToken cancellationToken) { ValidateEmbedBatchRequest(texts); diff --git a/src/Core/Services/ExecutionHelper.cs b/src/Core/Services/ExecutionHelper.cs index 5ab002b4d6..51f0b1b093 100644 --- a/src/Core/Services/ExecutionHelper.cs +++ b/src/Core/Services/ExecutionHelper.cs @@ -13,6 +13,7 @@ using Azure.DataApiBuilder.Core.Resolvers; using Azure.DataApiBuilder.Core.Resolvers.Factories; using Azure.DataApiBuilder.Core.Telemetry; +using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Service.Exceptions; using Azure.DataApiBuilder.Service.GraphQLBuilder; using Azure.DataApiBuilder.Service.GraphQLBuilder.CustomScalars; @@ -55,6 +56,21 @@ public ExecutionHelper( /// The middleware context. /// public async ValueTask ExecuteQueryAsync(IMiddlewareContext context) + { + using EngineTelemetryMeasurementScope? operation = BeginTelemetryOperation(context, EngineTelemetryOperation.Read); + try + { + await ExecuteQueryCoreAsync(context); + operation?.Complete(GetTelemetryOutcome(context)); + } + catch (OperationCanceledException) + { + operation?.Complete(EngineTelemetryOutcome.Canceled); + throw; + } + } + + private async ValueTask ExecuteQueryCoreAsync(IMiddlewareContext context) { using Activity? activity = StartQueryActivity(context); @@ -101,6 +117,21 @@ public async ValueTask ExecuteQueryAsync(IMiddlewareContext context) /// The middleware context. /// public async ValueTask ExecuteMutateAsync(IMiddlewareContext context) + { + using EngineTelemetryMeasurementScope? operation = BeginTelemetryOperation(context, EngineTelemetryOperation.Write); + try + { + await ExecuteMutateCoreAsync(context); + operation?.Complete(GetTelemetryOutcome(context)); + } + catch (OperationCanceledException) + { + operation?.Complete(EngineTelemetryOutcome.Canceled); + throw; + } + } + + private async ValueTask ExecuteMutateCoreAsync(IMiddlewareContext context) { using Activity? activity = StartQueryActivity(context); @@ -143,6 +174,56 @@ public async ValueTask ExecuteMutateAsync(IMiddlewareContext context) } } + private EngineTelemetryMeasurementScope? BeginTelemetryOperation(IMiddlewareContext context, EngineTelemetryOperation operation) + { + EngineTelemetrySession? session = _runtimeConfigProvider.ProductTelemetry; + if (session?.IsEnabled != true) + { + return null; + } + + // Introspection is not a data operation, even in a request that also selects data. + if (context.Selection.Field.Name.StartsWith("__", StringComparison.Ordinal)) + { + return null; + } + + string? entityName = null; + try + { + // Use the same model directive/pagination mapping as query execution, not the + // exposed field name or alias. Names are used locally, never emitted or retained here. + entityName = GraphQLUtils.GetEntityNameFromContext(context); + if (_runtimeConfigProvider.TryGetLoadedConfig(out RuntimeConfig? config) && + config.Entities.TryGetValue(entityName, out Entity? entity) && + entity?.Source?.Type is EntitySourceType.StoredProcedure) + { + operation = EngineTelemetryOperation.Execute; + } + } + catch (Exception) + { + // An unmappable field still runs through the original execution/validation path. + // Telemetry must not introduce a new failure or expose mapping exception details. + } + + return session.BeginOperation(entityName, operation); + } + + private static EngineTelemetryOutcome GetTelemetryOutcome(IMiddlewareContext context) + { + // This is the root action outcome only. Errors during subsequent child-field + // completion/serialization are classified by the request-level GraphQL adapter. + if (!context.HasErrors) + { + return EngineTelemetryOutcome.Success; + } + + return context.Result is null or JsonElement { ValueKind: JsonValueKind.Null or JsonValueKind.Undefined } + ? EngineTelemetryOutcome.Failure + : EngineTelemetryOutcome.PartialFailure; + } + /// /// Starts the activity for the query /// diff --git a/src/Core/Services/RestService.cs b/src/Core/Services/RestService.cs index 5014d942bb..12c354f071 100644 --- a/src/Core/Services/RestService.cs +++ b/src/Core/Services/RestService.cs @@ -14,10 +14,12 @@ using Azure.DataApiBuilder.Core.Resolvers; using Azure.DataApiBuilder.Core.Resolvers.Factories; using Azure.DataApiBuilder.Core.Services.MetadataProviders; +using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Service.Exceptions; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Infrastructure; namespace Azure.DataApiBuilder.Core.Services { @@ -64,6 +66,27 @@ RequestValidator requestValidator string entityName, EntityActionOperation operationType, string? primaryKeyRoute) + { + // Begin before validation so rejected data operations are not lost. The session + // suppresses this scope when a surrounding MCP operation already owns the call. + using EngineTelemetryMeasurementScope? operation = BeginTelemetryOperation(entityName, operationType); + try + { + IActionResult? result = await ExecuteCoreAsync(entityName, operationType, primaryKeyRoute); + operation?.Complete(GetTelemetryOutcome(result)); + return result; + } + catch (OperationCanceledException) + { + operation?.Complete(EngineTelemetryOutcome.Canceled); + throw; + } + } + + private async Task ExecuteCoreAsync( + string entityName, + EntityActionOperation operationType, + string? primaryKeyRoute) { _requestValidator.ValidateEntity(entityName); string dataSourceName = _runtimeConfigProvider.GetConfig().GetDataSourceNameFromEntityName(entityName); @@ -223,6 +246,67 @@ RequestValidator requestValidator } } + private EngineTelemetryMeasurementScope? BeginTelemetryOperation(string entityName, EntityActionOperation operationType) + { + EngineTelemetrySession? session = _runtimeConfigProvider.ProductTelemetry; + if (session?.IsEnabled != true) + { + return null; + } + + EngineTelemetryOperation operation = operationType switch + { + EntityActionOperation.Read => EngineTelemetryOperation.Read, + EntityActionOperation.Execute => EngineTelemetryOperation.Execute, + EntityActionOperation.Create or EntityActionOperation.Insert or EntityActionOperation.Delete or + EntityActionOperation.Update or EntityActionOperation.UpdateGraphQL or EntityActionOperation.Patch or + EntityActionOperation.UpdateIncremental or EntityActionOperation.Upsert or EntityActionOperation.UpsertIncremental + => EngineTelemetryOperation.Write, + _ => EngineTelemetryOperation.Unknown + }; + + // Stored procedures execute regardless of the HTTP verb used to expose them. + // Do not load configuration or inspect request content solely for telemetry. + if (!string.IsNullOrEmpty(entityName) && + _runtimeConfigProvider.TryGetLoadedConfig(out RuntimeConfig? config) && + config.Entities.TryGetValue(entityName, out Entity? entity) && + entity?.Source?.Type is EntitySourceType.StoredProcedure) + { + operation = EngineTelemetryOperation.Execute; + } + + return session.BeginOperation(entityName, operation); + } + + private static EngineTelemetryOutcome GetTelemetryOutcome(IActionResult? result) + { + // RestController turns a null result into a 404, not a successful empty response. + if (result is null) + { + return EngineTelemetryOutcome.Failure; + } + + int? statusCode = (result as IStatusCodeActionResult)?.StatusCode; + if (statusCode is null) + { + statusCode = result switch + { + ObjectResult { Value: ProblemDetails problem } => problem.Status ?? StatusCodes.Status500InternalServerError, + ObjectResult or JsonResult or ContentResult or EmptyResult => StatusCodes.Status200OK, + ForbidResult => StatusCodes.Status403Forbidden, + ChallengeResult => StatusCodes.Status401Unauthorized, + _ => null + }; + } + + return statusCode switch + { + >= StatusCodes.Status200OK and < StatusCodes.Status300MultipleChoices => EngineTelemetryOutcome.Success, + >= StatusCodes.Status400BadRequest and < 600 => EngineTelemetryOutcome.Failure, + _ => EngineTelemetryOutcome.Unknown + }; + } + /// /// Dispatch execution of a request context to the query engine /// The two overloads to ExecuteAsync take FindRequestContext and StoredProcedureRequestContext diff --git a/src/Core/Telemetry/Product/EngineTelemetryAggregator.cs b/src/Core/Telemetry/Product/EngineTelemetryAggregator.cs new file mode 100644 index 0000000000..7503f73a5e --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryAggregator.cs @@ -0,0 +1,364 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Immutable; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// Bounded, memory-only aggregation isolated from customer diagnostics. No sender, identity + /// storage, timers, or process-global telemetry providers are created here. All mutation and + /// window sealing share one short critical section; no external callbacks run inside it, + /// except the injected local clock. Disabled instances do not allocate aggregation state. + /// + internal sealed class EngineTelemetryAggregator + { + private static readonly TimeSpan _windowLength = TimeSpan.FromHours(6); + private readonly object _sync = new(); + private State? _state; + + private EngineTelemetryAggregator(State? state) + { + _state = state; + } + + public bool IsEnabled + { + get + { + lock (_sync) + { + return _state is not null; + } + } + } + + /// + /// Default-off even in Release builds. The only enabling path is explicit internal + /// synthetic validation. Read the umbrella veto before clocks, counters, or identity work. + /// Environment changes do not change an already-created instance; use Disable instead. + /// + public static EngineTelemetryAggregator Create( + EngineTelemetryOptions? options = null, + TimeProvider? timeProvider = null, + Func? readEnvironmentVariable = null) + { + if (options?.EnableSyntheticCollection != true) + { + return new(null); + } + + readEnvironmentVariable ??= Environment.GetEnvironmentVariable; + if (EngineTelemetryOptions.IsOptedOut(readEnvironmentVariable(EngineTelemetryOptions.OPT_OUT_ENVIRONMENT_VARIABLE))) + { + return new(null); + } + + options.Validate(); + return new(new State(options, timeProvider ?? TimeProvider.System)); + } + + /// + /// Call only after accepting a configuration, never on a rejected reload. This does not + /// imply serving readiness, emit a snapshot, flush counters, or relabel existing work. + /// + public EngineTelemetryConfiguration AcceptConfiguration() + { + lock (_sync) + { + if (_state is null || _state.Epoch == long.MaxValue) + { + return default; + } + + return new(_state.Owner, ++_state.Epoch); + } + } + + public void RecordRequest(EngineTelemetryConfiguration configuration, EngineTelemetryApi api, EngineTelemetryTransport transport, + EngineTelemetryRole role, EngineTelemetryOutcome outcome, TimeSpan? duration) + => Record(configuration, EngineTelemetryDimensions.ForRequest(api, transport, role), outcome, duration); + + public void RecordOperation(EngineTelemetryConfiguration configuration, EngineTelemetryApi api, EngineTelemetryOperation operation, + EngineTelemetryProvider provider, EngineTelemetryObject objectType, EngineTelemetryOutcome outcome) + => Record(configuration, EngineTelemetryDimensions.ForOperation(api, operation, provider, objectType), outcome); + + public void RecordDatabaseAttempt(EngineTelemetryConfiguration configuration, EngineTelemetryProvider provider, EngineTelemetryOutcome outcome) + => Record(configuration, EngineTelemetryDimensions.ForDatabaseAttempt(provider), outcome); + + public void RecordCacheLookup(EngineTelemetryConfiguration configuration, EngineTelemetryCacheLayer layer, EngineTelemetryCacheResult result) + => Record(configuration, EngineTelemetryDimensions.ForCacheLookup(layer, result), EngineTelemetryOutcome.Unknown); + + public void RecordEmbedding(EngineTelemetryConfiguration configuration, EngineTelemetryApi api, EngineTelemetryOutcome outcome) + => Record(configuration, EngineTelemetryDimensions.ForEmbedding(api), outcome); + + public void RecordHttpOutcome(EngineTelemetryConfiguration configuration, EngineTelemetryApi api, int? status) + => Record(configuration, EngineTelemetryDimensions.ForHttpOutcome(api, status), EngineTelemetryOutcome.Unknown); + + /// Detach completed windows once; the still-open window is not flushed. + public EngineTelemetryDrain DrainCompletedWindows() + { + lock (_sync) + { + if (_state is null) + { + return EngineTelemetryDrain.Empty; + } + + AdvanceWindow(_state, _state.Clock.GetUtcNow().ToUniversalTime()); + return Drain(_state); + } + } + + /// + /// Seal the remaining delta once and stop recording. The caller, not request processing, + /// owns any later bounded shutdown delivery. This method never performs network I/O. + /// + public EngineTelemetryDrain Complete() + { + lock (_sync) + { + if (_state is null) + { + return EngineTelemetryDrain.Empty; + } + + State state = _state; + DateTimeOffset now = state.Clock.GetUtcNow().ToUniversalTime(); + AdvanceWindow(state, now); + SealWindow(state, now > state.LastObservation ? now : state.LastObservation, isFinal: true); + _state = null; + return Drain(state); + } + } + + /// Stop recording and discard all owned data, without producing a final delta. + public void Disable() + { + lock (_sync) + { + _state = null; + } + } + + private void Record(EngineTelemetryConfiguration configuration, EngineTelemetryDimensions dimensions, EngineTelemetryOutcome outcome, TimeSpan? duration = null) + { + lock (_sync) + { + State? state = _state; + if (state is null || !ReferenceEquals(configuration.Owner, state.Owner) || configuration.Epoch <= 0 || configuration.Epoch > state.Epoch) + { + return; + } + + DateTimeOffset now = state.Clock.GetUtcNow().ToUniversalTime(); + if (now < state.LastObservation) + { + // Never reopen a sealed calendar segment or silently move old-time work into + // a later segment after a wall-clock correction. Report the local loss. + AddLoss(ref state.ClockRegressionDrops, 1, ref state.WindowLossCountsCapped); + return; + } + + AdvanceWindow(state, now); + state.LastObservation = now; + SeriesKey key = new(configuration.Epoch, dimensions); + if (!state.Series.TryGetValue(key, out Counters? counters)) + { + if (state.Series.Count == state.Options.SeriesCapacity) + { + AddLoss(ref state.SeriesCapacityDrops, 1, ref state.WindowLossCountsCapped); + return; + } + + counters = new Counters(dimensions.Measurement); + state.Series.Add(key, counters); + } + + if (!counters.TryRecord(EngineTelemetryDimensions.Normalize(outcome), duration, state.Options.CounterCeiling)) + { + AddLoss(ref state.CounterCapacityDrops, 1, ref state.WindowLossCountsCapped); + } + } + } + + private static void AdvanceWindow(State state, DateTimeOffset now) + { + if (now < state.WindowEnd) + { + return; + } + + SealWindow(state, state.WindowEnd, isFinal: false); + // Skip empty elapsed windows in O(1), even after a long suspension or clock jump. + state.WindowStart = FloorToWindow(now); + state.WindowEnd = state.WindowStart + _windowLength; + state.LastObservation = state.WindowStart; + } + + private static DateTimeOffset FloorToWindow(DateTimeOffset now) + => new(now.UtcTicks - (now.UtcTicks % _windowLength.Ticks), TimeSpan.Zero); + + private static void SealWindow(State state, DateTimeOffset end, bool isFinal) + { + if (state.Series.Count == 0 && state.SeriesCapacityDrops == 0 && state.CounterCapacityDrops == 0 && state.ClockRegressionDrops == 0) + { + return; + } + + if (state.Pending.Count < state.Options.PendingWindowCapacity) + { + ImmutableArray.Builder series = ImmutableArray.CreateBuilder(state.Series.Count); + foreach ((SeriesKey key, Counters counters) in state.Series) + { + series.Add(counters.Snapshot(key)); + } + + state.Pending.Enqueue(new(state.WindowStart, end, isFinal, series.MoveToImmutable(), + state.SeriesCapacityDrops, state.CounterCapacityDrops, state.ClockRegressionDrops, state.WindowLossCountsCapped)); + } + else + { + // Do not duplicate a delta later to compensate for loss. The delivery layer can + // retry the same immutable result, but collection always starts a new segment. + AddLoss(ref state.DroppedWindows, 1, ref state.DrainLossCountsCapped); + foreach (Counters counters in state.Series.Values) + { + AddLoss(ref state.DroppedMeasurements, counters.Count, ref state.DrainLossCountsCapped); + } + + // Preserve measurable losses even when their window cannot be queued. These + // observations were not included in any of the recorded series counts above. + AddLoss(ref state.DroppedMeasurements, state.SeriesCapacityDrops, ref state.DrainLossCountsCapped); + AddLoss(ref state.DroppedMeasurements, state.CounterCapacityDrops, ref state.DrainLossCountsCapped); + AddLoss(ref state.DroppedMeasurements, state.ClockRegressionDrops, ref state.DrainLossCountsCapped); + state.DrainLossCountsCapped |= state.WindowLossCountsCapped; + } + + state.Series.Clear(); + state.SeriesCapacityDrops = 0; + state.CounterCapacityDrops = 0; + state.ClockRegressionDrops = 0; + state.WindowLossCountsCapped = false; + } + + private static EngineTelemetryDrain Drain(State state) + { + if (state.Pending.Count == 0 && state.DroppedWindows == 0) + { + return EngineTelemetryDrain.Empty; + } + + EngineTelemetryDrain result = new(state.Pending.ToImmutableArray(), state.DroppedWindows, state.DroppedMeasurements, state.DrainLossCountsCapped); + state.Pending.Clear(); + state.DroppedWindows = 0; + state.DroppedMeasurements = 0; + state.DrainLossCountsCapped = false; + return result; + } + + private static void AddLoss(ref long count, long increment, ref bool capped) + { + if (increment > long.MaxValue - count) + { + count = long.MaxValue; + capped = true; + } + else + { + count += increment; + } + } + + private readonly record struct SeriesKey(long Epoch, EngineTelemetryDimensions Dimensions); + + private sealed class State + { + public State(EngineTelemetryOptions options, TimeProvider clock) + { + Options = options; + Clock = clock; + WindowStart = clock.GetUtcNow().ToUniversalTime(); + LastObservation = WindowStart; + WindowEnd = FloorToWindow(WindowStart) + _windowLength; + } + + public EngineTelemetryOptions Options { get; } + public TimeProvider Clock { get; } + public object Owner { get; } = new(); + public Dictionary Series { get; } = new(); + public Queue Pending { get; } = new(); + public long Epoch { get; set; } + public DateTimeOffset WindowStart { get; set; } + public DateTimeOffset WindowEnd { get; set; } + public DateTimeOffset LastObservation { get; set; } + public long SeriesCapacityDrops; + public long CounterCapacityDrops; + public long ClockRegressionDrops; + public bool WindowLossCountsCapped; + public long DroppedWindows; + public long DroppedMeasurements; + public bool DrainLossCountsCapped; + } + + private sealed class Counters + { + private readonly long[]? _outcomes; + private readonly long[]? _latency; + private long _timedCount; + private bool _capped; + + public Counters(EngineTelemetryMeasurement measurement) + { + if (measurement is not (EngineTelemetryMeasurement.CacheLookup or EngineTelemetryMeasurement.HttpOutcome)) + { + _outcomes = new long[5]; + } + + if (measurement == EngineTelemetryMeasurement.Request) + { + _latency = new long[EngineTelemetryHistogram.UpperBoundsMilliseconds.Length + 1]; + } + } + + public long Count { get; private set; } + + public bool TryRecord(EngineTelemetryOutcome outcome, TimeSpan? duration, long ceiling) + { + if (Count == ceiling) + { + _capped = true; + return false; + } + + Count++; + if (_outcomes is not null) + { + _outcomes[(int)outcome]++; + } + + if (_latency is not null && duration.HasValue && duration.Value >= TimeSpan.Zero) + { + int bucket = 0; + while (bucket < EngineTelemetryHistogram.UpperBoundsMilliseconds.Length + && duration.Value.Ticks > EngineTelemetryHistogram.UpperBoundsMilliseconds[bucket] * TimeSpan.TicksPerMillisecond) + { + bucket++; + } + + _latency[bucket]++; + _timedCount++; + } + + return true; + } + + public EngineTelemetrySeries Snapshot(SeriesKey key) + { + EngineTelemetryOutcomeCounts? outcomes = _outcomes is null ? null : new(_outcomes[0], _outcomes[1], _outcomes[2], _outcomes[3], _outcomes[4]); + EngineTelemetryHistogram? latency = _latency is null ? null : new(_latency.ToImmutableArray(), _timedCount, _timedCount == Count && !_capped); + return new(key.Epoch, key.Dimensions, Count, outcomes, latency, _capped); + } + } + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryCacheObserver.cs b/src/Core/Telemetry/Product/EngineTelemetryCacheObserver.cs new file mode 100644 index 0000000000..e700aed079 --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryCacheObserver.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using ZiggyCreatures.Caching.Fusion; +using ZiggyCreatures.Caching.Fusion.Events; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// Observes actual lookups independently at each FusionCache layer. The owner attaches one + /// observer to its data cache and disposes it before releasing the engine session. + /// + /// + /// FusionCache may dispatch events asynchronously. Recording uses only the eligible request + /// context flowing into the callback; no context is captured at subscription time and no + /// request is reconstructed from cache keys. The session skips callbacks without an active + /// eligible request. Hit/Miss arguments do not identify background work that inherits such a + /// context, so its suppression belongs to the session/owner. Coverage remains unknown when + /// context is missing or expired; asynchronous delivery is not proof of complete cache + /// coverage. Reporting lost attribution requires a separate session completeness contract. + /// + internal sealed class EngineTelemetryCacheObserver : IDisposable + { + private readonly EngineTelemetrySession _session; + private readonly FusionCacheEventsHub? _events; + private int _disposed; + + public EngineTelemetryCacheObserver(IFusionCache cache, EngineTelemetrySession session) + { + ArgumentNullException.ThrowIfNull(cache); + ArgumentNullException.ThrowIfNull(session); + _session = session; + + if (!session.IsEnabled) + { + return; + } + + _events = cache.Events; + _events.Memory.Hit += OnMemoryHit; + _events.Memory.Miss += OnMemoryMiss; + _events.Distributed.Hit += OnDistributedHit; + _events.Distributed.Miss += OnDistributedMiss; + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0 || _events is null) + { + return; + } + + _events.Memory.Hit -= OnMemoryHit; + _events.Memory.Miss -= OnMemoryMiss; + _events.Distributed.Hit -= OnDistributedHit; + _events.Distributed.Miss -= OnDistributedMiss; + } + + // Deliberately ignore event arguments: keys can contain SQL, parameters and other + // customer data. Do not subscribe to the aggregate Hit/Miss events as well as these. + private void OnMemoryHit(object? sender, FusionCacheEntryHitEventArgs args) + => RecordLookup(EngineTelemetryCacheLayer.Level1, EngineTelemetryCacheResult.Hit); + + private void OnMemoryMiss(object? sender, FusionCacheEntryEventArgs args) + => RecordLookup(EngineTelemetryCacheLayer.Level1, EngineTelemetryCacheResult.Miss); + + private void OnDistributedHit(object? sender, FusionCacheEntryHitEventArgs args) + => RecordLookup(EngineTelemetryCacheLayer.Level2, EngineTelemetryCacheResult.Hit); + + private void OnDistributedMiss(object? sender, FusionCacheEntryEventArgs args) + => RecordLookup(EngineTelemetryCacheLayer.Level2, EngineTelemetryCacheResult.Miss); + + private void RecordLookup(EngineTelemetryCacheLayer layer, EngineTelemetryCacheResult result) + { + if (Volatile.Read(ref _disposed) != 0 || !_session.IsEnabled) + { + return; + } + + try + { + _session.RecordCacheLookup(layer, result); + } + catch (Exception) + { + // Never let observer failures affect caching or enter FusionCache diagnostics, + // which may include the cache key. No request context is fabricated on failure. + } + } + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryConfigurationSnapshot.cs b/src/Core/Telemetry/Product/EngineTelemetryConfigurationSnapshot.cs new file mode 100644 index 0000000000..e331772150 --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryConfigurationSnapshot.cs @@ -0,0 +1,751 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Immutable; +using System.Text.Json; +using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.ObjectModel.Embeddings; +using Azure.DataApiBuilder.Config.Telemetry; +using static Azure.DataApiBuilder.Config.Telemetry.TelemetryConfigurationPresence; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// Projects an accepted configuration into a fixed, immutable categorical schema. This does + /// not validate or accept configurations, initialize telemetry, or perform any external I/O. + /// Effective values describe configuration/runtime gates, not readiness, successful service + /// registration, permission to a particular caller, or observed use. + /// + internal static class EngineTelemetrySnapshotFactory + { + private const string ENABLED = "enabled"; + private const string DISABLED = "disabled"; + private const string MISSING = "missing"; + private const string UNSUPPORTED = "unsupported"; + private const string UNKNOWN = "unknown"; + private const string NOT_APPLICABLE = "not_applicable"; + + // Order is schema-defined, never derived from customer names, enum numeric values, or + // dictionary enumeration order. Even combinations of these labels have bounded size. + private static readonly ImmutableArray _databaseTypes = + ["mssql", "dwsql", "postgresql", "mysql", "cosmosdb_nosql", "cosmosdb_postgresql", UNKNOWN]; + + public static ImmutableDictionary Create(RuntimeConfig config) + => Create(config, explicitConfiguration: null); + + /// + /// Optional provenance must be the original input corresponding to this accepted config, + /// NOT RuntimeConfig.ToJson(): serializers write defaults back as explicit values. The + /// caller owns the live document for this call only; no JSON, config references, names, + /// or customer strings are retained. Without that transient input, use the safe presence + /// metadata captured by the loader. Root JSON alone cannot establish omission in merged + /// child or generated entities. Their lost provenance remains unknown. + /// + public static ImmutableDictionary Create(RuntimeConfig config, JsonElement? explicitConfiguration) + { + ArgumentNullException.ThrowIfNull(config); + ImmutableDictionary.Builder result = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + Input input = Input.FromRoot(explicitConfiguration, config.TelemetryPresence); + Input runtimeInput = input.Child("runtime"); + RuntimeOptions? runtime = config.Runtime; + RuntimeCacheOptions? cache = runtime?.Cache; + EmbeddingsOptions? embeddings = runtime?.Embeddings; + bool canUseCache = config.CanUseCache(); + + result.Add("snapshot_schema", "configuration-v1"); + Add(result, "runtime.rest", + Configured(runtimeInput.Child("rest").Enablement(), State(runtime?.Rest?.Enabled), UNKNOWN), + State(config.IsRestEnabled)); + Add(result, "runtime.graphql", + Configured(runtimeInput.Child("graphql").Enablement(), State(runtime?.GraphQL?.Enabled), UNKNOWN), + State(config.IsGraphQLEnabled)); + Add(result, "runtime.mcp", + Configured(runtimeInput.Child("mcp").Enablement(), State(runtime?.Mcp?.Enabled), UNKNOWN), + State(config.IsMcpEnabled)); + Add(result, "runtime.health", + Configured(runtimeInput.Child("health").Child("enabled"), State(runtime?.Health?.Enabled), + Provided(runtime?.Health?.Enabled, runtime?.Health?.UserProvidedEnabled == true)), + State(config.IsHealthEnabled)); + Add(result, "runtime.cache", + Configured(runtimeInput.Child("cache").Child("enabled"), State(cache?.Enabled), NullableSetting(cache?.Enabled)), + State(canUseCache)); + + // Startup installs L2 only under the runtime cache gate; query engines additionally + // require CanUseCache(), including its session-context rule. No Redis probe is made. + Add(result, "runtime.cache.l2", + Configured(runtimeInput.Child("cache").Child("level-2").Child("enabled"), + State(cache?.Level2?.Enabled), NullableSetting(cache?.Level2?.Enabled)), + State(canUseCache && cache?.Level2?.Enabled == true)); + Add(result, "runtime.rest.strict_body", + Configured(runtimeInput.Child("rest").Child("request-body-strict"), State(runtime?.Rest?.RequestBodyStrict), UNKNOWN), + config.IsRestEnabled ? State(config.IsRequestBodyStrict) : NOT_APPLICABLE); + + MultipleCreateOptions? multipleCreate = runtime?.GraphQL?.MultipleMutationOptions?.MultipleCreateOptions; + Input multipleCreateInput = runtimeInput.Child("graphql").Child("multiple-mutations").Child("create").Child("enabled"); + Add(result, "runtime.graphql.multiple_create", + Configured(multipleCreateInput, State(multipleCreate?.Enabled), multipleCreate is null ? UNKNOWN : State(multipleCreate.Enabled)), + State(config.IsGraphQLEnabled && config.IsMultipleCreateOperationEnabled())); + + AddKeyVault(result, config, input); + bool hasAutoentities = config.Autoentities.Any(); + // RuntimeConfig replaces an omitted autoentities collection with an empty one. + // Nonempty definitions survive merging, but an empty model alone proves no omission. + Add(result, "integrations.autoentities", + hasAutoentities ? ENABLED : Configured(input.Child("autoentities"), DISABLED, UNKNOWN), + State(hasAutoentities)); + bool hasSourceFiles = config.DataSourceFiles?.SourceFiles?.Any() == true; + Add(result, "integrations.multiple_source_files", + Configured(input.Child("data-source-files"), State(hasSourceFiles), config.DataSourceFiles is null ? UNKNOWN : State(hasSourceFiles)), + State(hasSourceFiles)); + + Input embeddingsInput = runtimeInput.Child("embeddings"); + Add(result, "runtime.embeddings", + Configured(embeddingsInput.Child("enabled", ignoreCase: true), State(embeddings?.Enabled), + Provided(embeddings?.Enabled, embeddings?.UserProvidedEnabled == true)), + State(embeddings?.Enabled == true)); + Add(result, "runtime.embeddings.endpoint", + Configured(embeddingsInput.Child("endpoint", ignoreCase: true).Child("enabled", ignoreCase: true), + State(embeddings?.Endpoint?.Enabled), Provided(embeddings?.Endpoint?.Enabled, embeddings?.Endpoint?.UserProvidedEnabled == true)), + State(embeddings?.Enabled == true && embeddings.IsEndpointEnabled)); + result.Add("runtime.embeddings.endpoint_present", State(embeddings?.Endpoint is not null)); + + AddCustomerTelemetry(result, runtime?.Telemetry, runtimeInput.Child("telemetry")); + string authentication = AuthenticationCategory(config); + Add(result, "authentication.provider", + Configured(runtimeInput.Child("host").Child("authentication").Child("provider"), authentication, UNKNOWN), authentication); + string hostMode = runtime?.Host?.Mode switch + { + null or HostMode.Production => "production", + HostMode.Development => "development", + _ => UNKNOWN + }; + Add(result, "host.mode", Configured(runtimeInput.Child("host").Child("mode"), hostMode, UNKNOWN), hostMode); + + AddDataSources(result, config, input); + AddEntities(result, config, input, canUseCache); + AddLimits(result, config, runtimeInput); + return result.ToImmutable(); + } + + private static void AddCustomerTelemetry(ImmutableDictionary.Builder result, TelemetryOptions? telemetry, Input input) + { + OpenTelemetryOptions? otel = telemetry?.OpenTelemetry; + ApplicationInsightsOptions? insights = telemetry?.ApplicationInsights; + AzureLogAnalyticsOptions? analytics = telemetry?.AzureLogAnalytics; + FileSinkOptions? file = telemetry?.File; + Add(result, "customer_telemetry.open_telemetry", + Configured(input.Child("open-telemetry").Child("enabled"), State(otel?.Enabled), UNKNOWN), + State(otel?.Enabled == true && Uri.TryCreate(otel.Endpoint, UriKind.Absolute, out _))); + Add(result, "customer_telemetry.application_insights", + Configured(input.Child("application-insights").Child("enabled"), State(insights?.Enabled), UNKNOWN), + State(insights?.Enabled == true)); + + // These are Startup's local registration/configuration predicates, not exporter health. + bool analyticsAvailable = analytics?.Enabled == true && analytics.Auth is not null + && !string.IsNullOrWhiteSpace(analytics.Auth.CustomTableName) + && !string.IsNullOrWhiteSpace(analytics.Auth.DcrImmutableId) + && !string.IsNullOrWhiteSpace(analytics.Auth.DceEndpoint); + Add(result, "customer_telemetry.log_analytics", + Configured(input.Child("azure-log-analytics").Child("enabled"), State(analytics?.Enabled), + Provided(analytics?.Enabled, analytics?.UserProvidedEnabled == true)), + State(analyticsAvailable)); + Add(result, "customer_telemetry.file", + Configured(input.Child("file").Child("enabled"), State(file?.Enabled), Provided(file?.Enabled, file?.UserProvidedEnabled == true)), + State(file?.Enabled == true && !string.IsNullOrWhiteSpace(file.Path))); + } + + private static void AddKeyVault(ImmutableDictionary.Builder result, RuntimeConfig config, Input input) + { + string configured = NOT_APPLICABLE; + bool hasEndpoint = false; + HashSet visited = new(ReferenceEqualityComparer.Instance); + Stack pending = new(); + pending.Push(config); + while (pending.TryPop(out RuntimeConfig? current)) + { + if (!visited.Add(current)) + { + continue; + } + + bool currentHasEndpoint = !string.IsNullOrEmpty(current.AzureKeyVault?.Endpoint); + hasEndpoint |= currentHasEndpoint; + string fallback = current.AzureKeyVault is null ? UNKNOWN : State(currentHasEndpoint); + Input currentInput = ReferenceEquals(current, config) ? input : Input.FromPresence(current.TelemetryPresence); + string state = Configured(currentInput.Child("azure-key-vault"), State(currentHasEndpoint), fallback); + configured = Any(configured, state); + foreach ((_, RuntimeConfig child) in current.ChildConfigs) + { + pending.Push(child); + } + } + + // DoReplaceAkvVar belongs to deserialization settings, not RuntimeConfig. An endpoint + // alone cannot prove that variable replacement was enabled (or that AKV was used). + Add(result, "integrations.key_vault", configured, hasEndpoint ? UNKNOWN : DISABLED); + } + + private static void AddDataSources(ImmutableDictionary.Builder result, RuntimeConfig config, Input input) + { + int typeMask = 0; + long count = 0; + string oboConfigured = NOT_APPLICABLE; + string oboEffective = NOT_APPLICABLE; + string sessionConfigured = NOT_APPLICABLE; + string sessionEffective = NOT_APPLICABLE; + foreach ((string name, DataSource source) in config.GetDataSourceNamesToDataSourcesIterator()) + { + count++; + int type = source.DatabaseType switch + { + DatabaseType.MSSQL => 0, + DatabaseType.DWSQL => 1, + DatabaseType.PostgreSQL => 2, + DatabaseType.MySQL => 3, + DatabaseType.CosmosDB_NoSQL => 4, + DatabaseType.CosmosDB_PostgreSQL => 5, + _ => 6 + }; + typeMask |= 1 << type; + Input sourceInput = string.Equals(name, config.DefaultDataSourceName, StringComparison.Ordinal) + ? input.Child("data-source") : ChildSourceInput(config, name).Child("data-source"); + + // OBO validation permits MSSQL only. Session context is read by the MSSQL/DWSQL + // executor, using GetTypedOptions (whose actual omitted value is false). + if (source.DatabaseType == DatabaseType.MSSQL) + { + oboConfigured = Any(oboConfigured, Configured(sourceInput.Child("user-delegated-auth").Child("enabled"), State(source.IsUserDelegatedAuthEnabled), UNKNOWN)); + oboEffective = Any(oboEffective, State(source.IsUserDelegatedAuthEnabled)); + } + else + { + string unsupported = type == 6 ? UNKNOWN : UNSUPPORTED; + oboConfigured = Any(oboConfigured, unsupported); + oboEffective = Any(oboEffective, unsupported); + } + + if (source.DatabaseType is DatabaseType.MSSQL or DatabaseType.DWSQL) + { + string setting = source.Options is null ? UNKNOWN : MISSING; + if (source.Options?.TryGetValue("set-session-context", out object? value) == true) + { + setting = value is bool enabled ? State(enabled) : UNKNOWN; + } + + sessionConfigured = Any(sessionConfigured, Configured(sourceInput.Child("options").Child("set-session-context"), setting, setting)); + sessionEffective = Any(sessionEffective, State(source.GetTypedOptions()?.SetSessionContext == true)); + } + else + { + string unsupported = type == 6 ? UNKNOWN : UNSUPPORTED; + sessionConfigured = Any(sessionConfigured, unsupported); + sessionEffective = Any(sessionEffective, unsupported); + } + } + + Add(result, "data_sources.obo", oboConfigured, oboEffective); + Add(result, "data_sources.session_context", sessionConfigured, sessionEffective); + result.Add("scale.data_source_count", CountBucket(count)); + result.Add("data_sources.types", typeMask == 0 ? "none" : string.Join(",", _databaseTypes.Where((_, index) => (typeMask & (1 << index)) != 0))); + int distinctTypes = _databaseTypes.Where((_, index) => (typeMask & (1 << index)) != 0).Count(); + result.Add("data_sources.distinct_type_count", (typeMask & (1 << 6)) != 0 ? UNKNOWN : distinctTypes switch + { + 0 => "0", + 1 => "1", + 2 => "2", + 3 => "3", + 4 => "4", + 5 => "5", + 6 => "6", + _ => UNKNOWN + }); + } + + private static void AddEntities(ImmutableDictionary.Builder result, RuntimeConfig config, Input input, bool canUseCache) + { + // A fixed descriptor list prevents entity/role/property names becoming output keys. + (string Key, EntityFeature Feature, Func Configured, Func Effective)[] features = + [ + ("entities.any.cache", EntityFeature.Cache, + (_, entity, raw) => Configured(raw.Child("cache").Child("enabled"), State(entity.Cache?.Enabled), Provided(entity.Cache?.Enabled, entity.Cache?.UserProvidedEnabledOptions == true)), + (name, _) => State(canUseCache && config.IsEntityCachingEnabled(name))), + ("entities.any.rest", EntityFeature.Rest, + (_, entity, raw) => Configured(raw.Child("rest").Enablement(allowString: true), State(entity.IsRestEnabled), UNKNOWN), + (_, entity) => State(config.IsRestEnabled && entity.IsRestEnabled)), + ("entities.any.graphql", EntityFeature.GraphQL, + (_, entity, raw) => Configured(raw.Child("graphql").Enablement(allowString: true), State(entity.IsGraphQLEnabled), UNKNOWN), + (_, entity) => State(config.IsGraphQLEnabled && entity.IsGraphQLEnabled)), + ("entities.any.mcp_dml", EntityFeature.McpDml, + (_, entity, raw) => Configured(raw.Child("mcp").ShorthandOrChild("dml-tools"), State(entity.Mcp?.DmlToolEnabled ?? true), Provided(entity.Mcp?.DmlToolEnabled, entity.Mcp?.UserProvidedDmlToolsEnabled == true)), + (_, entity) => McpDmlEffective(config, entity)), + ("entities.any.mcp_custom_tool", EntityFeature.McpCustomTool, + (_, entity, raw) => CustomToolApplicability(entity) ?? Configured(raw.Child("mcp").Child("custom-tool"), State(entity.Mcp?.CustomToolEnabled ?? false), Provided(entity.Mcp?.CustomToolEnabled, entity.Mcp?.UserProvidedCustomToolEnabled == true)), + (_, entity) => CustomToolApplicability(entity) ?? State(config.IsMcpEnabled && entity.Mcp?.CustomToolEnabled == true)) + ]; + + foreach ((string key, EntityFeature feature, Func configured, Func effective) in features) + { + string configuredAny = NOT_APPLICABLE; + string effectiveAny = NOT_APPLICABLE; + foreach ((string name, Entity entity) in config.Entities) + { + // A missing entity in the root input can be a merged child or an expansion. + // It is NOT evidence that that entity omitted all its settings. + Input entityInput = input.Child("entities").ExistingObject(name); + configuredAny = Any(configuredAny, configured(name, entity, entityInput)); + effectiveAny = Any(effectiveAny, effective(name, entity)); + } + + if (input.Provenance is not null) + { + configuredAny = ConfiguredEntityPresence(config, input.Provenance, feature, configuredAny); + } + + Add(result, key, configuredAny, effectiveAny); + } + + result.Add("entities.any.table", AnyEntity(config, entity => SourceType(entity, EntitySourceType.Table))); + result.Add("entities.any.view", AnyEntity(config, entity => SourceType(entity, EntitySourceType.View))); + result.Add("entities.any.stored_procedure", AnyEntity(config, entity => SourceType(entity, EntitySourceType.StoredProcedure))); + // Neither is a modeled entity capability. Cosmos DB is not MCP persisted documents. + result.Add("entities.any.persisted_document", UNSUPPORTED); + result.Add("entities.any.parameter_embeddings", UNSUPPORTED); + result.Add("entities.any.custom_roles", AnyEntity(config, entity => State(entity.Permissions?.Any(permission => + !string.Equals(permission.Role, "anonymous", StringComparison.OrdinalIgnoreCase) + && !string.Equals(permission.Role, "authenticated", StringComparison.OrdinalIgnoreCase)) == true))); + result.Add("entities.any.request_policy", AnyEntity(config, entity => State(HasPolicy(entity, request: true)))); + result.Add("entities.any.database_policy", AnyEntity(config, entity => State(HasPolicy(entity, request: false)))); + result.Add("entities.any.policies", AnyEntity(config, entity => State(HasPolicy(entity, request: true) || HasPolicy(entity, request: false)))); + result.Add("entities.any.descriptions", AnyEntity(config, entity => State(!string.IsNullOrEmpty(entity.Description)))); + result.Add("entities.any.relationships", AnyEntity(config, entity => State(entity.Relationships?.Count > 0))); + result.Add("scale.entity_count", CountBucket(config.Entities.Entities.Count)); + } + + private static Input ChildSourceInput(RuntimeConfig config, string dataSourceName) + { + foreach (RuntimeConfig child in Configurations(config)) + { + if (!ReferenceEquals(child, config) && string.Equals(child.DefaultDataSourceName, dataSourceName, StringComparison.Ordinal)) + { + return Input.FromPresence(child.TelemetryPresence); + } + } + + return default; + } + + private static IEnumerable Configurations(RuntimeConfig root) + { + HashSet visited = new(ReferenceEqualityComparer.Instance); + Stack pending = new(); + pending.Push(root); + while (pending.TryPop(out RuntimeConfig? current)) + { + if (!visited.Add(current)) + { + continue; + } + + yield return current; + foreach ((_, RuntimeConfig child) in current.ChildConfigs) + { + pending.Push(child); + } + } + } + + private static string ConfiguredEntityPresence(RuntimeConfig config, TelemetryConfigurationPresence rootPresence, EntityFeature feature, string fallback) + { + EntityPresence presence = default; + foreach (RuntimeConfig current in Configurations(config)) + { + TelemetryConfigurationPresence? original = ReferenceEquals(current, config) ? rootPresence : current.TelemetryPresence; + if (original is not null) + { + presence = presence.Combine(original.Entities[feature]); + } + } + + long applicable = 0; + long enabled = 0; + bool unknownApplicability = false; + foreach ((_, Entity entity) in config.Entities) + { + if (feature == EntityFeature.McpCustomTool && CustomToolApplicability(entity) is string applicability) + { + unknownApplicability |= applicability == UNKNOWN; + continue; + } + + applicable++; + bool enabledInModel = feature switch + { + EntityFeature.Rest => entity.IsRestEnabled, + EntityFeature.GraphQL => entity.IsGraphQLEnabled, + EntityFeature.Cache => entity.Cache?.UserProvidedEnabledOptions == true && entity.Cache.Enabled == true, + EntityFeature.McpDml => entity.Mcp?.UserProvidedDmlToolsEnabled == true && entity.Mcp.DmlToolEnabled, + EntityFeature.McpCustomTool => entity.Mcp?.UserProvidedCustomToolEnabled == true && entity.Mcp.CustomToolEnabled, + _ => false + }; + enabled += enabledInModel ? 1 : 0; + } + + string configured = ResolveEntityPresence(presence, applicable, enabled, + defaultEnabled: feature is EntityFeature.Rest or EntityFeature.GraphQL, fallback: fallback); + return Any(configured, unknownApplicability ? UNKNOWN : NOT_APPLICABLE); + } + + private static string ResolveEntityPresence(EntityPresence presence, long applicable, long enabled, bool defaultEnabled, string fallback) + { + if (presence.Total == 0) + { + return fallback; + } + + if (presence.Total > applicable) + { + // The model no longer matches the captured declarations (for example after + // programmatic removal). Do not present an old aggregate as current provenance. + return UNKNOWN; + } + + long unavailable = applicable - presence.Total; + // REST/GraphQL converters erase explicitness and default missing/null options to on. + // Subtract those known defaults before claiming an explicitly enabled entity. The + // other three features retain UserProvided flags, so their defaults were not counted. + long defaultedEnabled = defaultEnabled ? presence.Missing + presence.ExplicitNull : 0; + if (presence.Present > 0 && enabled > defaultedEnabled + presence.Indeterminate + unavailable) + { + return ENABLED; + } + + if (unavailable > 0) + { + // Uncaptured children/generated entities are not omitted declarations. A retained + // UserProvided flag can still prove a positive, but negatives cannot erase doubt. + return fallback == ENABLED ? ENABLED : UNKNOWN; + } + + if (presence.ExplicitNull > 0 || presence.Indeterminate > 0) + { + return UNKNOWN; + } + + return presence.Present > 0 ? DISABLED : MISSING; + } + + private static string McpDmlEffective(RuntimeConfig config, Entity entity) + { + if (!config.IsMcpEnabled || entity.Mcp?.DmlToolEnabled == false) + { + return DISABLED; + } + + DmlToolsConfig? tools = config.McpDmlTools; + if (tools is null) + { + // IsEnabled advertises default-on tools, but ExecuteAsync rejects a null + // McpDmlTools. Do not invent effective data-serving enablement from discovery. + return UNKNOWN; + } + + return entity.Source.Type switch + { + EntitySourceType.StoredProcedure => State(tools.ExecuteEntity == true), + EntitySourceType.Table or EntitySourceType.View => State(tools.CreateRecord == true || tools.ReadRecords == true + || tools.UpdateRecord == true || tools.DeleteRecord == true || tools.AggregateRecords == true), + _ => UNKNOWN + }; + } + + private static string? CustomToolApplicability(Entity entity) => entity.Source.Type switch + { + EntitySourceType.StoredProcedure => null, + EntitySourceType.Table or EntitySourceType.View => NOT_APPLICABLE, + _ => UNKNOWN + }; + + private static bool HasPolicy(Entity entity, bool request) => entity.Permissions?.Any(permission => + permission.Actions?.Any(action => (request ? action.Policy?.Request : action.Policy?.Database) is not null) == true) == true; + + private static string SourceType(Entity entity, EntitySourceType expected) => entity.Source.Type switch + { + EntitySourceType.Table or EntitySourceType.View or EntitySourceType.StoredProcedure => State(entity.Source.Type == expected), + _ => UNKNOWN + }; + + private static string AnyEntity(RuntimeConfig config, Func select) + { + string state = NOT_APPLICABLE; + foreach ((_, Entity entity) in config.Entities) + { + state = Any(state, select(entity)); + } + + return state; + } + + private static void AddLimits(ImmutableDictionary.Builder result, RuntimeConfig config, Input input) + { + PaginationOptions? pagination = config.Runtime?.Pagination; + HostOptions? host = config.Runtime?.Host; + RuntimeCacheOptions? cache = config.Runtime?.Cache; + string defaultPage = PageBucket(config.DefaultPageSize()); + string maxPage = PageBucket(config.MaxPageSize()); + string maxBytes = BytesBucket((long)config.MaxResponseSizeMB() * 1024 * 1024); + Add(result, "limits.default_page_size", Configured(input.Child("pagination").Child("default-page-size"), defaultPage, pagination?.UserProvidedDefaultPageSize == true ? defaultPage : UNKNOWN), defaultPage); + Add(result, "limits.max_page_size", Configured(input.Child("pagination").Child("max-page-size"), maxPage, pagination?.UserProvidedMaxPageSize == true ? maxPage : UNKNOWN), maxPage); + Add(result, "limits.max_response_bytes", Configured(input.Child("host").Child("max-response-size-mb"), maxBytes, host?.UserProvidedMaxResponseSizeMB == true ? maxBytes : UNKNOWN), maxBytes); + result.Add("limits.max_response_enforced", State(config.MaxResponseSizeLogicEnabled())); + Add(result, "limits.cache_ttl_seconds", Configured(input.Child("cache").Child("ttl-seconds"), SecondsBucket(cache?.TtlSeconds), cache?.UserProvidedTtlOptions == true ? SecondsBucket(cache.TtlSeconds) : UNKNOWN), SecondsBucket(config.GlobalCacheEntryTtl())); + + // There is no global query-timeout setting. The modeled timeout is specifically for + // MCP aggregate-records; do not label it as the limit on REST/GraphQL/all SQL queries. + Add(result, "limits.query_timeout_seconds", UNSUPPORTED, UNSUPPORTED); + DmlToolsConfig? tools = config.McpDmlTools; + int timeout = tools?.EffectiveAggregateRecordsQueryTimeoutSeconds ?? DmlToolsConfig.DEFAULT_QUERY_TIMEOUT_SECONDS; + bool valid = timeout is >= 1 and <= DmlToolsConfig.MAX_QUERY_TIMEOUT_SECONDS; + string timeoutBucket = valid ? SecondsBucket(timeout) : UNKNOWN; + string configuredTimeout = Configured(input.Child("mcp").Child("dml-tools").Child("aggregate-records", ignoreCase: true).Child("query-timeout", ignoreCase: true), + timeoutBucket, tools?.UserProvidedAggregateRecordsQueryTimeout == true ? timeoutBucket : UNKNOWN); + // AggregateRecordsTool applies this same defensive fallback, even for a programmatic + // out-of-range value that has not passed configuration validation. + string effectiveTimeout = config.IsMcpEnabled && tools?.AggregateRecords == true + ? SecondsBucket(valid ? timeout : DmlToolsConfig.DEFAULT_QUERY_TIMEOUT_SECONDS) : NOT_APPLICABLE; + Add(result, "limits.mcp_aggregate_query_timeout_seconds", configuredTimeout, effectiveTimeout); + } + + private static string AuthenticationCategory(RuntimeConfig config) + { + if (config.IsUnauthenticatedIdentityProvider) + { + return "unauthenticated"; + } + + if (config.IsStaticWebAppsIdentityProvider) + { + return "static_web_apps"; + } + + if (config.IsAppServiceIdentityProvider) + { + return "app_service"; + } + + string? provider = config.Runtime?.Host?.Authentication?.Provider; + if (string.IsNullOrWhiteSpace(provider)) + { + return UNKNOWN; + } + + if (string.Equals(provider, AuthenticationOptions.SIMULATOR_AUTHENTICATION, StringComparison.OrdinalIgnoreCase)) + { + return "simulator"; + } + + return string.Equals(provider, "AzureAD", StringComparison.OrdinalIgnoreCase) + || string.Equals(provider, "EntraID", StringComparison.OrdinalIgnoreCase) ? "entra_id" : "jwt"; + } + + private static void Add(ImmutableDictionary.Builder result, string key, string configured, string effective) + { + result.Add(key + ".configured", configured); + result.Add(key + ".effective", effective); + } + + private static string State(bool? value) => value switch { true => ENABLED, false => DISABLED, _ => UNKNOWN }; + // A null or false UserProvided flag can also represent explicit JSON null. Only positive + // provenance (or a preserved nullable value) proves an explicit assignment without input. + private static string NullableSetting(bool? value) => value.HasValue ? State(value) : UNKNOWN; + private static string Provided(bool? value, bool provided) => provided ? State(value) : UNKNOWN; + + // Enabled proves "any"; otherwise uncertainty must not be erased by a known negative. + // Unsupported-only inputs stay unsupported; no applicable targets stay not_applicable. + private static string Any(string left, string right) + { + if (left == ENABLED || right == ENABLED) + { + return ENABLED; + } + + if (left == UNKNOWN || right == UNKNOWN) + { + return UNKNOWN; + } + + if (left == DISABLED || right == DISABLED) + { + return DISABLED; + } + + if (left == MISSING || right == MISSING) + { + return MISSING; + } + + if (left == UNSUPPORTED || right == UNSUPPORTED) + { + return UNSUPPORTED; + } + + return NOT_APPLICABLE; + } + + // Upper endpoints are inclusive; labels and boundaries are part of configuration-v1. + private static string CountBucket(long value) => value switch + { + < 0 => UNKNOWN, + 0 => "0", + 1 => "1", + <= 10 => "2-10", + <= 50 => "11-50", + <= 100 => "51-100", + <= 500 => "101-500", + _ => "501+" + }; + + private static string PageBucket(long value) => value switch + { + <= 0 => UNKNOWN, + <= 10 => "1-10", + <= 100 => "11-100", + <= 1000 => "101-1000", + <= 10000 => "1001-10000", + <= 100000 => "10001-100000", + _ => "100001+" + }; + + private static string BytesBucket(long value) => value switch + { + <= 0 => UNKNOWN, + <= 1048576 => "1-1048576", + <= 16777216 => "1048577-16777216", + <= 67108864 => "16777217-67108864", + <= 268435456 => "67108865-268435456", + _ => "268435457+" + }; + + private static string SecondsBucket(long? value) => value switch + { + null or <= 0 => UNKNOWN, + <= 5 => "1-5", + <= 30 => "6-30", + <= 60 => "31-60", + <= 300 => "61-300", + <= 3600 => "301-3600", + _ => "3601+" + }; + + private static string Configured(Input input, string resolved, string fallback) => input.Kind switch + { + InputKind.Missing => MISSING, + InputKind.Value => input.IsNull ? UNKNOWN : resolved, + InputKind.Indeterminate => UNKNOWN, + _ => fallback + }; + + private enum InputKind { Unavailable, Missing, Value, Indeterminate } + + /// Transient presence cursor, never retained in a snapshot or static state. + private readonly struct Input + { + private readonly JsonElement _value; + private InputKind OriginalKind { get; } + private readonly string? _path; + public TelemetryConfigurationPresence? Provenance { get; } + private Presence Captured => _path is not null && Provenance is not null && Provenance.Settings.TryGetValue(_path, out Presence value) + ? value : Presence.Unavailable; + public InputKind Kind => Provenance is null ? OriginalKind : Captured switch + { + Presence.Missing => InputKind.Missing, + Presence.Present or Presence.ExplicitNull => InputKind.Value, + Presence.Indeterminate => InputKind.Indeterminate, + _ => InputKind.Unavailable + }; + public bool IsNull => Provenance is null + ? _value.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined + : Captured == Presence.ExplicitNull; + + private Input(InputKind kind, JsonElement value = default) + { + OriginalKind = kind; + _value = value; + _path = null; + Provenance = null; + } + + private Input(TelemetryConfigurationPresence provenance, string path) + { + Provenance = provenance; + _path = path; + OriginalKind = InputKind.Unavailable; + _value = default; + } + + public static Input FromPresence(TelemetryConfigurationPresence? presence) => presence is null ? default : new(presence, string.Empty); + + public static Input FromRoot(JsonElement? value, TelemetryConfigurationPresence? fallback) => value is null ? FromPresence(fallback) + : value.Value.ValueKind == JsonValueKind.Object ? new(InputKind.Value, value.Value) : new(InputKind.Indeterminate); + + public Input Child(string property, bool ignoreCase = false) + { + if (Provenance is not null) + { + // Paths here contain only factory-owned schema names. Case handling and + // shorthand/ancestor-null normalization already happened during capture. + return new(Provenance, string.IsNullOrEmpty(_path) ? property : _path + "." + property); + } + + if (Kind != InputKind.Value) + { + return this; + } + + if (_value.ValueKind is JsonValueKind.True or JsonValueKind.False or JsonValueKind.String) + { + return new(InputKind.Missing); + } + + if (_value.ValueKind != JsonValueKind.Object) + { + return new(InputKind.Indeterminate); + } + + if (!ignoreCase) + { + return _value.TryGetProperty(property, out JsonElement child) ? new(InputKind.Value, child) : new(InputKind.Missing); + } + + Input found = new(InputKind.Missing); + foreach (JsonProperty candidate in _value.EnumerateObject()) + { + if (string.Equals(candidate.Name, property, StringComparison.OrdinalIgnoreCase)) + { + found = new(InputKind.Value, candidate.Value); + } + } + + return found; + } + + public Input Enablement(bool allowString = false) => Provenance is null && Kind == InputKind.Value + && (_value.ValueKind is JsonValueKind.True or JsonValueKind.False || (allowString && _value.ValueKind == JsonValueKind.String)) + ? this : Child("enabled"); + + public Input ShorthandOrChild(string property) => Provenance is null && Kind == InputKind.Value + && _value.ValueKind is JsonValueKind.True or JsonValueKind.False ? this : Child(property); + + public Input ExistingObject(string property) + { + if (Provenance is not null) + { + // Safe metadata has aggregate provenance, never customer-named children. + return default; + } + + Input child = Child(property); + return child.Kind == InputKind.Value && child._value.ValueKind == JsonValueKind.Object ? child : default; + } + } + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryContext.cs b/src/Core/Telemetry/Product/EngineTelemetryContext.cs new file mode 100644 index 0000000000..31c85fa42b --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryContext.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Immutable; +using System.Reflection; +using System.Runtime.InteropServices; +using Azure.DataApiBuilder.Product; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// Fixed, categorical context for one engine run. Never emits raw environment values. + internal static class EngineTelemetryContext + { + internal static ImmutableDictionary Create(string executionMode, Func readEnvironmentVariable) + { + string operatingSystem = GetOperatingSystem(); + bool? container = bool.TryParse(readEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), out bool value) ? value : null; + string launcher = Assembly.GetEntryAssembly()?.GetName().Name switch + { + "Microsoft.DataApiBuilder" => "cli", + "Azure.DataApiBuilder.Service" => "standalone", + _ => "unknown" + }; + + return ImmutableDictionary.Empty + .Add("dab_version", ProductInfo.GetProductVersion()) + .Add("os_family", operatingSystem) + .Add("os_version", operatingSystem == "unknown" ? "unknown" : $"{Environment.OSVersion.Version.Major}.{Environment.OSVersion.Version.Minor}") + .Add("architecture", GetArchitecture()) + .Add("dotnet_version", $"{Environment.Version.Major}.{Environment.Version.Minor}.{Environment.Version.Build}") + .Add("execution_mode", executionMode is "web" or "mcp_stdio" or "embedded" ? executionMode : "unknown") + .Add("launcher", launcher) + .Add("hosting", container == true ? "generic_container" : "unknown") + .Add("container", container switch { true => "enabled", false => "disabled", _ => "unknown" }) + // Test mode is not evidence of packaging or a release channel. + .Add("distribution", "unknown") + .Add("release_channel", "unknown") + .Add("packaging", "unknown"); + } + + private static string GetOperatingSystem() + { + if (OperatingSystem.IsWindows()) + { + return "windows"; + } + + if (OperatingSystem.IsLinux()) + { + return "linux"; + } + + return OperatingSystem.IsMacOS() ? "macos" : "unknown"; + } + + private static string GetArchitecture() => RuntimeInformation.ProcessArchitecture switch + { + Architecture.X64 => "x64", + Architecture.Arm64 => "arm64", + Architecture.X86 => "x86", + Architecture.Arm => "arm", + _ => "unknown" + }; + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryDelivery.cs b/src/Core/Telemetry/Product/EngineTelemetryDelivery.cs new file mode 100644 index 0000000000..4c7cf58e76 --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryDelivery.cs @@ -0,0 +1,409 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Threading.Channels; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// Best-effort, bounded, memory-only delivery. Capacity includes the current record throughout + /// initialization, export, and retry delays. No file spool or customer diagnostic pipeline is used. + /// Export admission and Disable share a short gate, never a lock held across exporter code. + /// An attempt admitted before Disable is in flight and receives cancellation; cancellation + /// cannot retract data already sent. Successful Export is not an end-to-end delivery guarantee. + /// + internal sealed class EngineTelemetryDelivery : IDisposable + { + private static readonly TimeSpan _maximumFlushTimeout = TimeSpan.FromSeconds(2); + private readonly object _sync = new(); + private readonly Func _factory; + private readonly Channel _queue; + private readonly CancellationTokenSource _cancellation; + private readonly TimeProvider _timeProvider; + private readonly int _capacity; + private readonly int _maxAttempts; + private readonly TimeSpan _flushTimeout; + private readonly Task _worker; + private Task _cancellationCompletion = Task.CompletedTask; + private Task? _stopTask; + private IEngineTelemetryExporter? _exporter; + private EngineTelemetryEvent? _inFlight; + private int _accepting = 1; + private int _outstanding; + private long _droppedEvents; + private bool _disabled; + + public EngineTelemetryDelivery( + Func factory, + int capacity = 256, + int maxAttempts = 3, + TimeSpan? flushTimeout = null) + : this(factory, capacity, maxAttempts, flushTimeout, TimeProvider.System) + { + } + + // Keep the integration constructor unchanged while allowing deterministic timer tests. + internal EngineTelemetryDelivery( + Func factory, + int capacity, + int maxAttempts, + TimeSpan? flushTimeout, + TimeProvider timeProvider) + { + ArgumentNullException.ThrowIfNull(factory); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(maxAttempts, 1); + TimeSpan timeout = flushTimeout ?? _maximumFlushTimeout; + if (timeout < TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(flushTimeout)); + } + + _factory = factory; + _capacity = capacity; + _maxAttempts = maxAttempts; + _flushTimeout = timeout < _maximumFlushTimeout ? timeout : _maximumFlushTimeout; + _timeProvider = timeProvider; + _cancellation = new(); + _queue = Channel.CreateBounded(new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + AllowSynchronousContinuations = false, + SingleReader = false, // Disable also drains queued records. + SingleWriter = false + }); + + // Start exactly one worker, even before the first enqueue. Do not capture a request's + // Activity, logger scopes, or other AsyncLocal state. The factory remains lazy. + if (ExecutionContext.IsFlowSuppressed()) + { + _worker = Task.Run(RunWorkerAsync); + } + else + { + using (ExecutionContext.SuppressFlow()) + { + _worker = Task.Run(RunWorkerAsync); + } + } + } + + /// + /// Saturating count of rejected or abandoned records, not failed attempts. An abandoned + /// in-flight record may already have reached the receiver before cancellation. + /// + public long DroppedEvents => Interlocked.Read(ref _droppedEvents); + + /// + /// Never waits for capacity, exporter initialization, network I/O, or retry work. No task + /// is created per enqueue. Reservations bound queued plus in-flight records, not just the + /// channel's buffer. Completion of the writer resolves races with StopAsync and Disable. + /// + public bool TryEnqueue(EngineTelemetryEvent record) + { + ArgumentNullException.ThrowIfNull(record); + if (Volatile.Read(ref _accepting) == 0 || !TryReserveSlot()) + { + AddDroppedEvents(1); + return false; + } + + if (_queue.Writer.TryWrite(record)) + { + return true; + } + + Interlocked.Decrement(ref _outstanding); + AddDroppedEvents(1); + return false; + } + + /// + /// Close admission and share one graceful drain deadline (at most two seconds). Caller + /// cancellation also disables delivery; it does not surface an application exception. + /// Expiry does not wait for exporter code or cancellation callbacks to finish. + /// + public Task StopAsync(CancellationToken cancellationToken = default) + { + if (cancellationToken.IsCancellationRequested) + { + Disable(); + return Task.CompletedTask; + } + + Task stopTask; + lock (_sync) + { + Volatile.Write(ref _accepting, 0); + _queue.Writer.TryComplete(); + stopTask = _stopTask ??= StopCoreAsync(); + } + + return cancellationToken.CanBeCanceled ? WaitForStopAsync(stopTask, cancellationToken) : stopTask; + } + + /// + /// Disable first, cancel the in-flight attempt, and discard owned records without a final + /// flush. Cancellation callbacks run asynchronously, outside the gate, so a callback may + /// reenter Dispose safely. Exporter disposal is exclusively the worker's responsibility. + /// + public void Disable() + { + lock (_sync) + { + if (_disabled) + { + return; + } + + _disabled = true; + Volatile.Write(ref _accepting, 0); + _queue.Writer.TryComplete(); + + // CancelAsync marks the token synchronously but does not execute arbitrary + // exporter callbacks on this thread. Observe faults even if the exporter stalls. + if (ExecutionContext.IsFlowSuppressed()) + { + _cancellationCompletion = ObserveCancellationAsync(_cancellation.CancelAsync()); + } + else + { + using (ExecutionContext.SuppressFlow()) + { + _cancellationCompletion = ObserveCancellationAsync(_cancellation.CancelAsync()); + } + } + + DiscardOwnedRecords(); + } + } + + public void Dispose() => Disable(); + + private bool TryReserveSlot() + { + int outstanding = Volatile.Read(ref _outstanding); + while (outstanding < _capacity) + { + int previous = Interlocked.CompareExchange(ref _outstanding, outstanding + 1, outstanding); + if (previous == outstanding) + { + return true; + } + + outstanding = previous; + } + + return false; + } + + private async Task RunWorkerAsync() + { + try + { + while (await _queue.Reader.WaitToReadAsync(_cancellation.Token).ConfigureAwait(false)) + { + EngineTelemetryEvent? record; + lock (_sync) + { + if (_disabled) + { + return; + } + + if (!_queue.Reader.TryRead(out record)) + { + continue; + } + + _inFlight = record; + } + + bool exported = await ExportWithRetriesAsync(record).ConfigureAwait(false); + lock (_sync) + { + // Disable may already have accounted for this in-flight record. + if (_inFlight is not null) + { + _inFlight = null; + if (!exported) + { + AddDroppedEvents(1); + } + + Interlocked.Decrement(ref _outstanding); + } + } + } + } + catch (OperationCanceledException) when (_cancellation.IsCancellationRequested) + { + // The reader or retry delay was canceled; the awaited task is observed. + } + catch (Exception) + { + // Telemetry failure must not fault an unobserved worker or use customer logs. + Disable(); + } + finally + { + Task cancellationCompletion; + lock (_sync) + { + // Prevent a later Disable from canceling a disposed token source. + _disabled = true; + Volatile.Write(ref _accepting, 0); + _queue.Writer.TryComplete(); + DiscardOwnedRecords(); + cancellationCompletion = _cancellationCompletion; + } + + // No caller waits indefinitely for this cleanup. In particular, never join + // cancellation callbacks from inside Disable or while holding the gate. + await cancellationCompletion.ConfigureAwait(false); + try + { + _exporter?.Dispose(); + } + catch (Exception) + { + // Exporter cleanup is best effort and isolated from application shutdown. + } + finally + { + _exporter = null; + _cancellation.Dispose(); + } + } + } + + private async Task ExportWithRetriesAsync(EngineTelemetryEvent record) + { + CancellationToken cancellationToken = _cancellation.Token; + for (int attempt = 0; attempt < _maxAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + // Initialization failures consume the same finite attempt budget. A null + // factory result is also a failed attempt, never an application failure. + _exporter ??= _factory(); + if (_exporter is not null) + { + lock (_sync) + { + if (_disabled) + { + return false; + } + + // This is the export-admission (logical start) boundary shared with + // Disable. No new attempt is admitted after it disables delivery. + // The adapter must honor cancellation if Disable wins between this + // admission and the invocation below; do not lock across network I/O. + } + + cancellationToken.ThrowIfCancellationRequested(); + if (await _exporter.ExportAsync(record, cancellationToken).ConfigureAwait(false)) + { + return true; + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return false; + } + catch (Exception) + { + // Factory/export exceptions and false results have identical retry limits. + } + + if (attempt < _maxAttempts - 1) + { + // 100 ms, 200 ms, 400 ms, 800 ms, then 1 s; cap before shifting to avoid + // overflow even when a caller supplies a large (but finite) attempt limit. + TimeSpan delay = TimeSpan.FromMilliseconds(Math.Min(100 * (1 << Math.Min(attempt, 4)), 1000)); + await Task.Delay(delay, _timeProvider, cancellationToken).ConfigureAwait(false); + } + } + + return false; + } + + private async Task StopCoreAsync() + { + try + { + await _worker.WaitAsync(_flushTimeout, _timeProvider).ConfigureAwait(false); + } + catch (Exception) + { + // Includes deadline expiry. Do not add a second wait after requesting cancel. + Disable(); + } + } + + private async Task WaitForStopAsync(Task stopTask, CancellationToken cancellationToken) + { + try + { + await stopTask.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + Disable(); + } + } + + private static async Task ObserveCancellationAsync(Task cancellation) + { + try + { + await cancellation.ConfigureAwait(false); + } + catch (Exception) + { + // Cancellation callbacks are exporter code too; their faults are observed. + } + } + + // Caller holds _sync. Slots reserved by concurrent producers but not yet written are + // released by those producers when TryWrite sees the completed writer. + private void DiscardOwnedRecords() + { + int discarded = 0; + while (_queue.Reader.TryRead(out _)) + { + Interlocked.Decrement(ref _outstanding); + discarded++; + } + + if (_inFlight is not null) + { + _inFlight = null; + Interlocked.Decrement(ref _outstanding); + discarded++; + } + + AddDroppedEvents(discarded); + } + + private void AddDroppedEvents(int count) + { + long current = Interlocked.Read(ref _droppedEvents); + while (count > 0 && current != long.MaxValue) + { + long next = current > long.MaxValue - count ? long.MaxValue : current + count; + long previous = Interlocked.CompareExchange(ref _droppedEvents, next, current); + if (previous == current) + { + return; + } + + current = previous; + } + } + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryEvent.cs b/src/Core/Telemetry/Product/EngineTelemetryEvent.cs new file mode 100644 index 0000000000..1eddae1eed --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryEvent.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Immutable; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// Immutable delivery envelope. Retries retain this instance, including its event identity + /// and occurrence time; they do not represent additional observations. + /// + internal sealed record EngineTelemetryEvent( + Guid EventId, + Guid SessionId, + long Sequence, + DateTimeOffset OccurredAt, + long ConfigurationEpoch, + string Name, + ImmutableDictionary Properties) + { + // Production collection/export enablement is not part of this implementation. + public bool IsSynthetic { get; } = true; + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryIdentityStore.cs b/src/Core/Telemetry/Product/EngineTelemetryIdentityStore.cs new file mode 100644 index 0000000000..0eb7e0e3c5 --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryIdentityStore.cs @@ -0,0 +1,515 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.InteropServices; +using System.Security; +using System.Text.Json; +using Azure.DataApiBuilder.Config.Telemetry; +using Microsoft.Win32.SafeHandles; +using Path = System.IO.Path; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// A random deployment identity, never a configuration or installation fingerprint. + internal sealed record EngineTelemetryIdentity(Guid ApiId, string Stability); + + /// + /// Best-effort engine-only identity storage beside an existing root/merged configuration. + /// The session must enforce enablement before calling and retain the result for its entire run, + /// including reloads: an ephemeral identity is deliberately not cached or recovered here. + /// + /// + /// Reset locally by deleting the configuration's .dab-telemetry.json sidecar while all affected + /// processes are stopped. The next enabled run generates an unrelated ID; no old/new linkage is + /// retained. Reset does not erase previously collected data, and opting out is not a reset. + /// + /// Persistence requires an existing, deployment-controlled directory. Windows inherits its ACL; + /// Linux requires an owner-controlled directory and a private, owner-readable identity file. + /// Static directory links and final-component links are rejected; final reads also use native + /// no-follow handles. This is not a sandbox against a directory owner replacing ancestors or + /// mounts during a call. Persistence supports Windows and Linux x64/arm64 with statx; other + /// platforms/native APIs fail closed to ephemeral identity. No permission changes are attempted. + /// No directories, installation IDs, network clients, locks or retry loops are created. Local + /// filesystem calls themselves have no hard time limit; filesystem crash durability is best effort. + /// + internal static class EngineTelemetryIdentityStore + { + private const string SIDECAR_SUFFIX = ".dab-telemetry.json"; + private const int SCHEMA_VERSION = 1; + private const int MAX_STATE_BYTES = 1024; + private const UnixFileMode PRIVATE_FILE_MODE = UnixFileMode.UserRead | UnixFileMode.UserWrite; + private const UnixFileMode UNSAFE_DIRECTORY_MODE = UnixFileMode.GroupWrite | UnixFileMode.OtherWrite; + private const FileAttributes UNSAFE_FILE_ATTRIBUTES = FileAttributes.Directory + | FileAttributes.ReparsePoint | FileAttributes.Device | FileAttributes.Offline; + + /// + /// Resolves once for the supplied root/merged configuration, not individual constituent files. + /// Returns only newly_saved, reused or ephemeral stability. Opt-out performs no filesystem I/O; + /// its returned ephemeral value is not permission to collect or transmit telemetry. + /// + public static EngineTelemetryIdentity Resolve(string? configPath) + { + return Resolve(configPath, Environment.GetEnvironmentVariable); + } + + /// Per-call test seam; does not mutate or depend on the process environment. + internal static EngineTelemetryIdentity Resolve(string? configPath, Func readEnvironmentVariable) + { + string? temporaryPath = null; + bool ownsTemporaryFile = false; + + try + { + ArgumentNullException.ThrowIfNull(readEnvironmentVariable); + if (ProductTelemetryPolicy.IsOptedOut(readEnvironmentVariable(ProductTelemetryPolicy.OPT_OUT_ENV_VAR)) + || !TryGetStoragePaths(configPath, out string directory, out string sidecar)) + { + return CreateEphemeral(); + } + + ReadResult result = ReadIdentity(sidecar, out Guid savedId); + if (result == ReadResult.Valid) + { + return IsSafeDirectory(directory) ? new(savedId, "reused") : CreateEphemeral(); + } + + // An existing but invalid/unreadable file belongs to its owner. Never repair, + // truncate, replace or remove it, including future versions of our own schema. + if (result != ReadResult.Missing || !CanCreateInDirectory(directory)) + { + return CreateEphemeral(); + } + + Guid candidate = Guid.NewGuid(); + temporaryPath = Path.Combine(directory, $".dab-telemetry-{Guid.NewGuid():N}.tmp"); + FileStreamOptions options = new() + { + Mode = FileMode.CreateNew, + Access = FileAccess.Write, + Share = FileShare.None, + BufferSize = MAX_STATE_BYTES + }; + + if (OperatingSystem.IsLinux()) + { + // Set mode at creation, never chmod a briefly world-readable temporary file. + options.UnixCreateMode = PRIVATE_FILE_MODE; + } + + using (FileStream stream = new(temporaryPath, options)) + { + // A failed CreateNew must not trigger cleanup of an existing file. + ownsTemporaryFile = true; + using Utf8JsonWriter writer = new(stream); + writer.WriteStartObject(); + writer.WriteNumber("version", SCHEMA_VERSION); + writer.WriteString("apiId", candidate); + writer.WriteEndObject(); + writer.Flush(); + stream.Flush(flushToDisk: true); + } + + // Recheck the directory before publishing; no overwrite also protects a file + // or link created by another process after the initial missing-file observation. + if (!IsSafeDirectory(directory) + || ReadIdentity(temporaryPath, out Guid stagedId) != ReadResult.Valid || stagedId != candidate) + { + return CreateEphemeral(); + } + + bool published = false; + try + { + File.Move(temporaryPath, sidecar, overwrite: false); + ownsTemporaryFile = false; + published = true; + } + catch (IOException) + { + // Possibly lost a creation race. Read the winner once, never retry publication. + } + + if (!IsSafeDirectory(directory) || ReadIdentity(sidecar, out savedId) != ReadResult.Valid + || !IsSafeDirectory(directory) || (published && savedId != candidate)) + { + return CreateEphemeral(); + } + + return new(savedId, published ? "newly_saved" : "reused"); + } + catch (Exception exception) when (IsStorageFailure(exception)) + { + // Telemetry must not fail the engine or log paths, file contents or exception text. + return CreateEphemeral(); + } + finally + { + if (ownsTemporaryFile && temporaryPath is not null) + { + try + { + if (IsSafeDirectory(Path.GetDirectoryName(temporaryPath)!)) + { + File.Delete(temporaryPath); + } + } + catch (Exception exception) when (IsStorageFailure(exception)) + { + // Only this invocation's successfully created temporary file is eligible + // for cleanup. A crash or cleanup failure may leave this tiny file behind. + } + } + } + } + + /// + /// Returns false because persistent notice state is unavailable in this implementation. + /// Does not read or write anything, even when opted out. The session must show the notice + /// once per enabled run before its first transmission; false must not suppress that notice. + /// + public static bool TryMarkNoticeShown(string? configPath) + { + _ = configPath; + return false; + } + + private static EngineTelemetryIdentity CreateEphemeral() => new(Guid.NewGuid(), "ephemeral"); + + private static bool TryGetStoragePaths(string? configPath, out string directory, out string sidecar) + { + directory = string.Empty; + sidecar = string.Empty; + if (string.IsNullOrWhiteSpace(configPath) + || (!OperatingSystem.IsWindows() + && (!OperatingSystem.IsLinux() || RuntimeInformation.ProcessArchitecture is not (Architecture.X64 or Architecture.Arm64)))) + { + return false; + } + + // Reject remote/URI/device and ambiguous drive-relative inputs before path I/O. + if (configPath.StartsWith(@"\\", StringComparison.Ordinal) + || configPath.StartsWith("//", StringComparison.Ordinal) + || configPath.Contains("://", StringComparison.Ordinal) + || (Path.IsPathRooted(configPath) && !Path.IsPathFullyQualified(configPath))) + { + return false; + } + + string fullPath = Path.GetFullPath(configPath); + if (OperatingSystem.IsWindows()) + { + // No UNC/device namespace, alternate data streams or mapped network drives. + if (fullPath.StartsWith(@"\\", StringComparison.Ordinal) || fullPath.AsSpan(2).Contains(':')) + { + return false; + } + + DriveType driveType = new DriveInfo(Path.GetPathRoot(fullPath)!).DriveType; + if (driveType is not (DriveType.Fixed or DriveType.Removable or DriveType.Ram)) + { + return false; + } + } + + directory = Path.GetDirectoryName(fullPath) ?? string.Empty; + if (directory.Length == 0 + || (OperatingSystem.IsLinux() && new DriveInfo(directory).DriveType is not (DriveType.Fixed or DriveType.Ram)) + || !IsSafeDirectory(directory)) + { + return false; + } + + // Configuration contents are never opened. An absent/late/in-memory configuration + // has no safe persistent target, including when a relative filename was supplied. + if (OperatingSystem.IsLinux()) + { + if (!TryGetLinuxStatus(fullPath, out LinuxFileStatus status) + || (status.Mode & NativeMethods.FILE_TYPE_MASK) != NativeMethods.REGULAR_FILE) + { + return false; + } + } + else if ((File.GetAttributes(fullPath) & UNSAFE_FILE_ATTRIBUTES) != 0) + { + return false; + } + + sidecar = fullPath + SIDECAR_SUFFIX; + return true; + } + + private static bool IsSafeDirectory(string path) + { + uint userId = OperatingSystem.IsLinux() ? NativeMethods.GetEffectiveUserId() : 0; + bool immediateParent = true; + for (DirectoryInfo? directory = new(path); directory is not null; directory = directory.Parent) + { + if (OperatingSystem.IsLinux()) + { + if (!TryGetLinuxStatus(directory.FullName, out LinuxFileStatus status) + || (status.Mode & NativeMethods.FILE_TYPE_MASK) != NativeMethods.DIRECTORY + || (immediateParent ? status.UserId != userId : status.UserId != userId && status.UserId != 0)) + { + return false; + } + + UnixFileMode mode = (UnixFileMode)status.Mode; + if ((mode & UNSAFE_DIRECTORY_MODE) != 0 + && (immediateParent || (mode & UnixFileMode.StickyBit) == 0)) + { + return false; + } + } + else + { + FileAttributes attributes = File.GetAttributes(directory.FullName); + if ((attributes & UNSAFE_FILE_ATTRIBUTES) != FileAttributes.Directory) + { + return false; + } + } + + // A sticky ancestor such as /tmp is allowed, but never a shared-writable + // immediate parent. All checks are metadata-only and do not repair permissions. + immediateParent = false; + } + + return true; + } + + private static bool CanCreateInDirectory(string path) + { + // On Windows this bit is advisory rather than an ACL. Be conservative even when + // the caller could technically write; actual ACL/volume failures are caught too. + if ((File.GetAttributes(path) & FileAttributes.ReadOnly) != 0) + { + return false; + } + + if (!OperatingSystem.IsLinux()) + { + // Windows ACLs and read-only volumes are enforced by CreateNew, not changed here. + return true; + } + + const UnixFileMode required = UnixFileMode.UserWrite | UnixFileMode.UserExecute; + return (File.GetUnixFileMode(path) & required) == required; + } + + private static ReadResult ReadIdentity(string path, out Guid apiId) + { + apiId = default; + if (OperatingSystem.IsLinux()) + { + // Reject static special files before opening; the handle check below also + // rejects replacements during the race between inspection and open. + if (!TryGetLinuxStatus(path, out LinuxFileStatus status)) + { + return Marshal.GetLastPInvokeError() == 2 ? ReadResult.Missing : ReadResult.Unavailable; + } + + if (!IsSafeLinuxIdentity(status)) + { + return ReadResult.Unavailable; + } + } + + using SafeFileHandle handle = OpenIdentityNoFollow(path, out bool missing); + if (handle.IsInvalid) + { + return missing ? ReadResult.Missing : ReadResult.Unavailable; + } + + if (OperatingSystem.IsLinux()) + { + if (NativeMethods.StatHandle(handle, string.Empty, NativeMethods.AT_EMPTY_PATH, + NativeMethods.REQUIRED_STATUS, out LinuxFileStatus status) != 0 + || (status.Mask & NativeMethods.REQUIRED_STATUS) != NativeMethods.REQUIRED_STATUS + || !IsSafeLinuxIdentity(status)) + { + return ReadResult.Unavailable; + } + } + else if (NativeMethods.GetFileType(handle) != NativeMethods.DISK_FILE + || (File.GetAttributes(handle) & UNSAFE_FILE_ATTRIBUTES) != 0) + { + return ReadResult.Unavailable; + } + + // Disable stream read-ahead and cap actual bytes too, not only the initial length. + // NONBLOCK + a regular-file check prevents a substituted Unix FIFO from blocking reads. + using FileStream stream = new(handle, FileAccess.Read, bufferSize: 1, isAsync: false); + long expectedLength = stream.Length; + if (expectedLength is <= 0 or > MAX_STATE_BYTES) + { + return ReadResult.Unavailable; + } + + byte[] bytes = new byte[MAX_STATE_BYTES + 1]; + int count = 0; + while (count < bytes.Length) + { + int read = stream.Read(bytes, count, bytes.Length - count); + if (read == 0) + { + break; + } + + count += read; + } + + if (count != expectedLength || count > MAX_STATE_BYTES || stream.Length != expectedLength) + { + return ReadResult.Unavailable; + } + + using JsonDocument document = JsonDocument.Parse(bytes.AsMemory(0, count), new() { MaxDepth = 2 }); + if (document.RootElement.ValueKind != JsonValueKind.Object) + { + return ReadResult.Unavailable; + } + + bool hasVersion = false; + bool hasId = false; + foreach (JsonProperty property in document.RootElement.EnumerateObject()) + { + if (property.NameEquals("version") && !hasVersion + && property.Value.ValueKind == JsonValueKind.Number + && property.Value.TryGetInt32(out int version) && version == SCHEMA_VERSION) + { + hasVersion = true; + } + else if (property.NameEquals("apiId") && !hasId + && property.Value.ValueKind == JsonValueKind.String + && TryParseRandomId(property.Value.GetString(), out apiId)) + { + hasId = true; + } + else + { + // Reject duplicate/unknown fields, unexpected kinds and future schemas. + return ReadResult.Unavailable; + } + } + + return hasVersion && hasId ? ReadResult.Valid : ReadResult.Unavailable; + } + + private static bool TryParseRandomId(string? value, out Guid apiId) + { + // UUID v4/RFC variant only; a file cannot prove how the saved bytes were generated. + apiId = default; + return value is { Length: 36 } && value[14] == '4' + && value[19] is '8' or '9' or 'a' or 'b' or 'A' or 'B' + && Guid.TryParseExact(value, "D", out apiId); + } + + private static bool IsSafeLinuxIdentity(LinuxFileStatus status) + { + return (status.Mode & NativeMethods.FILE_TYPE_MASK) == NativeMethods.REGULAR_FILE + && status.UserId == NativeMethods.GetEffectiveUserId() + && ((UnixFileMode)(status.Mode & 0xFFF) & ~PRIVATE_FILE_MODE) == 0 + && ((UnixFileMode)status.Mode & UnixFileMode.UserRead) != 0; + } + + private static SafeFileHandle OpenIdentityNoFollow(string path, out bool missing) + { + SafeFileHandle handle; + int error; + if (OperatingSystem.IsWindows()) + { + handle = NativeMethods.OpenWindowsFile(path, NativeMethods.GENERIC_READ, FileShare.Read, + IntPtr.Zero, FileMode.Open, NativeMethods.OPEN_REPARSE_POINT, IntPtr.Zero); + error = Marshal.GetLastPInvokeError(); + } + else + { + int descriptor = NativeMethods.OpenLinuxFile(path, + NativeMethods.O_NOFOLLOW | NativeMethods.O_NONBLOCK | NativeMethods.O_CLOEXEC); + error = Marshal.GetLastPInvokeError(); + handle = new((IntPtr)descriptor, ownsHandle: true); + } + + // ENOENT and ERROR_FILE_NOT_FOUND are both 2. Other failures never permit creation. + missing = handle.IsInvalid && error == 2; + return handle; + } + + private static bool TryGetLinuxStatus(string path, out LinuxFileStatus status) + { + return NativeMethods.StatPath(NativeMethods.AT_FDCWD, path, NativeMethods.AT_SYMLINK_NOFOLLOW, + NativeMethods.REQUIRED_STATUS, out status) == 0 + && (status.Mask & NativeMethods.REQUIRED_STATUS) == NativeMethods.REQUIRED_STATUS; + } + + private static bool IsStorageFailure(Exception exception) => exception is IOException + or UnauthorizedAccessException or SecurityException or ArgumentException or NotSupportedException + or JsonException or InvalidOperationException or DllNotFoundException or EntryPointNotFoundException + or BadImageFormatException or MarshalDirectiveException; + + private enum ReadResult + { + Missing, + Valid, + Unavailable + } + + // Linux statx has a fixed, architecture-independent layout (unlike struct stat). + [StructLayout(LayoutKind.Explicit, Size = 256)] + private struct LinuxFileStatus + { + [FieldOffset(0)] + public uint Mask; + + [FieldOffset(20)] + public uint UserId; + + [FieldOffset(28)] + public ushort Mode; + } + + private static class NativeMethods + { + internal const uint GENERIC_READ = 0x80000000; + internal const uint OPEN_REPARSE_POINT = 0x00200000; + internal const uint DISK_FILE = 1; + internal const int O_NOFOLLOW = 0x20000; + internal const int O_NONBLOCK = 0x800; + internal const int O_CLOEXEC = 0x80000; + internal const int AT_FDCWD = -100; + internal const int AT_SYMLINK_NOFOLLOW = 0x100; + internal const int AT_EMPTY_PATH = 0x1000; + internal const uint REQUIRED_STATUS = 0xB; // STATX_TYPE | STATX_MODE | STATX_UID + internal const int FILE_TYPE_MASK = 0xF000; + internal const int REGULAR_FILE = 0x8000; + internal const int DIRECTORY = 0x4000; + + [DllImport("kernel32.dll", EntryPoint = "CreateFileW", CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)] + [DefaultDllImportSearchPaths(DllImportSearchPath.System32)] + internal static extern SafeFileHandle OpenWindowsFile(string path, uint access, FileShare share, + IntPtr securityAttributes, FileMode creation, uint flags, IntPtr template); + + [DllImport("kernel32.dll", SetLastError = true)] + [DefaultDllImportSearchPaths(DllImportSearchPath.System32)] + internal static extern uint GetFileType(SafeFileHandle handle); + + [DllImport("libc", EntryPoint = "open", SetLastError = true)] + [DefaultDllImportSearchPaths(DllImportSearchPath.SafeDirectories)] + internal static extern int OpenLinuxFile([MarshalAs(UnmanagedType.LPUTF8Str)] string path, int flags); + + [DllImport("libc", EntryPoint = "statx", SetLastError = true)] + [DefaultDllImportSearchPaths(DllImportSearchPath.SafeDirectories)] + internal static extern int StatPath(int directory, [MarshalAs(UnmanagedType.LPUTF8Str)] string path, + int flags, uint mask, out LinuxFileStatus status); + + [DllImport("libc", EntryPoint = "statx", SetLastError = true)] + [DefaultDllImportSearchPaths(DllImportSearchPath.SafeDirectories)] + internal static extern int StatHandle(SafeFileHandle handle, [MarshalAs(UnmanagedType.LPUTF8Str)] string path, + int flags, uint mask, out LinuxFileStatus status); + + [DllImport("libc", EntryPoint = "geteuid")] + [DefaultDllImportSearchPaths(DllImportSearchPath.SafeDirectories)] + internal static extern uint GetEffectiveUserId(); + } + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryMeasurementScope.cs b/src/Core/Telemetry/Product/EngineTelemetryMeasurementScope.cs new file mode 100644 index 0000000000..66510c2228 --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryMeasurementScope.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// The owning operation sets its outcome; disposal records it once and restores nesting. + /// Failure is the default so an exception cannot accidentally become a successful operation. + /// + internal sealed class EngineTelemetryMeasurementScope : IDisposable + { + private readonly Action _record; + private readonly Action? _restore; + private int _disposed; + private EngineTelemetryOutcome _outcome = EngineTelemetryOutcome.Failure; + + internal EngineTelemetryMeasurementScope(Action record, Action? restore = null) + { + _record = record; + _restore = restore; + } + + public void Complete(EngineTelemetryOutcome outcome) + => _outcome = EngineTelemetryDimensions.Normalize(outcome); + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + try + { + _record(_outcome); + } + catch (Exception) + { + // Product measurements cannot fail customer work. + } + finally + { + _restore?.Invoke(); + } + } + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryMeasurements.cs b/src/Core/Telemetry/Product/EngineTelemetryMeasurements.cs new file mode 100644 index 0000000000..7306dadd40 --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryMeasurements.cs @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Immutable; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + internal enum EngineTelemetryApi { Unknown, Rest, GraphQL, Mcp } + + internal enum EngineTelemetryTransport { Unknown, Http, Stdio, InProcess } + + internal enum EngineTelemetryRole { Unknown, Anonymous, Authenticated, Custom } + + internal enum EngineTelemetryOperation { Unknown, Read, Write, Execute } + + internal enum EngineTelemetryProvider { Unknown, MsSql, DwSql, PostgreSql, MySql, CosmosDb } + + internal enum EngineTelemetryObject { Unknown, Table, View, StoredProcedure, Document } + + internal enum EngineTelemetryCacheLayer { Unknown, Level1, Level2 } + + internal enum EngineTelemetryCacheResult { Unknown, Hit, Miss } + + internal enum EngineTelemetryOutcome { Unknown, Success, Failure, PartialFailure, Canceled } + + internal enum EngineTelemetryMeasurement { Request, Operation, DatabaseAttempt, CacheLookup, Embedding, HttpOutcome } + + internal enum EngineTelemetryHttpStatus { Unknown, Informational, Success, Redirect, ClientError, ServerError } + + /// + /// An opaque accepted-configuration handle. Capture it when work starts and retain it until + /// completion; neither a reload nor a window transition changes its epoch. + /// The owner token prevents mixing measurements from different engine runs. + /// + internal readonly record struct EngineTelemetryConfiguration(object Owner, long Epoch); + + /// + /// Closed, categorical dimensions only. Factory methods select a small, family-specific + /// set of joint dimensions; unused fields are not additional measurement dimensions. + /// No customer-defined string is accepted by the aggregation API. + /// + internal readonly record struct EngineTelemetryDimensions + { + public EngineTelemetryMeasurement Measurement { get; private init; } + public EngineTelemetryApi Api { get; private init; } + public EngineTelemetryTransport Transport { get; private init; } + public EngineTelemetryRole Role { get; private init; } + public EngineTelemetryOperation Operation { get; private init; } + public EngineTelemetryProvider Provider { get; private init; } + public EngineTelemetryObject ObjectType { get; private init; } + public EngineTelemetryCacheLayer CacheLayer { get; private init; } + public EngineTelemetryCacheResult CacheResult { get; private init; } + public EngineTelemetryHttpStatus HttpStatusClass { get; private init; } + + public static EngineTelemetryDimensions ForRequest(EngineTelemetryApi api, EngineTelemetryTransport transport, EngineTelemetryRole role) + => new() { Measurement = EngineTelemetryMeasurement.Request, Api = Normalize(api), Transport = Normalize(transport), Role = Normalize(role) }; + + public static EngineTelemetryDimensions ForOperation(EngineTelemetryApi api, EngineTelemetryOperation operation, EngineTelemetryProvider provider, EngineTelemetryObject objectType) + => new() { Measurement = EngineTelemetryMeasurement.Operation, Api = Normalize(api), Operation = Normalize(operation), Provider = Normalize(provider), ObjectType = Normalize(objectType) }; + + public static EngineTelemetryDimensions ForDatabaseAttempt(EngineTelemetryProvider provider) + => new() { Measurement = EngineTelemetryMeasurement.DatabaseAttempt, Provider = Normalize(provider) }; + + public static EngineTelemetryDimensions ForCacheLookup(EngineTelemetryCacheLayer layer, EngineTelemetryCacheResult result) + => new() { Measurement = EngineTelemetryMeasurement.CacheLookup, CacheLayer = Normalize(layer), CacheResult = Normalize(result) }; + + public static EngineTelemetryDimensions ForEmbedding(EngineTelemetryApi api) + => new() { Measurement = EngineTelemetryMeasurement.Embedding, Api = Normalize(api) }; + + public static EngineTelemetryDimensions ForHttpOutcome(EngineTelemetryApi api, int? status) + => new() + { + Measurement = EngineTelemetryMeasurement.HttpOutcome, + Api = Normalize(api), + Transport = EngineTelemetryTransport.Http, + HttpStatusClass = status switch + { + >= 100 and < 200 => EngineTelemetryHttpStatus.Informational, + >= 200 and < 300 => EngineTelemetryHttpStatus.Success, + >= 300 and < 400 => EngineTelemetryHttpStatus.Redirect, + >= 400 and < 500 => EngineTelemetryHttpStatus.ClientError, + >= 500 and < 600 => EngineTelemetryHttpStatus.ServerError, + _ => EngineTelemetryHttpStatus.Unknown + } + }; + + internal static T Normalize(T value) where T : struct, Enum => Enum.IsDefined(value) ? value : default; + } + + internal sealed record EngineTelemetryOutcomeCounts(long Unknown, long Success, long Failure, long PartialFailure, long Canceled); + + /// + /// Noncumulative histogram counts. Upper bounds are inclusive, in milliseconds; the final + /// bucket has no upper bound. Missing/invalid timings do not become zero-duration samples. + /// + internal sealed record EngineTelemetryHistogram(ImmutableArray Buckets, long TimedCount, bool IsComplete) + { + public static ImmutableArray UpperBoundsMilliseconds { get; } = [1, 5, 10, 50, 100, 500, 1000, 5000, 30000]; + + public const string BUCKET_SCHEMA = "request-latency-ms-v1"; + } + + internal sealed record EngineTelemetrySeries( + long ConfigurationEpoch, + EngineTelemetryDimensions Dimensions, + long Count, + EngineTelemetryOutcomeCounts? Outcomes, + EngineTelemetryHistogram? Latency, + bool IsCapped); + + internal sealed record EngineTelemetryWindow( + DateTimeOffset Start, + DateTimeOffset End, + bool IsFinal, + ImmutableArray Series, + long SeriesCapacityDrops, + long CounterCapacityDrops, + long ClockRegressionDrops, + bool LossCountsCapped); + + /// + /// Ownership of immutable completed windows passes to the caller once. This is an internal + /// aggregation result, not the versioned event envelope or an exporter payload. + /// + internal sealed record EngineTelemetryDrain( + ImmutableArray Windows, + long DroppedWindows, + long DroppedMeasurements, + bool LossCountsCapped) + { + public static EngineTelemetryDrain Empty { get; } = new([], 0, 0, false); + + // Only explicit synthetic validation can construct an enabled aggregator in this phase. + public bool IsSynthetic { get; } = true; + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryOptions.cs b/src/Core/Telemetry/Product/EngineTelemetryOptions.cs new file mode 100644 index 0000000000..d3d1f8efdd --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryOptions.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.DataApiBuilder.Config.Telemetry; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// Internal validation controls, not customer configuration or production enablement. + /// Shipping collection remains disabled pending privacy review and exporter validation. + /// + internal sealed record EngineTelemetryOptions + { + public const string OPT_OUT_ENVIRONMENT_VARIABLE = ProductTelemetryPolicy.OPT_OUT_ENV_VAR; + public const int MAX_SERIES_PER_WINDOW = 256; + public const int MAX_PENDING_WINDOWS = 4; + + public bool EnableSyntheticCollection { get; init; } + + public int SeriesCapacity { get; init; } = MAX_SERIES_PER_WINDOW; + + public int PendingWindowCapacity { get; init; } = MAX_PENDING_WINDOWS; + + // A lower ceiling permits deterministic saturation tests without billions of observations. + public long CounterCeiling { get; init; } = long.MaxValue; + + internal void Validate() + { + ArgumentOutOfRangeException.ThrowIfLessThan(SeriesCapacity, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(SeriesCapacity, MAX_SERIES_PER_WINDOW); + ArgumentOutOfRangeException.ThrowIfLessThan(PendingWindowCapacity, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(PendingWindowCapacity, MAX_PENDING_WINDOWS); + ArgumentOutOfRangeException.ThrowIfLessThan(CounterCeiling, 1); + } + + internal static bool IsOptedOut(string? value) + { + return ProductTelemetryPolicy.IsOptedOut(value); + } + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryRequestScope.cs b/src/Core/Telemetry/Product/EngineTelemetryRequestScope.cs new file mode 100644 index 0000000000..35b3ac886d --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryRequestScope.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.DataApiBuilder.Config.ObjectModel; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// Captures one request's accepted configuration and timing. Complete records exactly once; + /// Dispose only restores ambient state because a response can complete after dispatch returns. + /// + internal sealed class EngineTelemetryRequestScope : IDisposable + { + private readonly EngineTelemetrySession _session; + private readonly EngineTelemetryRequestScope? _previous; + private const int COMPLETED_FLAG = 1 << 16; + private int _disposed; + private int _outcome; + private int _eligible; + + internal EngineTelemetryRequestScope( + EngineTelemetrySession session, + EngineTelemetryRequestScope? previous, + EngineTelemetryConfiguration configuration, + RuntimeConfig? config, + EngineTelemetryApi api, + EngineTelemetryTransport transport, + EngineTelemetryRole role, + bool eligible, + long started) + { + _session = session; + _previous = previous; + Configuration = configuration; + Config = config; + Api = api; + Transport = transport; + Role = role; + _eligible = eligible ? 1 : 0; + Started = started; + } + + internal EngineTelemetryConfiguration Configuration { get; } + internal RuntimeConfig? Config { get; } + internal EngineTelemetryApi Api { get; } + internal EngineTelemetryTransport Transport { get; } + internal EngineTelemetryRole Role { get; private set; } + internal long Started { get; } + internal bool IsEligible => Volatile.Read(ref _eligible) != 0; + internal bool IsCompleted => (Volatile.Read(ref _outcome) & COMPLETED_FLAG) != 0; + internal bool IsDisposed => Volatile.Read(ref _disposed) != 0; + public EngineTelemetryOutcome Outcome => (EngineTelemetryOutcome)(Volatile.Read(ref _outcome) & ~COMPLETED_FLAG); + + public void MarkEligible() => Volatile.Write(ref _eligible, 1); + + public void SetOutcome(EngineTelemetryOutcome outcome) + { + int normalized = (int)EngineTelemetryDimensions.Normalize(outcome); + int observed = Volatile.Read(ref _outcome); + while ((observed & COMPLETED_FLAG) == 0) + { + int previous = Interlocked.CompareExchange(ref _outcome, normalized, observed); + if (previous == observed) + { + return; + } + + observed = previous; + } + } + + public void SetRole(EngineTelemetryRole role) => Role = EngineTelemetryDimensions.Normalize(role); + + public EngineTelemetryRequestScope ForkForCompletion() + => new(_session, previous: null, Configuration, Config, Api, Transport, Role, IsEligible, Started); + + public void Complete(EngineTelemetryOutcome outcome, int? httpStatusCode = null) + { + EngineTelemetryOutcome normalized = EngineTelemetryDimensions.Normalize(outcome); + int terminal = (int)normalized | COMPLETED_FLAG; + int observed = Volatile.Read(ref _outcome); + while ((observed & COMPLETED_FLAG) == 0) + { + int previous = Interlocked.CompareExchange(ref _outcome, terminal, observed); + if (previous == observed) + { + // Completion and its terminal outcome are one atomic state transition. + // A tool result racing an HTTP abort cannot overwrite cancellation. + _session.CompleteRequest(this, normalized, httpStatusCode); + return; + } + + observed = previous; + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _session.RestoreRequest(this, _previous); + } + } + } +} diff --git a/src/Core/Telemetry/Product/EngineTelemetrySession.cs b/src/Core/Telemetry/Product/EngineTelemetrySession.cs new file mode 100644 index 0000000000..e673fa2456 --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetrySession.cs @@ -0,0 +1,594 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Immutable; +using System.Globalization; +using System.Text.Json; +using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Telemetry; +using static Azure.DataApiBuilder.Core.Telemetry.Product.EngineTelemetryValueFormatter; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// One engine's product telemetry, isolated from all customer diagnostic providers. Only + /// deliberate synthetic validation can enable this implementation before privacy approval. + /// Request adapters supply closed categories and never forward diagnostics or payloads. + /// + internal sealed class EngineTelemetrySession : IProductTelemetryControl, IDisposable + { + private readonly object _sync = new(); + private readonly TimeProvider _clock; + private readonly EngineTelemetryAggregator _aggregator; + private readonly AsyncLocal _request = new(); + private readonly AsyncLocal _operationDepth = new(); + private readonly EngineTelemetryDelivery? _delivery; + private readonly ImmutableDictionary _context; + private readonly long _started; + private readonly string? _configPath; + private readonly Func _resolveIdentity; + private ITimer? _timer; + private Guid _sessionId; + private long _sequence; + private long _readyAt; + private long _lastHeartbeat; + private bool _enabled; + private bool _hostReady; + private bool _ready; + private bool _firstServed; + private bool _firstSuccess; + private bool _startupFailed; + private RuntimeConfig? _config; + private EngineTelemetryConfiguration _configuration; + private long _configurationAcceptanceGeneration; + private ImmutableDictionary _snapshot = ImmutableDictionary.Empty; + private EngineTelemetryIdentity? _identity; + private Lazy? _identityResolution; + private Task? _stopTask; + + private EngineTelemetrySession(bool enabled, Func? exporterFactory, + string? configPath, string executionMode, TimeProvider clock, Action? showNotice, + Func? resolveIdentity, bool startTimer, Func readEnvironmentVariable) + { + _clock = clock; + _configPath = configPath; + _resolveIdentity = resolveIdentity ?? EngineTelemetryIdentityStore.Resolve; + _aggregator = EngineTelemetryAggregator.Create(new() { EnableSyntheticCollection = enabled }, clock, _ => null); + _context = ImmutableDictionary.Empty; + if (!enabled || exporterFactory is null) + { + return; + } + + try + { + // No notice/sender/identity work occurs on the disabled path. Failure to show + // the required notice fails closed, without using the customer's logging sinks. + (showNotice ?? ShowNotice)(); + _sessionId = Guid.NewGuid(); + HealthProbeToken = Guid.NewGuid().ToString("N"); + _started = clock.GetTimestamp(); + _lastHeartbeat = _started; + _context = EngineTelemetryContext.Create(executionMode, readEnvironmentVariable); + _delivery = new EngineTelemetryDelivery(exporterFactory); + _enabled = true; + Emit("dab.engine.process_started", 0, ImmutableDictionary.Empty); + if (startTimer) + { + if (ExecutionContext.IsFlowSuppressed()) + { + _timer = clock.CreateTimer(_ => Tick(), null, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1)); + } + else + { + using (ExecutionContext.SuppressFlow()) + { + _timer = clock.CreateTimer(_ => Tick(), null, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1)); + } + } + } + } + catch (Exception) + { + _enabled = false; + _aggregator.Disable(); + _delivery?.Disable(); + } + } + + internal static EngineTelemetrySession Create(Func? exporterFactory = null, + bool enableSyntheticCollection = false, string? configPath = null, string executionMode = "embedded", + TimeProvider? clock = null, Func? readEnvironmentVariable = null, + Action? showNotice = null, Func? resolveIdentity = null, bool startTimer = true) + { + bool enabled = enableSyntheticCollection && exporterFactory is not null; + if (enabled) + { + enabled = !ProductTelemetryPolicy.IsOptedOut((readEnvironmentVariable ?? Environment.GetEnvironmentVariable)(ProductTelemetryPolicy.OPT_OUT_ENV_VAR)); + } + + return new(enabled, exporterFactory, configPath, executionMode, clock ?? TimeProvider.System, showNotice, resolveIdentity, startTimer, + readEnvironmentVariable ?? Environment.GetEnvironmentVariable); + } + + public bool IsEnabled => Volatile.Read(ref _enabled); + + // In-memory self-probe correlation only. Never exported, persisted or used for auth. + internal string? HealthProbeToken { get; } + + public bool IsReady + { + get + { + lock (_sync) + { + return _enabled && _ready; + } + } + } + + public EngineTelemetryRequestScope? CurrentRequest => _request.Value; + + public void AcceptConfiguration(RuntimeConfig config, string delivery = "startup", string? configPath = null, + bool onlyIfUnconfigured = false) + { + if (!IsEnabled) + { + return; + } + + try + { + Lazy identityResolution; + long acceptanceGeneration; + lock (_sync) + { + if (!_enabled || (onlyIfUnconfigured && _configurationAcceptanceGeneration != 0)) + { + return; + } + + // Reserve acceptance order before projection or I/O. An older waiter must + // not overwrite a newer accepted model merely because it finishes later. + acceptanceGeneration = checked(++_configurationAcceptanceGeneration); + if (ReferenceEquals(config, _config)) + { + return; + } + + // Resolve once, but never hold the session gate across filesystem I/O. + // Disable/shutdown must not wait for an identity read on a slow volume. + identityResolution = _identityResolution ??= new( + () => _resolveIdentity(configPath ?? _configPath), LazyThreadSafetyMode.ExecutionAndPublication); + } + + ImmutableDictionary snapshot = EngineTelemetrySnapshotFactory.Create(config) + .Add("configuration_delivery", delivery is "startup" or "late_configuration" or "hot_reload" ? delivery : "unknown"); + EngineTelemetryIdentity identity = identityResolution.Value; + lock (_sync) + { + // An in-flight read can finish after stop. It cannot revive collection or + // emit a stale configuration after a newer acceptance superseded it. + if (!_enabled || acceptanceGeneration != _configurationAcceptanceGeneration || ReferenceEquals(config, _config)) + { + return; + } + + _identity ??= identity; + _configuration = _aggregator.AcceptConfiguration(); + _config = config; + _snapshot = snapshot; + if (_ready) + { + Emit("dab.engine.configuration_changed", _configuration.Epoch, snapshot); + } + else + { + TryReady(); + } + } + } + catch (Exception) + { + // Never retain a config that failed its telemetry projection as a normal + // snapshot, nor affect serving. Disable rather than emit misleading epochs. + Disable(); + } + } + + public void MarkHostReady() + { + lock (_sync) + { + _hostReady = true; + TryReady(); + } + } + + public void ConfigurationChangeFailed() + { + lock (_sync) + { + if (_enabled) + { + Emit("dab.engine.configuration_change_failed", _configuration.Epoch, + ImmutableDictionary.Empty.Add("failure_category", "configuration")); + } + } + } + + public void StartupFailed(string stage = "initialization") + { + lock (_sync) + { + if (_enabled && !_ready && !_startupFailed) + { + _startupFailed = true; + Emit("dab.engine.startup_failed", _configuration.Epoch, ImmutableDictionary.Empty + .Add("failure_stage", stage is "initialization" or "configuration" or "metadata" or "serving" ? stage : "unknown") + .Add("failure_category", "initialization")); + } + } + } + + public EngineTelemetryRequestScope BeginRequest(EngineTelemetryApi api, EngineTelemetryTransport transport, EngineTelemetryRole role, bool eligible = true) + { + lock (_sync) + { + bool ready = _enabled && _ready; + EngineTelemetryRequestScope scope = new( + this, _request.Value, _configuration, ready ? _config : null, + api, transport, role, eligible: ready && eligible, + started: ready ? _clock.GetTimestamp() : 0); + if (ready) + { + _request.Value = scope; + } + + return scope; + } + } + + public EngineTelemetryMeasurementScope? BeginOperation(string? entityName, EngineTelemetryOperation operation) + { + EngineTelemetryRequestScope? request = ActiveRequest(); + if (request is null) + { + return null; + } + + int depth = _operationDepth.Value; + _operationDepth.Value = depth + 1; + if (depth > 0) + { + return new(_ => { }, () => _operationDepth.Value = depth); + } + + EngineTelemetryProvider provider = EngineTelemetryProvider.Unknown; + EngineTelemetryObject objectType = EngineTelemetryObject.Unknown; + try + { + if (entityName is not null && request.Config!.Entities.TryGetValue(entityName, out Entity? entity)) + { + DataSource source = request.Config.GetDataSourceFromDataSourceName(request.Config.GetDataSourceNameFromEntityName(entityName)); + provider = Provider(source.DatabaseType); + objectType = source.DatabaseType == DatabaseType.CosmosDB_NoSQL ? EngineTelemetryObject.Document : entity.Source.Type switch + { + EntitySourceType.Table => EngineTelemetryObject.Table, + EntitySourceType.View => EngineTelemetryObject.View, + EntitySourceType.StoredProcedure => EngineTelemetryObject.StoredProcedure, + _ => EngineTelemetryObject.Unknown + }; + if (objectType == EngineTelemetryObject.StoredProcedure) + { + operation = EngineTelemetryOperation.Execute; + } + } + } + catch (Exception) + { + // Missing metadata remains unknown; never perform a metadata query for telemetry. + } + + return new(outcome => + { + if (IsEnabled) + { + _aggregator.RecordOperation(request.Configuration, request.Api, operation, provider, objectType, outcome); + } + }, () => _operationDepth.Value = depth); + } + + public EngineTelemetryMeasurementScope? BeginDatabaseAttempt(DatabaseType? provider) + { + EngineTelemetryRequestScope? request = ActiveRequest(); + return request is null ? null : new(outcome => + { + if (IsEnabled) + { + _aggregator.RecordDatabaseAttempt(request.Configuration, + provider is DatabaseType knownProvider ? Provider(knownProvider) : EngineTelemetryProvider.Unknown, outcome); + } + }); + } + + public EngineTelemetryMeasurementScope? BeginEmbedding() + { + EngineTelemetryRequestScope? request = ActiveRequest(); + return request is null ? null : new(outcome => + { + if (IsEnabled) + { + _aggregator.RecordEmbedding(request.Configuration, request.Api, outcome); + } + }); + } + + public void RecordCacheLookup(EngineTelemetryCacheLayer layer, EngineTelemetryCacheResult result) + { + EngineTelemetryRequestScope? request = ActiveRequest(); + if (request is not null) + { + _aggregator.RecordCacheLookup(request.Configuration, layer, result); + } + } + + internal void CompleteRequest(EngineTelemetryRequestScope request, EngineTelemetryOutcome outcome, int? status) + { + lock (_sync) + { + if (!_enabled || !_ready || !request.IsEligible || request.Config is null) + { + return; + } + + TimeSpan duration = _clock.GetElapsedTime(request.Started, _clock.GetTimestamp()); + _aggregator.RecordRequest(request.Configuration, request.Api, request.Transport, request.Role, outcome, duration); + if (request.Transport == EngineTelemetryTransport.Http) + { + _aggregator.RecordHttpOutcome(request.Configuration, request.Api, status); + } + + ImmutableDictionary milestone = ImmutableDictionary.Empty + .Add("api", Wire(request.Api)) + .Add("transport", Wire(request.Transport)) + .Add("outcome", Wire(outcome)) + .Add("since_ready_ms", Milliseconds(_clock.GetElapsedTime(_readyAt, _clock.GetTimestamp()))); + if (!_firstServed) + { + _firstServed = true; + Emit("dab.engine.first_request_served", request.Configuration.Epoch, milestone); + } + + if (!_firstSuccess && outcome == EngineTelemetryOutcome.Success) + { + _firstSuccess = true; + Emit("dab.engine.first_successful_request", request.Configuration.Epoch, milestone); + } + } + } + + internal void RestoreRequest(EngineTelemetryRequestScope scope, EngineTelemetryRequestScope? previous) + { + if (ReferenceEquals(_request.Value, scope)) + { + _request.Value = previous; + } + } + + public void Tick() + { + try + { + lock (_sync) + { + if (!_enabled) + { + return; + } + + EmitSummaries(_aggregator.DrainCompletedWindows()); + long now = _clock.GetTimestamp(); + if (_clock.GetElapsedTime(_lastHeartbeat, now) >= TimeSpan.FromMinutes(10)) + { + _lastHeartbeat = now; + Emit("dab.engine.heartbeat", _configuration.Epoch, ImmutableDictionary.Empty + .Add("run_state", _ready ? "ready" : _startupFailed ? "startup_failed" : "initializing") + .Add("uptime", UptimeBucket(_clock.GetElapsedTime(_started, now)))); + } + } + } + catch (Exception) + { + Disable(); + } + } + + public Task StopAsync(CancellationToken cancellationToken = default) + { + if (cancellationToken.IsCancellationRequested) + { + Disable(); + return Task.CompletedTask; + } + + lock (_sync) + { + if (!_enabled) + { + if (_stopTask is not null && cancellationToken.CanBeCanceled) + { + // Reuse the delivery deadline, but let every host stop caller cancel + // its pending drain instead of silently ignoring later cancellation. + return _delivery!.StopAsync(cancellationToken); + } + + return _stopTask ?? Task.CompletedTask; + } + + _timer?.Dispose(); + _timer = null; + EmitSummaries(_aggregator.Complete()); + Emit("dab.engine.stopped", _configuration.Epoch, ImmutableDictionary.Empty + .Add("reason", "graceful_shutdown") + .Add("uptime", UptimeBucket(_clock.GetElapsedTime(_started, _clock.GetTimestamp())))); + _enabled = false; + _config = null; + _identityResolution = null; + return _stopTask = _delivery?.StopAsync(cancellationToken) ?? Task.CompletedTask; + } + } + + public void Disable() + { + lock (_sync) + { + _enabled = false; + _timer?.Dispose(); + _timer = null; + _aggregator.Disable(); + _delivery?.Disable(); + _config = null; + _identityResolution = null; + _snapshot = ImmutableDictionary.Empty; + } + } + + public void Dispose() => Disable(); + + private EngineTelemetryRequestScope? ActiveRequest() + { + EngineTelemetryRequestScope? request = _request.Value; + if (!IsEnabled || request?.IsEligible != true || request.IsCompleted || request.IsDisposed || request.Config is null) + { + return null; + } + + return request; + } + + private void TryReady() + { + if (!_enabled || _ready || _startupFailed || !_hostReady || _config is null || _configuration.Epoch == 0 + || !(_config.IsRestEnabled || _config.IsGraphQLEnabled || _config.IsMcpEnabled)) + { + return; + } + + _ready = true; + _readyAt = _clock.GetTimestamp(); + Emit("dab.engine.ready", _configuration.Epoch, _snapshot.Add("startup_ms", Milliseconds(_clock.GetElapsedTime(_started, _readyAt)))); + } + + private void EmitSummaries(EngineTelemetryDrain drain) + { + foreach (EngineTelemetryWindow window in drain.Windows) + { + foreach (EngineTelemetrySeries series in window.Series) + { + ImmutableDictionary.Builder data = ImmutableDictionary.CreateBuilder(); + data.Add("window_start", window.Start.ToString("O", CultureInfo.InvariantCulture)); + data.Add("window_end", window.End.ToString("O", CultureInfo.InvariantCulture)); + data.Add("final", window.IsFinal ? "true" : "false"); + data.Add("family", Wire(series.Dimensions.Measurement)); + data.Add("api", Wire(series.Dimensions.Api)); + data.Add("transport", Wire(series.Dimensions.Transport)); + data.Add("role_class", Wire(series.Dimensions.Role)); + data.Add("operation", Wire(series.Dimensions.Operation)); + data.Add("provider", Wire(series.Dimensions.Provider)); + data.Add("object_type", Wire(series.Dimensions.ObjectType)); + data.Add("cache_layer", Wire(series.Dimensions.CacheLayer)); + data.Add("cache_result", Wire(series.Dimensions.CacheResult)); + data.Add("http_status_class", Wire(series.Dimensions.HttpStatusClass)); + data.Add("count", Number(series.Count)); + data.Add("capped", series.IsCapped ? "true" : "false"); + // Cosmos SDK hides internal retries and cache background attribution can be + // unavailable; never present those absent observations as exact zero coverage. + data.Add("database_attempt_coverage", "sql_commands_only"); + data.Add("cache_coverage", "request_context_observed"); + if (series.Outcomes is EngineTelemetryOutcomeCounts outcomes) + { + data.Add("unknown", Number(outcomes.Unknown)); + data.Add("success", Number(outcomes.Success)); + data.Add("failure", Number(outcomes.Failure)); + data.Add("partial_failure", Number(outcomes.PartialFailure)); + data.Add("canceled", Number(outcomes.Canceled)); + } + + if (series.Latency is EngineTelemetryHistogram histogram) + { + data.Add("latency_schema", EngineTelemetryHistogram.BUCKET_SCHEMA); + data.Add("latency_bounds_ms", JsonSerializer.Serialize(EngineTelemetryHistogram.UpperBoundsMilliseconds)); + data.Add("latency_buckets", JsonSerializer.Serialize(histogram.Buckets)); + data.Add("timed_count", Number(histogram.TimedCount)); + data.Add("latency_complete", histogram.IsComplete ? "true" : "false"); + } + + Emit("dab.engine.usage_summary", series.ConfigurationEpoch, data.ToImmutable()); + } + + if (window.SeriesCapacityDrops > 0 || window.CounterCapacityDrops > 0 || window.ClockRegressionDrops > 0) + { + Emit("dab.engine.usage_summary", 0, ImmutableDictionary.Empty.Add("family", "collection_loss") + .Add("window_start", window.Start.ToString("O", CultureInfo.InvariantCulture)) + .Add("window_end", window.End.ToString("O", CultureInfo.InvariantCulture)) + .Add("series_capacity_drops", Number(window.SeriesCapacityDrops)) + .Add("counter_capacity_drops", Number(window.CounterCapacityDrops)) + .Add("clock_regression_drops", Number(window.ClockRegressionDrops)) + .Add("capped", window.LossCountsCapped ? "true" : "false")); + } + } + + if (drain.DroppedWindows > 0) + { + Emit("dab.engine.usage_summary", 0, ImmutableDictionary.Empty.Add("family", "collection_loss") + .Add("dropped_windows", Number(drain.DroppedWindows)) + .Add("dropped_measurements", Number(drain.DroppedMeasurements)) + .Add("capped", drain.LossCountsCapped ? "true" : "false")); + } + } + + private void Emit(string name, long epoch, ImmutableDictionary properties) + { + if (!_enabled || _sequence == long.MaxValue) + { + return; + } + + properties = properties.SetItems(_context); + if (_identity is not null) + { + properties = properties.Add("dab_api_id", _identity.ApiId.ToString("D")) + .Add("dab_api_id_stability", _identity.Stability); + } + + properties = properties.Add("sender_dropped_events", Number(_delivery?.DroppedEvents ?? 0)); + _delivery?.TryEnqueue(new(Guid.NewGuid(), _sessionId, ++_sequence, _clock.GetUtcNow().ToUniversalTime(), epoch, name, properties)); + } + + public static EngineTelemetryRole ClassifyRole(string? role, bool authenticated = false) => role switch + { + null or "" => authenticated ? EngineTelemetryRole.Authenticated : EngineTelemetryRole.Anonymous, + _ when string.Equals(role, "anonymous", StringComparison.OrdinalIgnoreCase) => EngineTelemetryRole.Anonymous, + _ when string.Equals(role, "authenticated", StringComparison.OrdinalIgnoreCase) => EngineTelemetryRole.Authenticated, + _ => EngineTelemetryRole.Custom + }; + + private static EngineTelemetryProvider Provider(DatabaseType type) => type switch + { + DatabaseType.MSSQL => EngineTelemetryProvider.MsSql, + DatabaseType.DWSQL => EngineTelemetryProvider.DwSql, + DatabaseType.PostgreSQL => EngineTelemetryProvider.PostgreSql, + DatabaseType.MySQL => EngineTelemetryProvider.MySql, + DatabaseType.CosmosDB_NoSQL => EngineTelemetryProvider.CosmosDb, + _ => EngineTelemetryProvider.Unknown + }; + + private static void ShowNotice() + { + using StreamWriter writer = new(Console.OpenStandardError(), new System.Text.UTF8Encoding(false), leaveOpen: true); + writer.WriteLine("DAB synthetic product telemetry is enabled for validation: categorical configuration and aggregate usage are sent to the selected test destination. Disable with DAB_TELEMETRY_OPT_OUT=1. Fields: docs/telemetry.md (product repository). No request content is collected."); + } + } + +} diff --git a/src/Core/Telemetry/Product/EngineTelemetryValueFormatter.cs b/src/Core/Telemetry/Product/EngineTelemetryValueFormatter.cs new file mode 100644 index 0000000000..9c9af9d815 --- /dev/null +++ b/src/Core/Telemetry/Product/EngineTelemetryValueFormatter.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Globalization; + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// Schema-v1 value spellings, independent of CLR enum names and current culture. + internal static class EngineTelemetryValueFormatter + { + internal static string Milliseconds(TimeSpan value) + => Math.Max(0, (long)value.TotalMilliseconds).ToString(CultureInfo.InvariantCulture); + + internal static string Number(long value) => value.ToString(CultureInfo.InvariantCulture); + + internal static string UptimeBucket(TimeSpan elapsed) => elapsed.TotalMinutes switch + { + < 1 => "under_1m", + < 60 => "1m_1h", + < 360 => "1h_6h", + < 1440 => "6h_1d", + < 10080 => "1d_7d", + _ => "7d_plus" + }; + + internal static string Wire(T value) where T : struct, Enum => (object)value switch + { + EngineTelemetryApi.Rest => "rest", + EngineTelemetryApi.GraphQL => "graph_ql", + EngineTelemetryApi.Mcp => "mcp", + EngineTelemetryTransport.Http => "http", + EngineTelemetryTransport.Stdio => "stdio", + EngineTelemetryTransport.InProcess => "in_process", + EngineTelemetryRole.Anonymous => "anonymous", + EngineTelemetryRole.Authenticated => "authenticated", + EngineTelemetryRole.Custom => "custom", + EngineTelemetryOperation.Read => "read", + EngineTelemetryOperation.Write => "write", + EngineTelemetryOperation.Execute => "execute", + EngineTelemetryProvider.MsSql => "ms_sql", + EngineTelemetryProvider.DwSql => "dw_sql", + EngineTelemetryProvider.PostgreSql => "postgre_sql", + EngineTelemetryProvider.MySql => "my_sql", + EngineTelemetryProvider.CosmosDb => "cosmos_db", + EngineTelemetryObject.Table => "table", + EngineTelemetryObject.View => "view", + EngineTelemetryObject.StoredProcedure => "stored_procedure", + EngineTelemetryObject.Document => "document", + EngineTelemetryCacheLayer.Level1 => "level1", + EngineTelemetryCacheLayer.Level2 => "level2", + EngineTelemetryCacheResult.Hit => "hit", + EngineTelemetryCacheResult.Miss => "miss", + EngineTelemetryOutcome.Success => "success", + EngineTelemetryOutcome.Failure => "failure", + EngineTelemetryOutcome.PartialFailure => "partial_failure", + EngineTelemetryOutcome.Canceled => "canceled", + EngineTelemetryMeasurement.Request => "request", + EngineTelemetryMeasurement.Operation => "operation", + EngineTelemetryMeasurement.DatabaseAttempt => "database_attempt", + EngineTelemetryMeasurement.CacheLookup => "cache_lookup", + EngineTelemetryMeasurement.Embedding => "embedding", + EngineTelemetryMeasurement.HttpOutcome => "http_outcome", + EngineTelemetryHttpStatus.Informational => "informational", + EngineTelemetryHttpStatus.Success => "success", + EngineTelemetryHttpStatus.Redirect => "redirect", + EngineTelemetryHttpStatus.ClientError => "client_error", + EngineTelemetryHttpStatus.ServerError => "server_error", + _ => "unknown" + }; + } +} diff --git a/src/Core/Telemetry/Product/IEngineTelemetryExporter.cs b/src/Core/Telemetry/Product/IEngineTelemetryExporter.cs new file mode 100644 index 0000000000..7390985b1f --- /dev/null +++ b/src/Core/Telemetry/Product/IEngineTelemetryExporter.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// Owned by one delivery worker. Implementations must honor cancellation before transmission + /// and during I/O. Export and disposal never run on a request's enqueueing thread. Successful + /// acknowledgement does not guarantee later backend storage or exactly-once delivery. + /// + internal interface IEngineTelemetryExporter : IDisposable + { + ValueTask ExportAsync(EngineTelemetryEvent record, CancellationToken cancellationToken); + } +} diff --git a/src/Core/Telemetry/Product/IProductTelemetryControl.cs b/src/Core/Telemetry/Product/IProductTelemetryControl.cs new file mode 100644 index 0000000000..81b2fda1ea --- /dev/null +++ b/src/Core/Telemetry/Product/IProductTelemetryControl.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Azure.DataApiBuilder.Core.Telemetry.Product +{ + /// + /// Host control for DAB-owned product collection. It never enables collection or changes + /// customer diagnostic providers. Resolve from the engine's service provider when needed. + /// + public interface IProductTelemetryControl + { + bool IsEnabled { get; } + + /// + /// Permanently disables collection for this engine instance, cancels delivery and + /// discards unsent data without flushing. Does not erase saved identity or received data. + /// + void Disable(); + } +} diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 56523d0a94..532f644b66 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -6,9 +6,10 @@ - - + + + @@ -37,11 +38,11 @@ - + - - + + diff --git a/src/Service.Tests/Azure.DataApiBuilder.Service.Tests.csproj b/src/Service.Tests/Azure.DataApiBuilder.Service.Tests.csproj index f3f8042fac..5b8f8db941 100644 --- a/src/Service.Tests/Azure.DataApiBuilder.Service.Tests.csproj +++ b/src/Service.Tests/Azure.DataApiBuilder.Service.Tests.csproj @@ -3,6 +3,8 @@ net10.0 false + + $(MSBuildThisFileDirectory)telemetry.runsettings + true + true + + + \ No newline at end of file diff --git a/src/Service/Azure.DataApiBuilder.Service.csproj b/src/Service/Azure.DataApiBuilder.Service.csproj index 4c267eedbc..e25b2f2b68 100644 --- a/src/Service/Azure.DataApiBuilder.Service.csproj +++ b/src/Service/Azure.DataApiBuilder.Service.csproj @@ -63,6 +63,7 @@ + diff --git a/src/Service/Controllers/ConfigurationController.cs b/src/Service/Controllers/ConfigurationController.cs index 4ad8fb40f4..cbfed45bb7 100644 --- a/src/Service/Controllers/ConfigurationController.cs +++ b/src/Service/Controllers/ConfigurationController.cs @@ -39,10 +39,12 @@ public async Task Index([FromBody] ConfigurationPostParametersV2 c return new ConflictResult(); } + bool initializationStarted = false; try { string mergedConfiguration = MergeJsonProvider.Merge(configuration.Configuration, configuration.ConfigurationOverrides); + initializationStarted = true; bool initResult = await _configurationProvider.Initialize( mergedConfiguration, configuration.Schema, @@ -61,6 +63,12 @@ public async Task Index([FromBody] ConfigurationPostParametersV2 c } catch (Exception e) { + if (!initializationStarted) + { + // The provider owns all later failures; malformed merge input never reaches it. + _configurationProvider.ProductTelemetry?.ConfigurationChangeFailed(); + } + _logger.LogError( exception: e, message: "{correlationId} Exception during configuration initialization.", diff --git a/src/Service/Program.cs b/src/Service/Program.cs index 76af52ba97..9108fcedd3 100644 --- a/src/Service/Program.cs +++ b/src/Service/Program.cs @@ -12,6 +12,7 @@ using Azure.DataApiBuilder.Config; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Core.Telemetry; +using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Mcp.Core; using Azure.DataApiBuilder.Mcp.Telemetry; using Azure.DataApiBuilder.Service.Exceptions; @@ -78,23 +79,33 @@ public static void Main(string[] args) Console.InputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); } - if (!ValidateAspNetCoreUrls()) - { - Console.Error.WriteLine("Invalid ASPNETCORE_URLS format. e.g.: ASPNETCORE_URLS=\"http://localhost:5000;https://localhost:5001\""); - Environment.ExitCode = -1; - return; - } - - if (!StartEngine(args, runMcpStdio, mcpRole)) + using EngineTelemetrySession productTelemetry = EngineTelemetryHosting.CreateStandalone(runMcpStdio); + if (!StartEngineCore(args, runMcpStdio, mcpRole, productTelemetry, validateUrls: true)) { Environment.ExitCode = -1; } } public static bool StartEngine(string[] args, bool runMcpStdio, string? mcpRole) + { + using EngineTelemetrySession productTelemetry = EngineTelemetryHosting.CreateStandalone(runMcpStdio); + return StartEngineCore(args, runMcpStdio, mcpRole, productTelemetry, validateUrls: false); + } + + internal static bool StartEngineCore(string[] args, bool runMcpStdio, string? mcpRole, + EngineTelemetrySession productTelemetry, bool validateUrls) { try { + // Main's existing URL preflight belongs to the same bootstrap lifetime as + // other startup failures. Direct StartEngine callers retain their prior path. + if (validateUrls && !ValidateAspNetCoreUrls()) + { + productTelemetry.StartupFailed("configuration"); + Console.Error.WriteLine("Invalid ASPNETCORE_URLS format. e.g.: ASPNETCORE_URLS=\"http://localhost:5000;https://localhost:5001\""); + return false; + } + // Initialize log level EARLY, before building the host. // This ensures logging filters are effective during the entire host build process. // For MCP mode, we also read the config file early to check for log level override. @@ -119,11 +130,17 @@ public static bool StartEngine(string[] args, bool runMcpStdio, string? mcpRole) } } - IHost host = CreateHostBuilder(args, runMcpStdio, mcpRole).Build(); + using IHost host = CreateHostBuilder(args, runMcpStdio, mcpRole, productTelemetry).Build(); if (runMcpStdio) { - return McpStdioHelper.RunMcpStdioHost(host); + bool completed = McpStdioHelper.RunMcpStdioHost(host); + if (!completed) + { + productTelemetry.StartupFailed("metadata"); + } + + return completed; } // Normal web mode @@ -133,6 +150,7 @@ public static bool StartEngine(string[] args, bool runMcpStdio, string? mcpRole) // Catch exception raised by explicit call to IHostApplicationLifetime.StopApplication() catch (TaskCanceledException) { + productTelemetry.StartupFailed(); // Do not log the exception here because exceptions raised during startup // are already automatically written to the console. Console.Error.WriteLine("Unable to launch the Data API builder engine."); @@ -141,9 +159,14 @@ public static bool StartEngine(string[] args, bool runMcpStdio, string? mcpRole) // Catch all remaining unhandled exceptions which may be due to server host operation. catch (Exception ex) { + productTelemetry.StartupFailed(); Console.Error.WriteLine($"Unable to launch the runtime due to: {ex}"); return false; } + finally + { + productTelemetry.StopAsync().GetAwaiter().GetResult(); + } } // Compatibility overload used by external callers that do not pass the runMcpStdio flag. @@ -154,6 +177,9 @@ public static bool StartEngine(string[] args) } public static IHostBuilder CreateHostBuilder(string[] args, bool runMcpStdio, string? mcpRole) + => CreateHostBuilder(args, runMcpStdio, mcpRole, productTelemetry: null); + + internal static IHostBuilder CreateHostBuilder(string[] args, bool runMcpStdio, string? mcpRole, EngineTelemetrySession? productTelemetry) { return Host.CreateDefaultBuilder(args) .ConfigureAppConfiguration(builder => @@ -216,7 +242,7 @@ public static IHostBuilder CreateHostBuilder(string[] args, bool runMcpStdio, st ILoggerFactory loggerFactory = GetLoggerFactoryForLogLevel(Startup.MinimumLogLevel, stdio: runMcpStdio); ILogger startupLogger = loggerFactory.CreateLogger(); DisableHttpsRedirectionIfNeeded(args); - webBuilder.UseStartup(builder => new Startup(builder.Configuration, startupLogger)); + webBuilder.UseStartup(builder => new Startup(builder.Configuration, startupLogger) { ProductTelemetry = productTelemetry }); }); } diff --git a/src/Service/Startup.cs b/src/Service/Startup.cs index b41550bf2e..59f4e0e1d4 100644 --- a/src/Service/Startup.cs +++ b/src/Service/Startup.cs @@ -31,6 +31,7 @@ using Azure.DataApiBuilder.Core.Services.MetadataProviders; using Azure.DataApiBuilder.Core.Services.OpenAPI; using Azure.DataApiBuilder.Core.Telemetry; +using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Mcp.Core; using Azure.DataApiBuilder.Service.Controllers; using Azure.DataApiBuilder.Service.Exceptions; @@ -106,6 +107,7 @@ public class Startup(IConfiguration configuration, ILogger logger) private readonly HotReloadEventHandler _hotReloadEventHandler = new(); private RuntimeConfigProvider? _configProvider; + internal EngineTelemetrySession? ProductTelemetry { get; set; } private ILogger _logger = logger; private LogBuffer _logBuffer = new(); @@ -132,8 +134,24 @@ public void ConfigureServices(IServiceCollection services) null); IFileSystem fileSystem = new FileSystem(); FileSystemRuntimeConfigLoader configLoader = new(fileSystem, _hotReloadEventHandler, configFileName, connectionString); - RuntimeConfigProvider configProvider = new(configLoader); + ProductTelemetry ??= EngineTelemetrySession.Create(); // Embedded/default hosts never opt in through ambient environment alone. + services.AddSingleton(ProductTelemetry); + services.AddSingleton(ProductTelemetry); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + RuntimeConfigProvider configProvider = new(configLoader) { ProductTelemetry = ProductTelemetry }; _configProvider = configProvider; + configLoader.TelemetryReloadCompleted = (acceptedConfig, accepted) => + { + if (accepted && acceptedConfig is not null) + { + ProductTelemetry.AcceptConfiguration(acceptedConfig, "hot_reload", configLoader.ConfigFilePath); + } + else + { + ProductTelemetry?.ConfigurationChangeFailed(); + } + }; services.AddSingleton(fileSystem); services.AddSingleton(sp => configLoader); @@ -216,7 +234,7 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(sp => { AzureLogAnalyticsOptions options = runtimeConfig.Runtime.Telemetry.AzureLogAnalytics; - ManagedIdentityCredential credential = new(); + ManagedIdentityCredential credential = new(ManagedIdentityId.SystemAssigned); LogsIngestionClient logsIngestionClient = new(new Uri(options.Auth!.DceEndpoint!), credential); return new AzureLogAnalyticsFlusherService(options, CustomLogCollector, logsIngestionClient, _logger); }); @@ -396,6 +414,12 @@ public void ConfigureServices(IServiceCollection services) _logger.LogInformation($"Configured HealthCheck HttpClient BaseAddress as: {baseUri}"); client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + EngineTelemetrySession telemetry = serviceProvider.GetRequiredService(); + if (telemetry.IsEnabled && telemetry.HealthProbeToken is string probeToken) + { + client.DefaultRequestHeaders.Add(EngineTelemetryHealthProbe.HEADER_NAME, probeToken); + } + client.Timeout = TimeSpan.FromSeconds(200); }) .ConfigurePrimaryHttpMessageHandler(serviceProvider => @@ -491,7 +515,7 @@ public void ConfigureServices(IServiceCollection services) IHttpClientFactory httpClientFactory = serviceProvider.GetRequiredService(); HttpClient httpClient = httpClientFactory.CreateClient(nameof(EmbeddingService)); - return new EmbeddingService(httpClient, embeddingsOptions, logger, cache); + return new EmbeddingService(httpClient, embeddingsOptions, logger, cache) { ProductTelemetry = ProductTelemetry }; }); _logger.LogInformation( @@ -534,6 +558,13 @@ public void ConfigureServices(IServiceCollection services) { options.FactoryErrorsLogLevel = LogLevel.Debug; options.EventHandlingErrorsLogLevel = LogLevel.Debug; + // Product counters must observe each layer before its request scope closes. + // This only affects deliberately enabled validation runs; handlers do no I/O. + if (ProductTelemetry?.IsEnabled == true) + { + options.EnableSyncEventHandlersExecution = true; + } + string? cachePartition = runtimeConfig?.Runtime?.Cache?.Level2?.Partition; if (string.IsNullOrWhiteSpace(cachePartition) == false) { @@ -706,6 +737,8 @@ private void AddGraphQLService(IServiceCollection services, GraphQLRuntimeOption // See docs/design/HC16-upgrade.md for the full rationale. .ModifyOptions(options => options.LazyInitialization = true) .AddInstrumentation() + .AddDiagnosticEventListener(serviceProvider => new EngineTelemetryGraphQLListener( + serviceProvider.GetRootServiceProvider().GetRequiredService())) .AddType(new DateTimeType(new DateTimeOptions { ValidateInputFormat = !(graphQLRuntimeOptions?.EnableLegacyDateTimeScalar ?? true) })) .AddHttpRequestInterceptor() .ConfigureSchema((serviceProvider, schemaBuilder) => @@ -944,6 +977,8 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, RuntimeC app.UseRouting(); + app.UseMiddleware(); + // Adding CORS Middleware if (runtimeConfig is not null && runtimeConfig.Runtime?.Host?.Cors is not null) { @@ -1046,7 +1081,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, RuntimeC controller.ControllerContext = new ControllerContext { HttpContext = context }; IActionResult result = await controller.PostAsync(embedPath.TrimStart('/')); await result.ExecuteResultAsync(controller.ControllerContext); - }); + }).WithMetadata(EngineTelemetryEmbeddingEndpointMetadata.Instance); } endpoints.MapControllers(); @@ -1489,10 +1524,33 @@ private async Task PerformOnConfigChangeAsync(IApplicationBuilder app) } _logger.LogInformation("Successfully completed runtime initialization."); + if (ProductTelemetry?.IsEnabled == true && !runtimeConfigProvider.IsLateConfigured && + runtimeConfigProvider.TryGetLoadedConfig(out RuntimeConfig? acceptedConfig)) + { + // Metadata initialization can replace the model when it expands autoentities. + // Capture the configuration now used for serving, not the pre-initialization copy. + ProductTelemetry.AcceptConfiguration(acceptedConfig, "startup", + runtimeConfigProvider.ConfigFilePath, onlyIfUnconfigured: true); + } + return true; } catch (Exception ex) { + // RuntimeConfigProvider owns late-configuration failure reporting, including + // parse/merge and post-parse initialization failures, exactly once per attempt. + if (_configProvider?.IsLateConfigured != true) + { + if (ProductTelemetry?.IsReady == true) + { + ProductTelemetry.ConfigurationChangeFailed(); + } + else + { + ProductTelemetry?.StartupFailed("configuration"); + } + } + _logger.LogError(exception: ex, message: "Unable to complete runtime initialization. Refer to exception for error details."); return false; } @@ -1679,6 +1737,10 @@ private void ConfigureEmbeddingsCache( { options.FactoryErrorsLogLevel = LogLevel.Debug; options.EventHandlingErrorsLogLevel = LogLevel.Debug; + if (ProductTelemetry?.IsEnabled == true) + { + options.EnableSyncEventHandlersExecution = true; + } }) .WithDefaultEntryOptions(new FusionCacheEntryOptions { diff --git a/src/Service/Telemetry/ApplicationInsightsEventAttributes.cs b/src/Service/Telemetry/ApplicationInsightsEventAttributes.cs new file mode 100644 index 0000000000..e324a2ed63 --- /dev/null +++ b/src/Service/Telemetry/ApplicationInsightsEventAttributes.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Azure.DataApiBuilder.Core.Telemetry.Product; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// Maps the immutable product contract to the SDK's custom-event attributes. + internal static class ApplicationInsightsEventAttributes + { + internal const string EVENT_NAME_ATTRIBUTE = "microsoft.custom_event.name"; + internal const string CLIENT_IP_ATTRIBUTE = "microsoft.client.ip"; + internal const string SUPPRESSED_IP = "0.0.0.0"; + private const int MAX_PROPERTIES = 128; + private const int MAX_PROPERTY_NAME_LENGTH = 150; + private const int MAX_PROPERTY_VALUE_LENGTH = 8192; + + internal static KeyValuePair[] Create(EngineTelemetryEvent record) + { + ArgumentNullException.ThrowIfNull(record); + if (string.IsNullOrWhiteSpace(record.Name) || record.Name.Length > 512 || record.Properties.Count > MAX_PROPERTIES) + { + throw new ArgumentException("Product telemetry event exceeds the ingestion contract.", nameof(record)); + } + + Dictionary attributes = new(StringComparer.Ordinal); + foreach ((string key, string value) in record.Properties) + { + if (string.IsNullOrEmpty(key) || key.Length > MAX_PROPERTY_NAME_LENGTH || + value is null || value.Length > MAX_PROPERTY_VALUE_LENGTH) + { + throw new ArgumentException("Product telemetry property exceeds the ingestion contract.", nameof(record)); + } + + // Core owns the allowlist. Do not interpret supplied diagnostic attributes as + // SDK instructions or let them replace the event's immutable metadata. + if (key.StartsWith("microsoft.", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("ai.", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("enduser.", StringComparison.OrdinalIgnoreCase) || + key is "user_agent.original" or "{OriginalFormat}" or "CategoryName" or "EventId" or "EventName") + { + continue; + } + + attributes.Add(key, value); + } + + attributes[EVENT_NAME_ATTRIBUTE] = record.Name; + // Missing IP allows ingestion to geolocate the connection before masking it. + // The SDK maps this constant, never an observed address, to ai.location.ip. + attributes[CLIENT_IP_ATTRIBUTE] = SUPPRESSED_IP; + attributes["dab_event_id"] = record.EventId.ToString("D"); + attributes["dab_process_session_id"] = record.SessionId.ToString("D"); + attributes["dab_sequence"] = record.Sequence.ToString(CultureInfo.InvariantCulture); + attributes["dab_occurred_at"] = record.OccurredAt.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); + attributes["dab_config_epoch"] = record.ConfigurationEpoch.ToString(CultureInfo.InvariantCulture); + attributes["dab_schema_version"] = "1"; + attributes["dab_is_synthetic"] = record.IsSynthetic ? "true" : "false"; + return attributes.OrderBy(pair => pair.Key, StringComparer.Ordinal).ToArray(); + } + } +} diff --git a/src/Service/Telemetry/ApplicationInsightsTelemetryDestination.cs b/src/Service/Telemetry/ApplicationInsightsTelemetryDestination.cs new file mode 100644 index 0000000000..7a87dde6a3 --- /dev/null +++ b/src/Service/Telemetry/ApplicationInsightsTelemetryDestination.cs @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// + /// Validated routing for the product's Application Insights instance. Never reads customer + /// configuration or falls back to an ambient SDK destination. Does not contact the service. + /// + internal sealed class ApplicationInsightsTelemetryDestination + { + private const int MAX_CONNECTION_STRING_LENGTH = 4096; + + private ApplicationInsightsTelemetryDestination(Guid instrumentationKey, Uri ingestionEndpoint) + { + InstrumentationKey = instrumentationKey; + TrackEndpoint = new Uri(ingestionEndpoint, "v2.1/track"); + ConnectionString = $"InstrumentationKey={instrumentationKey:D};IngestionEndpoint={ingestionEndpoint.AbsoluteUri}"; + } + + internal Guid InstrumentationKey { get; } + internal Uri TrackEndpoint { get; } + internal string ConnectionString { get; } + + // A record's generated ToString would expose the routing key in diagnostics. + public override string ToString() => "Application Insights product telemetry destination"; + + internal static bool TryParse(string? connectionString, [NotNullWhen(true)] out ApplicationInsightsTelemetryDestination? destination) + { + destination = null; + if (string.IsNullOrWhiteSpace(connectionString) || connectionString.Length > MAX_CONNECTION_STRING_LENGTH || connectionString.Any(char.IsControl)) + { + return false; + } + + Dictionary values = new(StringComparer.OrdinalIgnoreCase); + string[] parts = connectionString.Trim().Split(';'); + for (int i = 0; i < parts.Length; i++) + { + string part = parts[i].Trim(); + if (part.Length == 0 && i == parts.Length - 1) + { + continue; + } + + int separator = part.IndexOf('='); + if (separator <= 0 || separator != part.LastIndexOf('=')) + { + return false; + } + + string key = part[..separator].Trim(); + string value = part[(separator + 1)..].Trim(); + if (value.Length == 0 || !values.TryAdd(key, value) || !IsSupportedSetting(key, value)) + { + return false; + } + } + + if (!values.TryGetValue("InstrumentationKey", out string? keyValue) || + !Guid.TryParseExact(keyValue, "D", out Guid instrumentationKey) || instrumentationKey == Guid.Empty) + { + return false; + } + + values.TryGetValue("EndpointSuffix", out string? suffix); + values.TryGetValue("Location", out string? location); + if ((suffix is not null && !IsAzureEndpointSuffix(suffix)) || + (location is not null && (suffix is null || location.Length > 63 || !location.All(char.IsAsciiLetterOrDigit)))) + { + return false; + } + + if (!values.TryGetValue("IngestionEndpoint", out string? endpoint)) + { + if (suffix is null) + { + return false; + } + + string regionPrefix = location is null ? string.Empty : location + "."; + endpoint = $"https://{regionPrefix}dc.{suffix}/"; + } + + if (!TryParseHttpsOrigin(endpoint, out Uri? ingestionEndpoint)) + { + return false; + } + + destination = new(instrumentationKey, ingestionEndpoint); + return true; + } + + private static bool IsSupportedSetting(string key, string value) => key.ToUpperInvariant() switch + { + "INSTRUMENTATIONKEY" or "INGESTIONENDPOINT" or "ENDPOINTSUFFIX" or "LOCATION" => true, + // These standard fields may appear in a portal connection string but are not used. + "LIVEENDPOINT" => TryParseHttpsOrigin(value, out _), + "APPLICATIONID" => Guid.TryParseExact(value, "D", out Guid id) && id != Guid.Empty, + "AUTHORIZATION" => string.Equals(value, "ikey", StringComparison.OrdinalIgnoreCase), + _ => false // No tokens, credentials, audience overrides or arbitrary extensions. + }; + + private static bool IsAzureEndpointSuffix(string suffix) => suffix.ToLowerInvariant() is + "applicationinsights.azure.com" or "applicationinsights.azure.cn" or "applicationinsights.us"; + + private static bool TryParseHttpsOrigin(string value, [NotNullWhen(true)] out Uri? endpoint) + { + endpoint = null; + // Inspect the original path too: Uri normalization must not turn /a/.. into /. + int schemeEnd = value.IndexOf("://", StringComparison.Ordinal); + int pathStart = schemeEnd < 0 ? -1 : value.IndexOf('/', schemeEnd + 3); + if (schemeEnd < 0 || (pathStart >= 0 && pathStart != value.Length - 1) || + value.Any(char.IsWhiteSpace) || value.IndexOfAny(['\\', '@', '?', '#', '%']) >= 0 || + !Uri.TryCreate(value, UriKind.Absolute, out Uri? uri) || !uri.IsWellFormedOriginalString() || + uri.Scheme != Uri.UriSchemeHttps || string.IsNullOrEmpty(uri.Host) || + uri.HostNameType == UriHostNameType.Unknown || uri.Port <= 0 || + uri.UserInfo.Length != 0 || uri.Query.Length != 0 || uri.Fragment.Length != 0 || uri.AbsolutePath != "/") + { + return false; + } + + endpoint = uri; + return true; + } + } +} diff --git a/src/Service/Telemetry/EngineTelemetryApplicationInsightsExporter.cs b/src/Service/Telemetry/EngineTelemetryApplicationInsightsExporter.cs new file mode 100644 index 0000000000..e30af9d0b8 --- /dev/null +++ b/src/Service/Telemetry/EngineTelemetryApplicationInsightsExporter.cs @@ -0,0 +1,279 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Pipeline; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using Azure.Monitor.OpenTelemetry.Exporter; +using Microsoft.Extensions.Logging; +using OpenTelemetry; +using OpenTelemetry.Logs; +using OpenTelemetry.Resources; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// + /// Uses the supported Azure Monitor custom-event exporter on the delivery worker, never a + /// request thread. DAB owns the only queue and retry loop; SDK offline storage is disabled. + /// + /// + /// Validation only: 1.9.0 shares SDK transmitters by connection string and caches some process + /// settings. Use a dedicated process with SDK statistics disabled before startup. Distinct + /// adapter objects do not guarantee isolation from a foreign exporter with the same routing. + /// Production and public embedded enablement remain off pending that integration review. + /// + internal sealed class EngineTelemetryApplicationInsightsExporter : IEngineTelemetryExporter + { + internal const string STATSBEAT_DISABLED_VARIABLE = "APPLICATIONINSIGHTS_STATSBEAT_DISABLED"; + internal const string SDK_STATS_DISABLED_VARIABLE = "APPLICATIONINSIGHTS_SDKSTATS_DISABLED"; + private static readonly TimeSpan _exportTimeout = TimeSpan.FromSeconds(1); + private readonly AsyncLocal _attempt; + private readonly SynchronousExportProcessor _processor; + private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _logger; + private readonly HttpClientTransport? _transport; + private int _disposed; + + internal EngineTelemetryApplicationInsightsExporter(ApplicationInsightsTelemetryDestination destination) + : this(CreateSdkComponents(destination, null)) + { + } + + /// Test-only HTTP injection. Both SDK statistics opt-outs must still be set. + internal EngineTelemetryApplicationInsightsExporter(ApplicationInsightsTelemetryDestination destination, HttpMessageHandler handler) + : this(CreateSdkComponents(destination, handler ?? throw new ArgumentNullException(nameof(handler)))) + { + } + + /// Unit-test seam with no Azure SDK initialization or network resources. + internal EngineTelemetryApplicationInsightsExporter(BaseExporter exporter) + : this(new Components(exporter ?? throw new ArgumentNullException(nameof(exporter)), new(), null)) + { + } + + private EngineTelemetryApplicationInsightsExporter(Components components) + { + _attempt = components.Attempt; + _transport = components.Transport; + _processor = new(components.Exporter, _attempt); + ILoggerFactory? factory = null; + try + { + // This private factory never registers with the host or reads its configuration. + factory = LoggerFactory.Create(builder => + { + builder.ClearProviders(); + builder.SetMinimumLevel(LogLevel.Information); + builder.AddOpenTelemetry(options => + { + options.IncludeScopes = false; + options.IncludeFormattedMessage = false; + options.ParseStateValues = true; + options.SetResourceBuilder(ResourceBuilder.CreateEmpty()); + options.AddProcessor(_processor); + }); + }); + _logger = factory.CreateLogger(string.Empty); + _loggerFactory = factory; + } + catch + { + try + { + factory?.Dispose(); + } + finally + { + _processor.Dispose(); + _transport?.Dispose(); + } + + throw new InvalidOperationException("Product telemetry logger initialization failed."); + } + } + + // The SDK exposes synchronous Export. Do not add Task.Run or another queue per event: + // the existing delivery worker owns this call and its bounded cancellation/deadline. + public ValueTask ExportAsync(EngineTelemetryEvent record, CancellationToken cancellationToken) + => ValueTask.FromResult(Export(record, cancellationToken)); + + private bool Export(EngineTelemetryEvent record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + if (Volatile.Read(ref _disposed) != 0 || cancellationToken.IsCancellationRequested || !record.IsSynthetic) + { + return false; + } + + EngineTelemetryExportAttempt? previous = _attempt.Value; + try + { + using CancellationTokenSource budget = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + budget.CancelAfter(_exportTimeout); + EngineTelemetryExportAttempt current = new(record, budget.Token); + _attempt.Value = current; + budget.Token.ThrowIfCancellationRequested(); + // Suppress inside OnEnd, not here: suppressing Log would discard the event. + _logger.Log(LogLevel.Information, default, ApplicationInsightsEventAttributes.Create(record), + null, static (_, _) => string.Empty); + return current.Result == ExportResult.Success && !budget.IsCancellationRequested; + } + catch (Exception) + { + // Optional telemetry never forwards SDK exceptions/payloads to customer logging. + return false; + } + finally + { + _attempt.Value = previous; + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + try + { + _loggerFactory.Dispose(); + } + finally + { + _processor.Dispose(); + _transport?.Dispose(); + } + } + + internal static bool AreSdkStatisticsDisabled() => + string.Equals(Environment.GetEnvironmentVariable(STATSBEAT_DISABLED_VARIABLE), "true", StringComparison.OrdinalIgnoreCase) && + string.Equals(Environment.GetEnvironmentVariable(SDK_STATS_DISABLED_VARIABLE), "true", StringComparison.OrdinalIgnoreCase); + + internal static AzureMonitorExporterOptions CreateExporterOptions(string connectionString, HttpClientTransport transport) + { + AzureMonitorExporterOptions options = new() + { + ConnectionString = connectionString, + Transport = transport, + DisableOfflineStorage = true, + EnableLiveMetrics = false, + EnableTraceBasedLogsSampler = false, + EnableStandardMetrics = false, + EnablePerformanceCounters = false + }; + options.Retry.MaxRetries = 0; + options.Retry.NetworkTimeout = _exportTimeout; + options.Diagnostics.IsLoggingEnabled = false; + options.Diagnostics.IsLoggingContentEnabled = false; + options.Diagnostics.IsDistributedTracingEnabled = false; + options.Diagnostics.IsTelemetryEnabled = false; + options.Diagnostics.ApplicationId = null; + options.Diagnostics.LoggedContentSizeLimit = 0; + options.Diagnostics.LoggedHeaderNames.Clear(); + options.Diagnostics.LoggedQueryParameters.Clear(); + return options; + } + + private static Components CreateSdkComponents(ApplicationInsightsTelemetryDestination destination, HttpMessageHandler? handler) + { + ArgumentNullException.ThrowIfNull(destination); + // Temporary validation prerequisite, not a custom-event requirement of Azure Monitor. + // Never mutate process-wide environment/AppContext settings from product code. + if (!AreSdkStatisticsDisabled()) + { + throw new InvalidOperationException("Synthetic product telemetry requires SDK statistics disabled before process startup."); + } + + AsyncLocal attempt = new(); + HttpClient client = new(new EngineTelemetrySdkTransportHandler(handler ?? CreateHttpHandler(), attempt, destination.TrackEndpoint)) + { + Timeout = _exportTimeout + }; + HttpClientTransport transport = new(client); + try + { + return new(new AzureMonitorLogExporter(CreateExporterOptions(destination.ConnectionString, transport)), attempt, transport); + } + catch + { + transport.Dispose(); + throw new InvalidOperationException("Product telemetry SDK initialization failed."); + } + } + + internal static SocketsHttpHandler CreateHttpHandler() => new() + { + AllowAutoRedirect = false, + UseCookies = false, + ActivityHeadersPropagator = null, + ConnectTimeout = _exportTimeout, + PooledConnectionLifetime = TimeSpan.FromMinutes(5) + }; + + private sealed record Components( + BaseExporter Exporter, AsyncLocal Attempt, HttpClientTransport? Transport); + + private sealed class SynchronousExportProcessor( + BaseExporter exporter, AsyncLocal attempt) : BaseProcessor + { + private int _disposed; + + public override void OnEnd(LogRecord data) + { + EngineTelemetryExportAttempt? current = attempt.Value; + if (current is null || current.Token.IsCancellationRequested) + { + return; + } + + data.Timestamp = current.Record.OccurredAt.UtcDateTime; + data.ObservedTimestamp = data.Timestamp; + data.TraceId = default; + data.SpanId = default; + data.TraceFlags = default; + data.TraceState = null; + data.Body = null; + data.FormattedMessage = null; + data.CategoryName = null; + data.EventId = default; + data.Exception = null; + using IDisposable suppression = SuppressInstrumentationScope.Begin(); + Activity? previous = Activity.Current; + try + { + Activity.Current = null; + current.Token.ThrowIfCancellationRequested(); + using Batch batch = new(data); + // BaseExportProcessor propagates a parent resource to the SDK, which can + // infer a hostname even from an empty resource. Direct public Export avoids + // that enrichment and owns no SDK batch queue or customer provider. + current.Result = exporter.Export(in batch); + } + catch (Exception) + { + current.Result = ExportResult.Failure; + } + finally + { + Activity.Current = previous; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing && Interlocked.Exchange(ref _disposed, 1) == 0) + { + exporter.Dispose(); + } + + base.Dispose(disposing); + } + } + } +} diff --git a/src/Service/Telemetry/EngineTelemetryEmbeddingEndpointMetadata.cs b/src/Service/Telemetry/EngineTelemetryEmbeddingEndpointMetadata.cs new file mode 100644 index 0000000000..548308c515 --- /dev/null +++ b/src/Service/Telemetry/EngineTelemetryEmbeddingEndpointMetadata.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// + /// Identifies the actual mapped embedding data endpoint, which has no MVC controller + /// descriptor and need not be under the configured REST entity path. + /// + internal sealed class EngineTelemetryEmbeddingEndpointMetadata + { + internal static EngineTelemetryEmbeddingEndpointMetadata Instance { get; } = new(); + + private EngineTelemetryEmbeddingEndpointMetadata() { } + } +} diff --git a/src/Service/Telemetry/EngineTelemetryExportAttempt.cs b/src/Service/Telemetry/EngineTelemetryExportAttempt.cs new file mode 100644 index 0000000000..bbc63960c1 --- /dev/null +++ b/src/Service/Telemetry/EngineTelemetryExportAttempt.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Threading; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using OpenTelemetry; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// One worker-owned SDK call; neither an event queue nor a process-wide context. + internal sealed class EngineTelemetryExportAttempt(EngineTelemetryEvent record, CancellationToken token) + { + private int _requests; + internal EngineTelemetryEvent Record { get; } = record; + internal CancellationToken Token { get; } = token; + internal ExportResult Result { get; set; } = ExportResult.Failure; + internal bool TryBeginRequest() => Interlocked.Exchange(ref _requests, 1) == 0; + } +} diff --git a/src/Service/Telemetry/EngineTelemetryGraphQLListener.cs b/src/Service/Telemetry/EngineTelemetryGraphQLListener.cs new file mode 100644 index 0000000000..caaf8f0bce --- /dev/null +++ b/src/Service/Telemetry/EngineTelemetryGraphQLListener.cs @@ -0,0 +1,332 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Threading; +using Azure.DataApiBuilder.Core.Authorization; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using HotChocolate; +using HotChocolate.Execution; +using HotChocolate.Execution.Instrumentation; +using HotChocolate.Language; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Primitives; +using GraphQLRequestContext = HotChocolate.Execution.RequestContext; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// + /// One completion per Hot Chocolate execution, including HTTP and variable batches. + /// Register with AddDiagnosticEventListener, independently of customer tracing listeners. + /// + internal sealed class EngineTelemetryGraphQLListener : ExecutionDiagnosticEventListener + { + private const string CONTEXT_KEY = "DAB.ProductTelemetry.GraphQLExecution"; + private readonly EngineTelemetrySession _session; + + public EngineTelemetryGraphQLListener(EngineTelemetrySession session) + { + _session = session; + } + + public override IDisposable ExecuteRequest(GraphQLRequestContext context) + { + if (!_session.IsEnabled || context.IsWarmupRequest()) + { + return EmptyScope; + } + + HttpContext? httpContext = GetHttpContext(context); + if (httpContext is not null && EngineTelemetryHealthProbe.IsProbe(httpContext, _session)) + { + return EmptyScope; + } + + EngineTelemetryRole role; + if (httpContext is not null) + { + role = EngineTelemetryHttpMiddleware.ClassifyRequestRole(httpContext); + } + else + { + context.ContextData.TryGetValue(AuthorizationResolver.CLIENT_ROLE_HEADER, out object? roleValue); + role = EngineTelemetrySession.ClassifyRole(roleValue is StringValues roles && roles.Count == 1 + ? roles[0] + : roleValue as string); + } + + // Deliberately synchronous: setting AsyncLocal inside an async helper would not + // establish it in Hot Chocolate's caller before the resolver tasks are created. + EngineTelemetryRequestScope request = _session.BeginRequest( + EngineTelemetryApi.GraphQL, + httpContext is null ? EngineTelemetryTransport.InProcess : EngineTelemetryTransport.Http, + role, + eligible: false); + request.SetOutcome(EngineTelemetryOutcome.Unknown); + ExecutionScope scope = new(context, request, httpContext); + context.ContextData[CONTEXT_KEY] = scope; + return scope; + } + + public override IDisposable ExecuteOperation(GraphQLRequestContext context) + { + // The parsed document is now available, including on the operation-cache path. + // Mark before resolving fields; BeginRequest already captured the correct epoch. + if (TryGetScope(context, out ExecutionScope? scope)) + { + scope.MarkEligible(); + } + + return EmptyScope; + } + + public override void RequestError(GraphQLRequestContext context, Exception exception) + { + if (TryGetScope(context, out ExecutionScope? scope)) + { + scope.Request.SetOutcome(exception is OperationCanceledException + ? EngineTelemetryOutcome.Canceled + : EngineTelemetryOutcome.Failure); + } + } + + public override void RequestError(GraphQLRequestContext context, IError error) + { + if (TryGetScope(context, out ExecutionScope? scope)) + { + scope.Request.SetOutcome(error.Exception is OperationCanceledException + ? EngineTelemetryOutcome.Canceled + : EngineTelemetryOutcome.Failure); + } + } + + public override void ValidationErrors(GraphQLRequestContext context, IReadOnlyList errors) + { + if (errors.Count > 0 && TryGetScope(context, out ExecutionScope? scope)) + { + scope.Request.SetOutcome(EngineTelemetryOutcome.Failure); + } + } + + internal static EngineTelemetryOutcome ClassifyResult( + IExecutionResult? result, EngineTelemetryOutcome diagnosticOutcome, bool canceled) + { + if (canceled || diagnosticOutcome == EngineTelemetryOutcome.Canceled) + { + return EngineTelemetryOutcome.Canceled; + } + + // HC 16 exposes OperationResult (not the older IOperationResult interface). + // Inspect only error presence and the typed null-data flag, never serialize/read + // JSON, error text, response values or extensions for product telemetry. + if (result is OperationResult operationResult) + { + if (operationResult.Errors.Count > 0) + { + return operationResult.Data is { IsValueNull: false } + ? EngineTelemetryOutcome.PartialFailure + : EngineTelemetryOutcome.Failure; + } + + if (diagnosticOutcome is EngineTelemetryOutcome.Failure or EngineTelemetryOutcome.PartialFailure) + { + return diagnosticOutcome; + } + + return operationResult.HasNext == true || operationResult.Data is null + ? EngineTelemetryOutcome.Unknown + : EngineTelemetryOutcome.Success; + } + + // A response stream has not completed when ExecuteRequest returns. Do not claim + // success from its HTTP status. DAB's ordinary queries/mutations use OperationResult. + return diagnosticOutcome; + } + + internal static bool IsDataOperation(DocumentNode? document, string? operationName) + { + if (document is null) + { + return false; + } + + operationName = string.IsNullOrEmpty(operationName) ? null : operationName; + OperationDefinitionNode? operation = null; + Dictionary fragments = new(StringComparer.Ordinal); + foreach (IDefinitionNode definition in document.Definitions) + { + if (definition is FragmentDefinitionNode fragment) + { + fragments.TryAdd(fragment.Name.Value, fragment); + } + else if (definition is OperationDefinitionNode candidate && + (operationName is null || string.Equals(candidate.Name?.Value, operationName, StringComparison.Ordinal))) + { + if (operation is not null) + { + // No unique selected operation means there was no data execution. + return false; + } + + operation = candidate; + } + } + + if (operation is null) + { + return false; + } + + // Expand root fragments iteratively, with cycle protection even for invalid + // documents. Aliases and an operation called "IntrospectionQuery" do not decide + // eligibility. Nested fields below a data field are not separate requests. + Stack pending = new(); + HashSet visitedFragments = new(StringComparer.Ordinal); + pending.Push(operation.SelectionSet); + while (pending.TryPop(out SelectionSetNode? selections)) + { + foreach (ISelectionNode selection in selections.Selections) + { + switch (selection) + { + case FieldNode field when field.Name.Value is not ("__schema" or "__type" or "__typename"): + return true; + case InlineFragmentNode inlineFragment: + pending.Push(inlineFragment.SelectionSet); + break; + case FragmentSpreadNode spread when visitedFragments.Add(spread.Name.Value) && + fragments.TryGetValue(spread.Name.Value, out FragmentDefinitionNode? fragment): + pending.Push(fragment.SelectionSet); + break; + } + } + } + + return false; + } + + private static bool TryGetScope( + GraphQLRequestContext context, + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out ExecutionScope? scope) + { + context.ContextData.TryGetValue(CONTEXT_KEY, out object? state); + scope = state as ExecutionScope; + return scope is not null; + } + + private static HttpContext? GetHttpContext(GraphQLRequestContext context) + { + if (context.ContextData.TryGetValue(nameof(HttpContext), out object? value) && value is HttpContext httpContext) + { + return httpContext; + } + + return context.RequestServices.GetService()?.HttpContext; + } + + private sealed class ExecutionScope : IDisposable + { + private readonly GraphQLRequestContext _context; + private readonly HttpContext? _httpContext; + private bool _eligible; + private int _disposed; + + internal ExecutionScope(GraphQLRequestContext context, EngineTelemetryRequestScope request, HttpContext? httpContext) + { + _context = context; + Request = request; + _httpContext = httpContext; + } + + internal EngineTelemetryRequestScope Request { get; } + + internal void MarkEligible() + { + // Parse failures may have no document info. Only the parsed AST can establish + // eligibility; the source text or operation name alone cannot do so. + if (!_eligible && IsDataOperation(_context.OperationDocumentInfo?.Document, _context.Request.OperationName)) + { + _eligible = true; + Request.MarkEligible(); + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + try + { + // Validation failures can finish without invoking ExecuteOperation. + MarkEligible(); + if (_eligible) + { + if (_context.Result is OperationResultBatch batch) + { + // Variable batching has one RequestContext, but each result is a + // separate execution. Fork the captured start/configuration rather + // than starting a request in the possibly reloaded current epoch. + // The original scope restores ambient state only; do not count it. + foreach (IExecutionResult result in batch.Results) + { + CompleteResult(Request.ForkForCompletion(), result); + } + } + else + { + CompleteResult(Request, _context.Result); + } + } + } + finally + { + _context.ContextData.Remove(CONTEXT_KEY); + Request.Dispose(); + } + } + + private void CompleteResult(EngineTelemetryRequestScope request, IExecutionResult? result) + { + CancellationToken aborted = _context.RequestAborted; + EngineTelemetryOutcome outcome = ClassifyResult(result, Request.Outcome, aborted.IsCancellationRequested); + request.SetOutcome(outcome); + if (_httpContext is not null) + { + // Capture only the scope and closed outcome: neither the pooled context nor + // the result (including a batch member's buffers) may survive in callbacks. + _ = new EngineTelemetryHttpCompletion( + _httpContext, request.Complete, () => outcome, inferSuccessFromHttp: false); + } + else if (result is IResponseStream stream) + { + // A stream is still running, including one in a variable batch. Its owner + // must dispose it after consumption. Without payload-level observation, + // cleanup preserves Unknown rather than manufacturing success. + stream.RegisterForCleanup(() => RecordCompletion(request, + aborted.IsCancellationRequested ? EngineTelemetryOutcome.Canceled : outcome)); + } + else + { + RecordCompletion(request, outcome); + } + } + + private static void RecordCompletion(EngineTelemetryRequestScope request, EngineTelemetryOutcome outcome) + { + try + { + request.Complete(outcome); + } + catch (Exception) + { + // Product collection must not turn an embedded result into an execution error. + } + } + } + } +} diff --git a/src/Service/Telemetry/EngineTelemetryHealthProbe.cs b/src/Service/Telemetry/EngineTelemetryHealthProbe.cs new file mode 100644 index 0000000000..82eed13e35 --- /dev/null +++ b/src/Service/Telemetry/EngineTelemetryHealthProbe.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// + /// Identifies this engine's internal HTTP health probes without inspecting query contents. + /// The per-session marker is not an authentication credential or a telemetry event field. + /// An arbitrary caller-supplied marker does not suppress usage. + /// + internal static class EngineTelemetryHealthProbe + { + internal const string HEADER_NAME = "X-DAB-Internal-Health-Probe"; + + internal static bool IsProbe(HttpContext context, EngineTelemetrySession session) + { + StringValues values = context.Request.Headers[HEADER_NAME]; + return session.IsEnabled && session.HealthProbeToken is string token && + values.Count == 1 && string.Equals(values[0], token, StringComparison.Ordinal); + } + } +} diff --git a/src/Service/Telemetry/EngineTelemetryHosting.cs b/src/Service/Telemetry/EngineTelemetryHosting.cs new file mode 100644 index 0000000000..bd1f3d8f06 --- /dev/null +++ b/src/Service/Telemetry/EngineTelemetryHosting.cs @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.DataApiBuilder.Config.ObjectModel.Embeddings; +using Azure.DataApiBuilder.Config.Telemetry; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using ZiggyCreatures.Caching.Fusion; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + internal sealed class EngineTelemetryHosting : IHostedService, IDisposable + { + internal const string TEST_MODE_VARIABLE = ProductTelemetryPolicy.TEST_MODE_ENV_VAR; + internal const string CONNECTION_STRING_VARIABLE = "DAB_PRODUCT_TELEMETRY_CONNECTION_STRING"; + private readonly EngineTelemetrySession _session; + private readonly IServiceProvider _services; + private readonly IHostApplicationLifetime _lifetime; + private IDisposable? _started; + private EngineTelemetryCacheObserver? _cache; + private EngineTelemetryCacheObserver? _embeddingCache; + + public EngineTelemetryHosting(EngineTelemetrySession session, IServiceProvider services, IHostApplicationLifetime lifetime) + { + _session = session; + _services = services; + _lifetime = lifetime; + } + + internal static EngineTelemetrySession CreateStandalone(bool stdio) + { + try + { + return CreateStandaloneCore(stdio); + } + catch (Exception) + { + // Environment/SDK configuration must never prevent the engine from starting. + return EngineTelemetrySession.Create(); + } + } + + private static EngineTelemetrySession CreateStandaloneCore(bool stdio) + { + // Setting the destination alone never enables collection. These switches are only + // for deliberately synthetic validation before the production privacy gate opens. + bool testModeEnabled = ProductTelemetryPolicy.IsSyntheticCollectionEnabled( + Environment.GetEnvironmentVariable(TEST_MODE_VARIABLE), + Environment.GetEnvironmentVariable(ProductTelemetryPolicy.OPT_OUT_ENV_VAR)); + if (!testModeEnabled || !EngineTelemetryApplicationInsightsExporter.AreSdkStatisticsDisabled()) + { + return EngineTelemetrySession.Create(); + } + + string? connectionString = Environment.GetEnvironmentVariable(CONNECTION_STRING_VARIABLE); + if (!ApplicationInsightsTelemetryDestination.TryParse(connectionString, out ApplicationInsightsTelemetryDestination? destination)) + { + return EngineTelemetrySession.Create(); + } + + return EngineTelemetrySession.Create(() => new EngineTelemetryApplicationInsightsExporter(destination), + enableSyntheticCollection: true, executionMode: stdio ? "mcp_stdio" : "web"); + } + + public Task StartAsync(CancellationToken cancellationToken) + { + try + { + if (_session.IsEnabled) + { + _started ??= _lifetime.ApplicationStarted.Register(_session.MarkHostReady); + IFusionCache? cache = _services.GetService(); + if (cache is not null) + { + _cache ??= new(cache, _session); + } + + // EmbeddingService uses a distinct named cache only when caching is enabled. + // Its default-cache fallback is already observed above, never subscribed twice. + if (_services.GetService() is { Enabled: true, IsCachingEnabled: true }) + { + IFusionCache? embeddingCache = _services.GetService()?.GetCache("EmbeddingsCache"); + if (embeddingCache is not null && !ReferenceEquals(cache, embeddingCache)) + { + _embeddingCache ??= new(embeddingCache, _session); + } + } + } + } + catch (Exception) + { + // Optional cache observation must never prevent the engine from starting. + _cache?.Dispose(); + _embeddingCache?.Dispose(); + _session.Disable(); + } + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _cache?.Dispose(); + _embeddingCache?.Dispose(); + return _session.StopAsync(cancellationToken); + } + + public void Dispose() + { + _started?.Dispose(); + _cache?.Dispose(); + _embeddingCache?.Dispose(); + // The bootstrap caller owns the externally registered session. Host disposal can + // precede its startup-failure catch; do not discard that final diagnostic here. + } + } +} diff --git a/src/Service/Telemetry/EngineTelemetryHttpCompletion.cs b/src/Service/Telemetry/EngineTelemetryHttpCompletion.cs new file mode 100644 index 0000000000..d585376110 --- /dev/null +++ b/src/Service/Telemetry/EngineTelemetryHttpCompletion.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using Microsoft.AspNetCore.Http; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// + /// The HTTP response owns this small completion state, not an event queue. Cancellation, + /// a failed pipeline and OnCompleted compete for one terminal observation. No execution + /// context is retained by the cancellation registration and no result body is inspected. + /// + internal sealed class EngineTelemetryHttpCompletion + { + private readonly HttpContext _context; + private readonly Action _complete; + private readonly Func _outcome; + private readonly bool _inferSuccessFromHttp; + private CancellationTokenRegistration _abortRegistration; + private int _completed; + + internal EngineTelemetryHttpCompletion( + HttpContext context, + Action complete, + Func outcome, + bool inferSuccessFromHttp) + { + _context = context; + _complete = complete; + _outcome = outcome; + _inferSuccessFromHttp = inferSuccessFromHttp; + // Batch executions can finish concurrently. ASP.NET's response callback collection + // is not a concurrent collection; serialize our registrations on this response. + lock (context) + { + context.Response.OnCompleted(static state => ((EngineTelemetryHttpCompletion)state).ResponseCompletedAsync(), this); + } + + _abortRegistration = context.RequestAborted.UnsafeRegister( + static state => ((EngineTelemetryHttpCompletion)state!).Fail(EngineTelemetryOutcome.Canceled), this); + + // UnsafeRegister can invoke synchronously for an already canceled token. + if (Volatile.Read(ref _completed) != 0) + { + _abortRegistration.Unregister(); + } + } + + internal void Fail(EngineTelemetryOutcome outcome) => Complete(outcome, httpStatusCode: null); + + internal static EngineTelemetryOutcome ClassifyOutcome( + EngineTelemetryOutcome diagnosticOutcome, int statusCode, bool inferSuccessFromHttp) + { + if (diagnosticOutcome is EngineTelemetryOutcome.Canceled or EngineTelemetryOutcome.Failure or EngineTelemetryOutcome.PartialFailure) + { + return diagnosticOutcome; + } + + if (statusCode >= StatusCodes.Status400BadRequest) + { + return EngineTelemetryOutcome.Failure; + } + + if (statusCode >= StatusCodes.Status200OK && statusCode < StatusCodes.Status300MultipleChoices) + { + return inferSuccessFromHttp ? EngineTelemetryOutcome.Success : diagnosticOutcome; + } + + return EngineTelemetryOutcome.Unknown; + } + + private Task ResponseCompletedAsync() + { + if (Volatile.Read(ref _completed) != 0) + { + return Task.CompletedTask; + } + + if (_context.RequestAborted.IsCancellationRequested) + { + Fail(EngineTelemetryOutcome.Canceled); + } + else + { + int statusCode = _context.Response.StatusCode; + Complete(ClassifyOutcome(_outcome(), statusCode, _inferSuccessFromHttp), statusCode); + } + + return Task.CompletedTask; + } + + private void Complete(EngineTelemetryOutcome outcome, int? httpStatusCode) + { + if (Interlocked.Exchange(ref _completed, 1) == 0) + { + _abortRegistration.Unregister(); + try + { + _complete(outcome, httpStatusCode); + } + catch (Exception) + { + // Product telemetry cannot fail the response or enter customer diagnostics. + } + } + } + } +} diff --git a/src/Service/Telemetry/EngineTelemetryHttpMiddleware.cs b/src/Service/Telemetry/EngineTelemetryHttpMiddleware.cs new file mode 100644 index 0000000000..99caa3f88d --- /dev/null +++ b/src/Service/Telemetry/EngineTelemetryHttpMiddleware.cs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading.Tasks; +using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Core.Authorization; +using Azure.DataApiBuilder.Core.Configurations; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using Azure.DataApiBuilder.Service.Controllers; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc.Controllers; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// + /// Counts REST data requests (including embeddings), not controller invocations. Register after routing and before + /// authentication so rejected data requests are included. The response, including result + /// execution and serialization, must finish before a request can be successful. + /// + internal sealed class EngineTelemetryHttpMiddleware + { + private readonly RequestDelegate _next; + private readonly RuntimeConfigProvider _configProvider; + private readonly EngineTelemetrySession _session; + + public EngineTelemetryHttpMiddleware( + RequestDelegate next, + RuntimeConfigProvider configProvider, + EngineTelemetrySession session) + { + _next = next; + _configProvider = configProvider; + _session = session; + } + + public async Task InvokeAsync(HttpContext context) + { + if (!_session.IsEnabled || + EngineTelemetryHealthProbe.IsProbe(context, _session) || + !_configProvider.TryGetLoadedConfig(out RuntimeConfig? config) || + !IsDataRequest(context, config)) + { + await _next(context); + return; + } + + // Begin before doing work: the session captures the accepted configuration epoch. + // Dispose only restores the ambient scope; it does not record a completion. + using EngineTelemetryRequestScope request = _session.BeginRequest( + EngineTelemetryApi.Rest, EngineTelemetryTransport.Http, ClassifyRequestRole(context)); + request.SetOutcome(EngineTelemetryOutcome.Unknown); + EngineTelemetryHttpCompletion completion = new( + context, (outcome, status) => + { + request.SetRole(ClassifyRequestRole(context)); + request.Complete(outcome, status); + }, () => request.Outcome, inferSuccessFromHttp: true); + + try + { + await _next(context); + } + catch (OperationCanceledException) + { + completion.Fail(EngineTelemetryOutcome.Canceled); + throw; + } + catch (Exception) + { + completion.Fail(context.RequestAborted.IsCancellationRequested + ? EngineTelemetryOutcome.Canceled + : EngineTelemetryOutcome.Failure); + throw; + } + } + + internal static bool IsDataRequest(HttpContext context, RuntimeConfig config) + { + // The standalone embedding endpoint is mapped independently of runtime.rest. + // Use routing metadata rather than guessing from the URL or inspecting its body. + if (context.GetEndpoint()?.Metadata.GetMetadata() is not null) + { + return HttpMethods.IsPost(context.Request.Method); + } + + if (!config.IsRestEnabled || + !(HttpMethods.IsGet(context.Request.Method) || HttpMethods.IsPost(context.Request.Method) || + HttpMethods.IsPut(context.Request.Method) || HttpMethods.IsPatch(context.Request.Method) || + HttpMethods.IsDelete(context.Request.Method))) + { + return false; + } + + // Other controllers own health/bootstrap endpoints. Swagger UI runs before this + // middleware; do not guess that similarly named entity routes are static assets. + ControllerActionDescriptor? action = context.GetEndpoint()?.Metadata.GetMetadata(); + if (action is null || !typeof(RestController).IsAssignableFrom(action.ControllerTypeInfo.AsType())) + { + return false; + } + + // Routing has already selected the controller. A valid REST entity can be below + // an MCP prefix without being one of that protocol's actual mapped endpoints. + PathString path = context.Request.Path; + string restPath = config.RestPath.TrimEnd('/'); + PathString remainder = path; + if (restPath.Length > 0 && + !path.StartsWithSegments(restPath, StringComparison.OrdinalIgnoreCase, out remainder)) + { + return false; + } + + // Unlike an arbitrary name such as swagger, openapi is explicitly handled as + // discovery inside the REST catch-all. Favicon requests are UI discovery too. + return remainder.HasValue && remainder.Value != "/" && + !IsPath(remainder, "/openapi") && !IsPath(remainder, "/favicon.ico"); + } + + internal static EngineTelemetryRole ClassifyRequestRole(HttpContext context) + { + bool authenticated = context.User.Identity?.IsAuthenticated == true; + Microsoft.Extensions.Primitives.StringValues role = context.Request.Headers[AuthorizationResolver.CLIENT_ROLE_HEADER]; + if (role.Count > 1) + { + return EngineTelemetryRole.Unknown; + } + + // This adapter runs before authentication. Credentials are not evidence of a + // successfully authenticated identity; do not parse tokens or infer their roles. + if (role.Count == 0 && !authenticated && + (context.Request.Headers.ContainsKey("Authorization") || + context.Request.Headers.ContainsKey("X-MS-CLIENT-PRINCIPAL"))) + { + return EngineTelemetryRole.Unknown; + } + + return EngineTelemetrySession.ClassifyRole(role.Count == 1 ? role[0] : null, authenticated); + } + + private static bool IsPath(PathString path, string prefix) + => !string.IsNullOrEmpty(prefix) && path.StartsWithSegments(prefix, StringComparison.OrdinalIgnoreCase); + } + +} diff --git a/src/Service/Telemetry/EngineTelemetrySdkTransportHandler.cs b/src/Service/Telemetry/EngineTelemetrySdkTransportHandler.cs new file mode 100644 index 0000000000..806e79e592 --- /dev/null +++ b/src/Service/Telemetry/EngineTelemetrySdkTransportHandler.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// + /// Public HttpClient boundary for the SDK: links worker cancellation and rejects unexpected + /// requests/enrichment without rewriting the SDK payload or accessing its private APIs. + /// + internal sealed class EngineTelemetrySdkTransportHandler( + HttpMessageHandler inner, AsyncLocal attempt, Uri trackEndpoint) : DelegatingHandler(inner) + { + private const long MAX_PAYLOAD_BYTES = 64 * 1024; + private const long MAX_RESPONSE_BYTES = 16 * 1024; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + EngineTelemetryExportAttempt? current = attempt.Value; + if (current is null || !current.TryBeginRequest() || request.Method != HttpMethod.Post || + !string.Equals(request.RequestUri?.AbsoluteUri, trackEndpoint.AbsoluteUri, StringComparison.Ordinal)) + { + throw new HttpRequestException("Product telemetry transport rejected a request."); + } + + // AzureMonitorLogExporter.Export is synchronous and passes CancellationToken.None. + // This public boundary carries the worker's token through send AND response-body I/O. + using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(current.Token, cancellationToken); + linked.Token.ThrowIfCancellationRequested(); + if (request.Content is null) + { + throw new HttpRequestException("Product telemetry transport requires a payload."); + } + + await request.Content.LoadIntoBufferAsync(MAX_PAYLOAD_BYTES, linked.Token).ConfigureAwait(false); + if (!HasOnlyApprovedEnvelopeTags(await request.Content.ReadAsStringAsync(linked.Token).ConfigureAwait(false))) + { + throw new HttpRequestException("Product telemetry transport rejected SDK enrichment."); + } + + linked.Token.ThrowIfCancellationRequested(); + HttpResponseMessage response = await base.SendAsync(request, linked.Token).ConfigureAwait(false); + try + { + // This temporary validation transport deliberately permits only the configured + // endpoint. Reject before Azure Monitor's separate redirect policy can replay it. + if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400) + { + throw new HttpRequestException("Product telemetry redirects are disabled during validation."); + } + + await response.Content.LoadIntoBufferAsync(MAX_RESPONSE_BYTES, linked.Token).ConfigureAwait(false); + linked.Token.ThrowIfCancellationRequested(); + return response; + } + catch + { + response.Dispose(); + throw; + } + } + + internal static bool HasOnlyApprovedEnvelopeTags(string payload) + { + try + { + using JsonDocument document = JsonDocument.Parse(payload, new JsonDocumentOptions { MaxDepth = 8 }); + if (document.RootElement.ValueKind != JsonValueKind.Object || + !document.RootElement.TryGetProperty("tags", out JsonElement tags) || tags.ValueKind != JsonValueKind.Object) + { + return false; + } + + bool hasSuppressedIp = false; + foreach (JsonProperty tag in tags.EnumerateObject()) + { + if (tag.Name == "ai.location.ip") + { + if (hasSuppressedIp || tag.Value.ValueKind != JsonValueKind.String || + tag.Value.GetString() != ApplicationInsightsEventAttributes.SUPPRESSED_IP) + { + return false; + } + + hasSuppressedIp = true; + continue; + } + + if (tag.Name == "ai.internal.sdkVersion" && tag.Value.ValueKind == JsonValueKind.String) + { + continue; + } + + if (tag.Name is not ("ai.cloud.role" or "ai.cloud.roleInstance" or "ai.application.ver") || + tag.Value.ValueKind != JsonValueKind.Null) + { + return false; + } + } + + return hasSuppressedIp; + } + catch (JsonException) + { + return false; + } + } + } +} diff --git a/src/Service/Utilities/McpStdioHelper.cs b/src/Service/Utilities/McpStdioHelper.cs index 65307336e3..83fc54f1d3 100644 --- a/src/Service/Utilities/McpStdioHelper.cs +++ b/src/Service/Utilities/McpStdioHelper.cs @@ -6,9 +6,12 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Text; +using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Services.MetadataProviders; +using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Mcp.Core; using Azure.DataApiBuilder.Mcp.Model; +using Azure.DataApiBuilder.Service.Telemetry; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -110,6 +113,15 @@ public static bool RunMcpStdioHost(IHost host) McpToolRegistry.InitializeAndRegisterTools(tools, registry, host.Services); + EngineTelemetrySession? productTelemetry = host.Services.GetService(); + RuntimeConfigProvider? configuration = host.Services.GetService(); + if (productTelemetry is not null && configuration?.TryGetLoadedConfig(out Config.ObjectModel.RuntimeConfig? runtimeConfig) == true) + { + productTelemetry.AcceptConfiguration(runtimeConfig!, "startup", configuration.ConfigFilePath, onlyIfUnconfigured: true); + host.Services.GetService()?.StartAsync(default).GetAwaiter().GetResult(); + productTelemetry.MarkHostReady(); + } + IHostApplicationLifetime lifetime = host.Services.GetRequiredService(); IMcpStdioServer stdio = @@ -119,8 +131,17 @@ public static bool RunMcpStdioHost(IHost host) return true; } - catch (Exception ex) when (ex is not OperationCanceledException) + catch (OperationCanceledException) + { + // Record pre-ready cancellation before finally stops/disables the session. + // StartupFailed is a no-op once ready; normal loop cancellation is not a + // startup failure. Preserve propagation to Program's existing handler. + host.Services.GetService()?.StartupFailed("metadata"); + throw; + } + catch (Exception ex) { + host.Services.GetService()?.StartupFailed("metadata"); // Mirrors Startup.PerformOnConfigChangeAsync: report and return false instead of letting // the exception escape a method whose contract is a bool, and Program.Main turns that // false into ExitCode -1. Cancellation is left to Program.StartEngine's own handler. @@ -148,6 +169,7 @@ public static bool RunMcpStdioHost(IHost host) } finally { + host.Services.GetService()?.StopAsync().GetAwaiter().GetResult(); host.Dispose(); } } From 2bb45c2e5b277aea7524995a798f641b51fc2dc2 Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Thu, 24 Sep 2026 23:19:02 -0700 Subject: [PATCH 2/6] Preserve telemetry failure stages across initialization and reload Use closed failure stages and attempt-scoped attribution for web, stdio, late configuration, validation, and hot reload. Preserve first-failure attribution, prior epochs, handler task semantics, and nested disabled-provider isolation. Add startup, recovery, concurrency, and compatibility regression coverage. --- docs/telemetry.md | 2 + src/Config/DabChangeToken.cs | 18 +- src/Config/FileSystemRuntimeConfigLoader.cs | 13 +- src/Config/RuntimeConfigLoader.cs | 59 ++- .../Telemetry/TelemetryFailureContext.cs | 45 ++ src/Config/Telemetry/TelemetryFailureStage.cs | 19 + .../Configurations/RuntimeConfigProvider.cs | 252 +++++++---- .../Configurations/RuntimeConfigValidator.cs | 98 +++-- .../Product/EngineTelemetrySession.cs | 10 +- .../Product/EngineTelemetryValueFormatter.cs | 7 + .../EngineTelemetryFailureStageTests.cs | 408 ++++++++++++++++++ .../Telemetry/EngineTelemetryReloadTests.cs | 146 ++++++- .../Telemetry/EngineTelemetrySessionTests.cs | 24 +- .../UnitTests/McpStdioHelperTests.cs | 55 +++ .../Controllers/ConfigurationController.cs | 3 +- src/Service/Program.cs | 5 +- src/Service/Startup.cs | 19 +- src/Service/Utilities/McpStdioHelper.cs | 7 +- 18 files changed, 1014 insertions(+), 176 deletions(-) create mode 100644 src/Config/Telemetry/TelemetryFailureContext.cs create mode 100644 src/Config/Telemetry/TelemetryFailureStage.cs create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryFailureStageTests.cs diff --git a/docs/telemetry.md b/docs/telemetry.md index 256f431cb3..d9807033f0 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -75,6 +75,8 @@ Where applicable, `.configured` and `.effective` are separate. States are `enabl Events are `dab.engine.process_started`, `ready`, `startup_failed`, `configuration_changed`, `configuration_change_failed`, `first_request_served`, `first_successful_request`, `heartbeat`, `usage_summary` and `stopped`, all under the `dab.engine.` prefix. +Both failure events include a closed `failure_stage`: `parsing`, `validation`, `metadata`, `serving`, `configuration`, `initialization` or `unknown`. It identifies the failing boundary, not exception text or a customer-supplied value. `failure_category` remains `initialization` for startup failures and `configuration` for rejected changes. Rejections retain the prior telemetry epoch and do not stop collection; the engine's existing configuration acceptance and retry rules are unchanged. Concurrent initialization handlers preserve the first observed failure within that attempt, not a later successful stage. An unclassified handler rejection uses `initialization` rather than guessing a cause. + The mapped embedding HTTP endpoint is included as REST request traffic even when its path is outside the entity REST prefix. Its embedding-service invocations form the separate embedding measurement family; cache-served invocations count without inventing a database attempt. Cache lookups include the dedicated embedding cache when enabled, without double-counting the default-cache fallback. This identification uses endpoint metadata, not request/response contents. Valid REST entity routes are not excluded merely because their names resemble documentation or static assets. Readiness requires accepted usable configuration and host/tool readiness. First-served and first-success are independently once per run. Discovery, health, documentation, introspection-only GraphQL, and MCP protocol-control/metadata traffic are excluded. HTTP 200 with GraphQL errors or an MCP tool error is not logical success. Variable-batch GraphQL results count independently. The current incremental/streaming GraphQL path reports `unknown` rather than inventing success from an unfinished stream. diff --git a/src/Config/DabChangeToken.cs b/src/Config/DabChangeToken.cs index a6ba4755e6..ee1d3e206e 100644 --- a/src/Config/DabChangeToken.cs +++ b/src/Config/DabChangeToken.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Azure.DataApiBuilder.Config.Telemetry; using Microsoft.Extensions.Primitives; namespace Azure.DataApiBuilder.Config; @@ -12,6 +13,8 @@ namespace Azure.DataApiBuilder.Config; public class DabChangeToken : IChangeToken { private CancellationTokenSource _cts = new(); + private static readonly FailureSignal _unobservedSignal = new(null); + private FailureSignal? _failureSignal; /// /// Gets a value that indicates if a change has occurred. @@ -35,12 +38,25 @@ public class DabChangeToken : IChangeToken /// public IDisposable RegisterChangeCallback(Action callback, object? state) { - return _cts.Token.Register(callback, state); + ArgumentNullException.ThrowIfNull(callback); + return _cts.Token.Register(value => + { + // Preserve the registration's normal ExecutionContext, but attribute failures to + // the particular reload that signaled this token, not the registration's attempt. + using IDisposable? failureScope = TelemetryFailureContext.Enter(Volatile.Read(ref _failureSignal)?.Context); + callback(value); + }, state); } public void SignalChange() { + // A token fires once. Freeze the first signal's context even if another caller also + // signals it while callbacks are running. The retained context contains only a stage. + FailureSignal signal = TelemetryFailureContext.Current is { } failure ? new(failure) : _unobservedSignal; + Interlocked.CompareExchange(ref _failureSignal, signal, null); _cts.Cancel(); } + + private sealed record FailureSignal(TelemetryFailureContext? Context); } diff --git a/src/Config/FileSystemRuntimeConfigLoader.cs b/src/Config/FileSystemRuntimeConfigLoader.cs index 3a32c6e9d0..b9f30afa5a 100644 --- a/src/Config/FileSystemRuntimeConfigLoader.cs +++ b/src/Config/FileSystemRuntimeConfigLoader.cs @@ -199,6 +199,8 @@ private bool TrySetupConfigFileWatcher() /// private void OnNewFileContentsDetected(object? sender, EventArgs e) { + TelemetryFailureContext? failure = TelemetryCaptureEnabled?.Invoke() == true ? new() : null; + using IDisposable? failureScope = TelemetryFailureContext.Enter(failure); try { if (RuntimeConfig is not null) @@ -208,7 +210,10 @@ private void OnNewFileContentsDetected(object? sender, EventArgs e) } catch (Exception ex) { - NotifyTelemetryReload(accepted: false); + // Dispatch/validation boundaries record more specific failures first. Otherwise + // the failure occurred while reading or parsing the replacement input. + failure?.RecordFailure(TelemetryFailureStage.Parsing); + NotifyTelemetryReload(accepted: false, failure?.FailureStage ?? TelemetryFailureStage.Unknown); // Need to remove the dependencies in startup on the RuntimeConfigProvider // before we can have an ILogger here. Console.WriteLine("Unable to hot reload configuration file due to " + ex.Message); @@ -375,13 +380,13 @@ private void HotReloadConfig(bool isDevMode, ILogger? logger = null) // Lifecycle observers cannot interrupt loading, expose exception contents or run before // the existing validation/metadata/schema subscribers finish accepting the replacement. - internal Action? TelemetryReloadCompleted { get; set; } + internal Action? TelemetryReloadCompleted { get; set; } - private void NotifyTelemetryReload(bool accepted) + private void NotifyTelemetryReload(bool accepted, TelemetryFailureStage stage = TelemetryFailureStage.Unknown) { try { - TelemetryReloadCompleted?.Invoke(accepted ? RuntimeConfig : null, accepted); + TelemetryReloadCompleted?.Invoke(accepted ? RuntimeConfig : null, accepted, stage); } catch (Exception) { diff --git a/src/Config/RuntimeConfigLoader.cs b/src/Config/RuntimeConfigLoader.cs index 0c8bca7d09..11bf73e6d0 100644 --- a/src/Config/RuntimeConfigLoader.cs +++ b/src/Config/RuntimeConfigLoader.cs @@ -111,32 +111,45 @@ protected virtual void OnConfigChangedEvent(HotReloadEventArgs args) /// protected void SignalConfigChanged(string message = "") { - // Signal that a change has occurred to all change token listeners. - RaiseChanged(); + TelemetryFailureStage stage = TelemetryFailureStage.Validation; + try + { + // Signal that a change has occurred to all change token listeners. + RaiseChanged(); - // All the data inside of the if statement should only update when DAB is in development mode. - if (RuntimeConfig!.IsDevelopmentMode()) + // All the data inside of the if statement should only update when DAB is in development mode. + if (RuntimeConfig!.IsDevelopmentMode()) + { + stage = TelemetryFailureStage.Configuration; + OnConfigChangedEvent(new HotReloadEventArgs(QUERY_MANAGER_FACTORY_ON_CONFIG_CHANGED, message)); + stage = TelemetryFailureStage.Metadata; + OnConfigChangedEvent(new HotReloadEventArgs(METADATA_PROVIDER_FACTORY_ON_CONFIG_CHANGED, message)); + stage = TelemetryFailureStage.Serving; + OnConfigChangedEvent(new HotReloadEventArgs(QUERY_ENGINE_FACTORY_ON_CONFIG_CHANGED, message)); + OnConfigChangedEvent(new HotReloadEventArgs(MUTATION_ENGINE_FACTORY_ON_CONFIG_CHANGED, message)); + OnConfigChangedEvent(new HotReloadEventArgs(DOCUMENTOR_ON_CONFIG_CHANGED, message)); + + // Order of event firing matters: Authorization rules can only be updated after the + // MetadataProviderFactory has been updated with latest database object metadata. + // RuntimeConfig must already be updated and is implied to have been updated by the time + // this function is called. + OnConfigChangedEvent(new HotReloadEventArgs(AUTHZ_RESOLVER_ON_CONFIG_CHANGED, message)); + + // Order of event firing matters: Eviction must be done before creating a new schema and then updating the schema. + OnConfigChangedEvent(new HotReloadEventArgs(GRAPHQL_SCHEMA_EVICTION_ON_CONFIG_CHANGED, message)); + OnConfigChangedEvent(new HotReloadEventArgs(GRAPHQL_SCHEMA_CREATOR_ON_CONFIG_CHANGED, message)); + OnConfigChangedEvent(new HotReloadEventArgs(GRAPHQL_SCHEMA_REFRESH_ON_CONFIG_CHANGED, message)); + } + + // Log Level Initializer is outside of if statement as it can be updated on both development and production mode. + stage = TelemetryFailureStage.Serving; + OnConfigChangedEvent(new HotReloadEventArgs(LOG_LEVEL_INITIALIZER_ON_CONFIG_CHANGE, message)); + } + catch (Exception) { - OnConfigChangedEvent(new HotReloadEventArgs(QUERY_MANAGER_FACTORY_ON_CONFIG_CHANGED, message)); - OnConfigChangedEvent(new HotReloadEventArgs(METADATA_PROVIDER_FACTORY_ON_CONFIG_CHANGED, message)); - OnConfigChangedEvent(new HotReloadEventArgs(QUERY_ENGINE_FACTORY_ON_CONFIG_CHANGED, message)); - OnConfigChangedEvent(new HotReloadEventArgs(MUTATION_ENGINE_FACTORY_ON_CONFIG_CHANGED, message)); - OnConfigChangedEvent(new HotReloadEventArgs(DOCUMENTOR_ON_CONFIG_CHANGED, message)); - - // Order of event firing matters: Authorization rules can only be updated after the - // MetadataProviderFactory has been updated with latest database object metadata. - // RuntimeConfig must already be updated and is implied to have been updated by the time - // this function is called. - OnConfigChangedEvent(new HotReloadEventArgs(AUTHZ_RESOLVER_ON_CONFIG_CHANGED, message)); - - // Order of event firing matters: Eviction must be done before creating a new schema and then updating the schema. - OnConfigChangedEvent(new HotReloadEventArgs(GRAPHQL_SCHEMA_EVICTION_ON_CONFIG_CHANGED, message)); - OnConfigChangedEvent(new HotReloadEventArgs(GRAPHQL_SCHEMA_CREATOR_ON_CONFIG_CHANGED, message)); - OnConfigChangedEvent(new HotReloadEventArgs(GRAPHQL_SCHEMA_REFRESH_ON_CONFIG_CHANGED, message)); + TelemetryFailureContext.Current?.RecordFailure(stage); + throw; } - - // Log Level Initializer is outside of if statement as it can be updated on both development and production mode. - OnConfigChangedEvent(new HotReloadEventArgs(LOG_LEVEL_INITIALIZER_ON_CONFIG_CHANGE, message)); } /// diff --git a/src/Config/Telemetry/TelemetryFailureContext.cs b/src/Config/Telemetry/TelemetryFailureContext.cs new file mode 100644 index 0000000000..db63cd87d4 --- /dev/null +++ b/src/Config/Telemetry/TelemetryFailureContext.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Azure.DataApiBuilder.Config.Telemetry; + +/// +/// One initialization/reload attempt's failure attribution. Nested boundaries record failures, +/// not progress: a later successful handler cannot overwrite a failure. Concurrent handlers +/// use the first observed failure; separate attempts never share this state. +/// +internal sealed class TelemetryFailureContext +{ + private static readonly AsyncLocal _current = new(); + private int _failure = -1; + + internal static TelemetryFailureContext? Current => _current.Value; + + internal bool HasFailure => Volatile.Read(ref _failure) >= 0; + + internal TelemetryFailureStage FailureStage => Volatile.Read(ref _failure) is int value && value >= 0 + ? (TelemetryFailureStage)value : TelemetryFailureStage.Unknown; + + internal void RecordFailure(TelemetryFailureStage stage) + { + TelemetryFailureStage normalized = Enum.IsDefined(stage) ? stage : TelemetryFailureStage.Unknown; + Interlocked.CompareExchange(ref _failure, (int)normalized, -1); + } + + internal static IDisposable? Enter(TelemetryFailureContext? context) + { + TelemetryFailureContext? previous = _current.Value; + if (ReferenceEquals(previous, context)) + { + return null; + } + + _current.Value = context; + return new ContextScope(previous); + } + + private sealed class ContextScope(TelemetryFailureContext? previous) : IDisposable + { + public void Dispose() => _current.Value = previous; + } +} diff --git a/src/Config/Telemetry/TelemetryFailureStage.cs b/src/Config/Telemetry/TelemetryFailureStage.cs new file mode 100644 index 0000000000..ad96fba35a --- /dev/null +++ b/src/Config/Telemetry/TelemetryFailureStage.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Azure.DataApiBuilder.Config.Telemetry; + +/// +/// Closed lifecycle boundaries. These describe where a failure was observed, never its +/// exception type, text, resource name or configuration contents. +/// +internal enum TelemetryFailureStage +{ + Unknown, + Initialization, + Configuration, + Parsing, + Validation, + Metadata, + Serving +} diff --git a/src/Core/Configurations/RuntimeConfigProvider.cs b/src/Core/Configurations/RuntimeConfigProvider.cs index 38af21a309..7fb943c9db 100644 --- a/src/Core/Configurations/RuntimeConfigProvider.cs +++ b/src/Core/Configurations/RuntimeConfigProvider.cs @@ -199,53 +199,76 @@ public Task Initialize( string configuration, string? schema, string? accessToken) - => ProductTelemetry?.IsEnabled == true - ? ObserveInitializationAsync(() => InitializeCoreAsync(configuration, schema, accessToken)) - : InitializeCoreAsync(configuration, schema, accessToken); - - private async Task InitializeCoreAsync(string configuration, string? schema, string? accessToken) { - using IDisposable? capture = TelemetryConfigurationPresence.BeginCapture(_configLoader.TelemetryCaptureEnabled); - if (string.IsNullOrEmpty(configuration)) + if (ProductTelemetry?.IsEnabled == true) { - throw new ArgumentException($"'{nameof(configuration)}' cannot be null or empty.", nameof(configuration)); + return ObserveInitializationAsync(() => InitializeCoreAsync(configuration, schema, accessToken)); } - if (RuntimeConfigLoader.TryParseConfig( - configuration, - out RuntimeConfig? runtimeConfig, - out _, - replacementSettings: null)) - { - _configLoader.RuntimeConfig = runtimeConfig; + // Disabled nested providers must not annotate the caller's enabled attempt. The + // async core captures this masked context, while its caller is restored immediately. + using IDisposable? failureScope = TelemetryFailureContext.Enter(null); + return InitializeCoreAsync(configuration, schema, accessToken); + } - if (string.IsNullOrEmpty(runtimeConfig.DataSource?.ConnectionString)) + private async Task InitializeCoreAsync(string configuration, string? schema, string? accessToken) + { + using IDisposable? capture = TelemetryConfigurationPresence.BeginCapture(_configLoader.TelemetryCaptureEnabled); + TelemetryFailureStage stage = TelemetryFailureStage.Parsing; + try + { + if (string.IsNullOrEmpty(configuration)) { - throw new ArgumentException($"'{nameof(runtimeConfig.DataSource.ConnectionString)}' cannot be null or empty.", nameof(runtimeConfig.DataSource.ConnectionString)); + throw new ArgumentException($"'{nameof(configuration)}' cannot be null or empty.", nameof(configuration)); } - if (runtimeConfig.DataSource.DatabaseType == DatabaseType.CosmosDB_NoSQL) + if (RuntimeConfigLoader.TryParseConfig( + configuration, + out RuntimeConfig? runtimeConfig, + out _, + replacementSettings: null)) { - _configLoader.RuntimeConfig = HandleCosmosNoSqlConfiguration(schema, runtimeConfig, runtimeConfig.DataSource.ConnectionString); - } + stage = TelemetryFailureStage.Validation; + _configLoader.RuntimeConfig = runtimeConfig; - // Hosted / late-config (V2) parses with telemetry injection skipped; embed it into every - // data source's connection string so hosted connection pools carry the usage snapshot. - _configLoader.RuntimeConfig = EmbedTelemetryInDataSourceConnectionStrings(_configLoader.RuntimeConfig, skipDataSourceName: null); + if (string.IsNullOrEmpty(runtimeConfig.DataSource?.ConnectionString)) + { + throw new ArgumentException($"'{nameof(runtimeConfig.DataSource.ConnectionString)}' cannot be null or empty.", nameof(runtimeConfig.DataSource.ConnectionString)); + } - // Flush the telemetry Debug log(s) buffered during embedding. The startup-time flush has - // already run by the time this late-config path executes, so without flushing here the - // buffered telemetry logs would never be emitted. - _configLoader.FlushLogBuffer(); + if (runtimeConfig.DataSource.DatabaseType == DatabaseType.CosmosDB_NoSQL) + { + _configLoader.RuntimeConfig = HandleCosmosNoSqlConfiguration(schema, runtimeConfig, runtimeConfig.DataSource.ConnectionString); + } - ManagedIdentityAccessToken[_configLoader.RuntimeConfig.DefaultDataSourceName] = accessToken; - } + // Hosted / late-config (V2) parses with telemetry injection skipped; embed it into every + // data source's connection string so hosted connection pools carry the usage snapshot. + _configLoader.RuntimeConfig = EmbedTelemetryInDataSourceConnectionStrings(_configLoader.RuntimeConfig, skipDataSourceName: null); + + // Flush the telemetry Debug log(s) buffered during embedding. The startup-time flush has + // already run by the time this late-config path executes, so without flushing here the + // buffered telemetry logs would never be emitted. + _configLoader.FlushLogBuffer(); - bool configLoadSucceeded = await InvokeConfigLoadedHandlersAsync(); + ManagedIdentityAccessToken[_configLoader.RuntimeConfig.DefaultDataSourceName] = accessToken; + } + else + { + TelemetryFailureContext.Current?.RecordFailure(TelemetryFailureStage.Parsing); + } - IsLateConfigured = true; + stage = TelemetryFailureStage.Initialization; + bool configLoadSucceeded = await InvokeConfigLoadedHandlersAsync(); - return configLoadSucceeded; + IsLateConfigured = true; + + return configLoadSucceeded; + } + catch (Exception) + { + TelemetryFailureContext.Current?.RecordFailure(stage); + throw; + } } /// @@ -292,75 +315,97 @@ public Task Initialize( string connectionString, string? accessToken, DeserializationVariableReplacementSettings? replacementSettings) - => ProductTelemetry?.IsEnabled == true - ? ObserveInitializationAsync(() => InitializeCoreAsync(jsonConfig, graphQLSchema, connectionString, accessToken, replacementSettings)) - : InitializeCoreAsync(jsonConfig, graphQLSchema, connectionString, accessToken, replacementSettings); - - private async Task InitializeCoreAsync(string jsonConfig, string? graphQLSchema, string connectionString, - string? accessToken, DeserializationVariableReplacementSettings? replacementSettings) { - using IDisposable? capture = TelemetryConfigurationPresence.BeginCapture(_configLoader.TelemetryCaptureEnabled); - if (string.IsNullOrEmpty(connectionString)) + if (ProductTelemetry?.IsEnabled == true) { - throw new ArgumentException($"'{nameof(connectionString)}' cannot be null or empty.", nameof(connectionString)); + return ObserveInitializationAsync(() => InitializeCoreAsync(jsonConfig, graphQLSchema, connectionString, accessToken, replacementSettings)); } - if (string.IsNullOrEmpty(jsonConfig)) - { - throw new ArgumentException($"'{nameof(jsonConfig)}' cannot be null or empty.", nameof(jsonConfig)); - } - - IsLateConfigured = true; + using IDisposable? failureScope = TelemetryFailureContext.Enter(null); + return InitializeCoreAsync(jsonConfig, graphQLSchema, connectionString, accessToken, replacementSettings); + } - if (RuntimeConfigLoader.TryParseConfig(jsonConfig, out RuntimeConfig? runtimeConfig, out _, replacementSettings)) + private async Task InitializeCoreAsync(string jsonConfig, string? graphQLSchema, string connectionString, + string? accessToken, DeserializationVariableReplacementSettings? replacementSettings) + { + using IDisposable? capture = TelemetryConfigurationPresence.BeginCapture(_configLoader.TelemetryCaptureEnabled); + TelemetryFailureStage stage = TelemetryFailureStage.Validation; + try { - // Late configuration injects a connection string into the parsed config's data source. - // A config with no data source (e.g. a root config that delegates to data-source-files) - // is not meaningful here. Return false to preserve pre-existing behavior — on main, the - // RuntimeConfig constructor threw when DataSource was null and TryParseConfig converted - // that into a 'false' return. Since DataSource is now nullable, we make the same - // determination explicitly rather than NRE'ing in the 'with' expression below. - if (runtimeConfig.DataSource is null) + if (string.IsNullOrEmpty(connectionString)) { - return false; + throw new ArgumentException($"'{nameof(connectionString)}' cannot be null or empty.", nameof(connectionString)); } - _configLoader.RuntimeConfig = runtimeConfig.DataSource.DatabaseType switch + stage = TelemetryFailureStage.Parsing; + if (string.IsNullOrEmpty(jsonConfig)) { - DatabaseType.CosmosDB_NoSQL => HandleCosmosNoSqlConfiguration(graphQLSchema, runtimeConfig, connectionString), - // Embed anonymous usage telemetry into the hosted / late-config connection string's - // Application Name (honoring the opt-out switch and the DAB_APP_NAME_ENV host label). - // Hosted deployments take this path, so it is exactly where the dab_hosted label matters. - _ => runtimeConfig with { DataSource = runtimeConfig.DataSource with { ConnectionString = RuntimeConfigLoader.GetConnectionStringWithApplicationName(connectionString, runtimeConfig, runtimeConfig.DataSource) } } - }; - ManagedIdentityAccessToken[_configLoader.RuntimeConfig.DefaultDataSourceName] = accessToken; - _configLoader.RuntimeConfig.UpdateDataSourceNameToDataSource(_configLoader.RuntimeConfig.DefaultDataSourceName, _configLoader.RuntimeConfig.DataSource!); + throw new ArgumentException($"'{nameof(jsonConfig)}' cannot be null or empty.", nameof(jsonConfig)); + } - // The default data source was supplemented with the separately-supplied connection string - // above. Embed telemetry into any additional (child / multi-database) data sources too, so - // every hosted connection pool carries the usage snapshot. - _configLoader.RuntimeConfig = EmbedTelemetryInDataSourceConnectionStrings(_configLoader.RuntimeConfig, skipDataSourceName: _configLoader.RuntimeConfig.DefaultDataSourceName); + IsLateConfigured = true; - // Flush the telemetry Debug log(s) buffered during embedding. The startup-time flush has - // already run by the time this late-config path executes, so without flushing here the - // buffered telemetry logs would never be emitted. - _configLoader.FlushLogBuffer(); + if (RuntimeConfigLoader.TryParseConfig(jsonConfig, out RuntimeConfig? runtimeConfig, out _, replacementSettings)) + { + stage = TelemetryFailureStage.Validation; + // Late configuration injects a connection string into the parsed config's data source. + // A config with no data source (e.g. a root config that delegates to data-source-files) + // is not meaningful here. Return false to preserve pre-existing behavior — on main, the + // RuntimeConfig constructor threw when DataSource was null and TryParseConfig converted + // that into a 'false' return. Since DataSource is now nullable, we make the same + // determination explicitly rather than NRE'ing in the 'with' expression below. + if (runtimeConfig.DataSource is null) + { + TelemetryFailureContext.Current?.RecordFailure(stage); + return false; + } + + _configLoader.RuntimeConfig = runtimeConfig.DataSource.DatabaseType switch + { + DatabaseType.CosmosDB_NoSQL => HandleCosmosNoSqlConfiguration(graphQLSchema, runtimeConfig, connectionString), + // Embed anonymous usage telemetry into the hosted / late-config connection string's + // Application Name (honoring the opt-out switch and the DAB_APP_NAME_ENV host label). + // Hosted deployments take this path, so it is exactly where the dab_hosted label matters. + _ => runtimeConfig with { DataSource = runtimeConfig.DataSource with { ConnectionString = RuntimeConfigLoader.GetConnectionStringWithApplicationName(connectionString, runtimeConfig, runtimeConfig.DataSource) } } + }; + ManagedIdentityAccessToken[_configLoader.RuntimeConfig.DefaultDataSourceName] = accessToken; + _configLoader.RuntimeConfig.UpdateDataSourceNameToDataSource(_configLoader.RuntimeConfig.DefaultDataSourceName, _configLoader.RuntimeConfig.DataSource!); + + // The default data source was supplemented with the separately-supplied connection string + // above. Embed telemetry into any additional (child / multi-database) data sources too, so + // every hosted connection pool carries the usage snapshot. + _configLoader.RuntimeConfig = EmbedTelemetryInDataSourceConnectionStrings(_configLoader.RuntimeConfig, skipDataSourceName: _configLoader.RuntimeConfig.DefaultDataSourceName); + + // Flush the telemetry Debug log(s) buffered during embedding. The startup-time flush has + // already run by the time this late-config path executes, so without flushing here the + // buffered telemetry logs would never be emitted. + _configLoader.FlushLogBuffer(); + + stage = TelemetryFailureStage.Initialization; + return await InvokeConfigLoadedHandlersAsync(); + } - return await InvokeConfigLoadedHandlersAsync(); + TelemetryFailureContext.Current?.RecordFailure(TelemetryFailureStage.Parsing); + return false; + } + catch (Exception) + { + TelemetryFailureContext.Current?.RecordFailure(stage); + throw; } - - return false; } private async Task ObserveInitializationAsync(Func> initialize) { + TelemetryFailureContext failure = new(); + using IDisposable? failureScope = TelemetryFailureContext.Enter(failure); bool accepted = false; try { bool initialized = await initialize(); // The existing V2 API can report true with no parsed model (no handlers ran). // Match the controller's acceptance condition without changing that public API. - if (initialized && TryGetLoadedConfig(out RuntimeConfig? acceptedConfig)) + if (initialized && !failure.HasFailure && TryGetLoadedConfig(out RuntimeConfig? acceptedConfig)) { // Publish telemetry only after every loaded-config handler has accepted it. // Startup's own success is not sufficient while another handler is pending. @@ -374,7 +419,10 @@ private async Task ObserveInitializationAsync(Func> initialize) { if (!accepted) { - ProductTelemetry?.ConfigurationChangeFailed(); + // Nested boundaries annotate failures before they are converted to false; + // unclassified/custom initialization failures use the closed fallback. + failure.RecordFailure(TelemetryFailureStage.Initialization); + ProductTelemetry?.ConfigurationChangeFailed(failure.FailureStage); } } } @@ -467,19 +515,57 @@ public void ValidateConfig() private async Task InvokeConfigLoadedHandlersAsync() { + TelemetryFailureContext? failure = TelemetryFailureContext.Current; List> configLoadedTasks = new(); + List? failureObservers = failure is null ? null : new(); if (_configLoader.RuntimeConfig is not null) { foreach (RuntimeConfigLoadedHandler configLoadedHandler in RuntimeConfigLoadedHandlers) { - configLoadedTasks.Add(configLoadedHandler(this, _configLoader.RuntimeConfig)); + // Invoke in the existing order before observing the returned task, preserving + // synchronous throws and concurrency between already-started handlers. + Task task; + try + { + task = configLoadedHandler(this, _configLoader.RuntimeConfig); + } + catch (Exception) + { + failure?.RecordFailure(TelemetryFailureStage.Initialization); + throw; + } + + configLoadedTasks.Add(task); + if (failure is not null && task is not null) + { + failureObservers!.Add(task.ContinueWith(static (completed, state) => + { + if (!completed.IsCompletedSuccessfully || !completed.Result) + { + ((TelemetryFailureContext)state!).RecordFailure(TelemetryFailureStage.Initialization); + } + }, failure, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default)); + } } } - bool[] results = await Task.WhenAll(configLoadedTasks); - - // Verify that all tasks succeeded. - return results.All(x => x); + // Preserve eager WhenAll argument validation (including invalid null handler tasks) + // before entering the observer-drain path. + Task joinedHandlers = Task.WhenAll(configLoadedTasks); + try + { + // Join the original tasks: async wrappers can turn faulted cancellation exceptions + // into canceled tasks and change which exception WhenAll exposes to callers. + bool[] results = await joinedHandlers; + return results.All(x => x); + } + finally + { + if (failureObservers is not null) + { + await Task.WhenAll(failureObservers); + } + } } private static RuntimeConfig HandleCosmosNoSqlConfiguration(string? schema, RuntimeConfig runtimeConfig, string connectionString, string dataSourceName = "") diff --git a/src/Core/Configurations/RuntimeConfigValidator.cs b/src/Core/Configurations/RuntimeConfigValidator.cs index 222b15647d..a8b598c60d 100644 --- a/src/Core/Configurations/RuntimeConfigValidator.cs +++ b/src/Core/Configurations/RuntimeConfigValidator.cs @@ -7,6 +7,7 @@ using Azure.DataApiBuilder.Config.DatabasePrimitives; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Config.ObjectModel.Embeddings; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Core.AuthenticationHelpers; using Azure.DataApiBuilder.Core.Authorization; using Azure.DataApiBuilder.Core.Models; @@ -464,54 +465,79 @@ public async Task TryValidateConfig( string configFilePath, ILoggerFactory loggerFactory) { - RuntimeConfig? runtimeConfig; - - if (!_runtimeConfigProvider.TryGetConfig(out runtimeConfig)) + TelemetryFailureStage stage = TelemetryFailureStage.Parsing; + try { - _logger.LogInformation("Failed to parse the config file"); - return false; - } + RuntimeConfig? runtimeConfig; - JsonSchemaValidationResult validationResult = await ValidateConfigSchema(runtimeConfig, configFilePath, loggerFactory); - ValidateConfigProperties(); - ValidatePermissionsInConfig(runtimeConfig); + if (!_runtimeConfigProvider.TryGetConfig(out runtimeConfig)) + { + TelemetryFailureContext.Current?.RecordFailure(stage); + _logger.LogInformation("Failed to parse the config file"); + return false; + } - ValidateRelationshipConfigCorrectness(runtimeConfig); + stage = TelemetryFailureStage.Validation; + JsonSchemaValidationResult validationResult = await ValidateConfigSchema(runtimeConfig, configFilePath, loggerFactory); + ValidateConfigProperties(); + ValidatePermissionsInConfig(runtimeConfig); - // This function initializes the metadata providers which in turn validates the connectivity to the - // database and also validates all the REST and GraphQL paths as well as the permissions of the entities - // that are created from the 'Entities' and 'Autoentities' configuration, including the relationships defined in the config against the database metadata. - // Any exceptions caught during this process are added to the ConfigValidationExceptions list and logged at the end of this function. - await ValidateEntitiesMetadata(runtimeConfig, loggerFactory); + ValidateRelationshipConfigCorrectness(runtimeConfig); + if (!validationResult.IsValid || ConfigValidationExceptions.Count > 0) + { + TelemetryFailureContext.Current?.RecordFailure(stage); + } - // Validate entity configuration (root vs non-root rules, entity counts) after autoentity resolution. - // Only run when there are no connection string errors, since autoentity resolution requires DB access. - if (!ConfigValidationExceptions.Any(x => x.Message.StartsWith(DataApiBuilderException.CONNECTION_STRING_ERROR_MESSAGE))) - { - // Re-read the config since autoentity resolution may have added new entities. - if (_runtimeConfigProvider.TryGetConfig(out RuntimeConfig? updatedConfig) && updatedConfig is not null) + // This function initializes the metadata providers which in turn validates the connectivity to the + // database and also validates all the REST and GraphQL paths as well as the permissions of the entities + // that are created from the 'Entities' and 'Autoentities' configuration, including the relationships defined in the config against the database metadata. + // Any exceptions caught during this process are added to the ConfigValidationExceptions list and logged at the end of this function. + stage = TelemetryFailureStage.Metadata; + int priorErrors = ConfigValidationExceptions.Count; + await ValidateEntitiesMetadata(runtimeConfig, loggerFactory); + if (ConfigValidationExceptions.Count > priorErrors) { - runtimeConfig = updatedConfig; + // Validate-only metadata reports collected errors as well as thrown failures. + // Do not let subsequent successful checks erase the failing boundary. + TelemetryFailureContext.Current?.RecordFailure(stage); } - ValidateDataSourceAndEntityPresence(runtimeConfig); - } + // Validate entity configuration (root vs non-root rules, entity counts) after autoentity resolution. + // Only run when there are no connection string errors, since autoentity resolution requires DB access. + stage = TelemetryFailureStage.Validation; + if (!ConfigValidationExceptions.Any(x => x.Message.StartsWith(DataApiBuilderException.CONNECTION_STRING_ERROR_MESSAGE))) + { + // Re-read the config since autoentity resolution may have added new entities. + if (_runtimeConfigProvider.TryGetConfig(out RuntimeConfig? updatedConfig) && updatedConfig is not null) + { + runtimeConfig = updatedConfig; + } - if (validationResult.IsValid && !ConfigValidationExceptions.Any()) - { - return true; - } - else - { - if (!validationResult.IsValid) + ValidateDataSourceAndEntityPresence(runtimeConfig); + } + + if (validationResult.IsValid && !ConfigValidationExceptions.Any()) { - // log schema validation errors - _logger.LogError(validationResult.ErrorMessage); + return true; } + else + { + TelemetryFailureContext.Current?.RecordFailure(stage); + if (!validationResult.IsValid) + { + // log schema validation errors + _logger.LogError(validationResult.ErrorMessage); + } - // log config validation errors - LogConfigValidationExceptions(); - return false; + // log config validation errors + LogConfigValidationExceptions(); + return false; + } + } + catch (Exception) + { + TelemetryFailureContext.Current?.RecordFailure(stage); + throw; } } diff --git a/src/Core/Telemetry/Product/EngineTelemetrySession.cs b/src/Core/Telemetry/Product/EngineTelemetrySession.cs index e673fa2456..b39444f439 100644 --- a/src/Core/Telemetry/Product/EngineTelemetrySession.cs +++ b/src/Core/Telemetry/Product/EngineTelemetrySession.cs @@ -205,19 +205,21 @@ public void MarkHostReady() } } - public void ConfigurationChangeFailed() + public void ConfigurationChangeFailed(TelemetryFailureStage stage = TelemetryFailureStage.Unknown) { lock (_sync) { if (_enabled) { Emit("dab.engine.configuration_change_failed", _configuration.Epoch, - ImmutableDictionary.Empty.Add("failure_category", "configuration")); + ImmutableDictionary.Empty + .Add("failure_stage", Wire(stage)) + .Add("failure_category", "configuration")); } } } - public void StartupFailed(string stage = "initialization") + public void StartupFailed(TelemetryFailureStage stage = TelemetryFailureStage.Initialization) { lock (_sync) { @@ -225,7 +227,7 @@ public void StartupFailed(string stage = "initialization") { _startupFailed = true; Emit("dab.engine.startup_failed", _configuration.Epoch, ImmutableDictionary.Empty - .Add("failure_stage", stage is "initialization" or "configuration" or "metadata" or "serving" ? stage : "unknown") + .Add("failure_stage", Wire(stage)) .Add("failure_category", "initialization")); } } diff --git a/src/Core/Telemetry/Product/EngineTelemetryValueFormatter.cs b/src/Core/Telemetry/Product/EngineTelemetryValueFormatter.cs index 9c9af9d815..5c576abb5b 100644 --- a/src/Core/Telemetry/Product/EngineTelemetryValueFormatter.cs +++ b/src/Core/Telemetry/Product/EngineTelemetryValueFormatter.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System.Globalization; +using Azure.DataApiBuilder.Config.Telemetry; namespace Azure.DataApiBuilder.Core.Telemetry.Product { @@ -25,6 +26,12 @@ internal static string Milliseconds(TimeSpan value) internal static string Wire(T value) where T : struct, Enum => (object)value switch { + TelemetryFailureStage.Initialization => "initialization", + TelemetryFailureStage.Configuration => "configuration", + TelemetryFailureStage.Parsing => "parsing", + TelemetryFailureStage.Validation => "validation", + TelemetryFailureStage.Metadata => "metadata", + TelemetryFailureStage.Serving => "serving", EngineTelemetryApi.Rest => "rest", EngineTelemetryApi.GraphQL => "graph_ql", EngineTelemetryApi.Mcp => "mcp", diff --git a/src/Service.Tests/Telemetry/EngineTelemetryFailureStageTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryFailureStageTests.cs new file mode 100644 index 0000000000..b3e3a4d3fc --- /dev/null +++ b/src/Service.Tests/Telemetry/EngineTelemetryFailureStageTests.cs @@ -0,0 +1,408 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO.Abstractions.TestingHelpers; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.DataApiBuilder.Config; +using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Telemetry; +using Azure.DataApiBuilder.Core.Configurations; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Azure.DataApiBuilder.Service.Tests.Telemetry; + +[TestClass] +[TestCategory("EngineTelemetry")] +public class EngineTelemetryFailureStageTests +{ + private static readonly TimeSpan _timeout = TimeSpan.FromSeconds(10); + + [DataTestMethod] + [DataRow((int)TelemetryFailureStage.Unknown, "unknown")] + [DataRow((int)TelemetryFailureStage.Initialization, "initialization")] + [DataRow((int)TelemetryFailureStage.Configuration, "configuration")] + [DataRow((int)TelemetryFailureStage.Parsing, "parsing")] + [DataRow((int)TelemetryFailureStage.Validation, "validation")] + [DataRow((int)TelemetryFailureStage.Metadata, "metadata")] + [DataRow((int)TelemetryFailureStage.Serving, "serving")] + [DataRow(int.MaxValue, "unknown")] + [DataRow(-1, "unknown")] + public async Task ConfigurationFailuresAlwaysHaveClosedStageAndCategory(int stage, string expected) + { + Capture exporter = new(); + using EngineTelemetrySession session = CreateSession(exporter); + session.AcceptConfiguration(new(null, new(DatabaseType.MSSQL, string.Empty), new(new Dictionary()))); + session.MarkHostReady(); + session.ConfigurationChangeFailed((TelemetryFailureStage)stage); + Assert.IsTrue(session.IsReady); + await session.StopAsync(); + EngineTelemetryEvent failure = exporter.Events.Single(record => record.Name == "dab.engine.configuration_change_failed"); + Assert.AreEqual(expected, failure.Properties["failure_stage"]); + Assert.AreEqual("configuration", failure.Properties["failure_category"]); + Assert.AreEqual(1L, failure.ConfigurationEpoch); + Assert.IsFalse(failure.Properties.ContainsKey("snapshot_schema")); + } + + [TestMethod] + public void ChangeTokenRejectsNullCallbacksBeforeSignaling() + { + DabChangeToken token = new(); + ArgumentNullException error = Assert.ThrowsException(() => token.RegisterChangeCallback(null!, null)); + Assert.AreEqual("callback", error.ParamName); + token.SignalChange(); + } + + [DataTestMethod] + [DataRow("faulted_cancellation")] + [DataRow("canceled_task")] + [DataRow("synchronous_throw")] + [DataRow("null_task")] + public async Task FailureObservationPreservesOriginalHandlerJoinBehavior(string behavior) + { + (Type? Error, TaskStatus Status) baseline = await ObserveAsync(enabled: false); + (Type? Error, TaskStatus Status) observed = await ObserveAsync(enabled: true); + Assert.AreEqual(baseline, observed, "Telemetry must not change public initialization task status or selected exception."); + + async Task<(Type? Error, TaskStatus Status)> ObserveAsync(bool enabled) + { + Capture exporter = new(); + using EngineTelemetrySession session = EngineTelemetrySession.Create(() => exporter, + enableSyntheticCollection: enabled, readEnvironmentVariable: _ => null, + showNotice: () => { }, startTimer: false); + using FileSystemRuntimeConfigLoader loader = new(new MockFileSystem()); + using RuntimeConfigProvider provider = new(loader) { ProductTelemetry = session }; + provider.RuntimeConfigLoadedHandlers.Add((_, _) => behavior switch + { + "faulted_cancellation" => Task.FromException(new OperationCanceledException()), + "canceled_task" => Task.FromCanceled(new CancellationToken(canceled: true)), + "synchronous_throw" => throw new InvalidOperationException("synthetic synchronous failure"), + "null_task" => null!, + _ => throw new InvalidOperationException() + }); + int laterCalls = 0; + provider.RuntimeConfigLoadedHandlers.Add((_, _) => + { + laterCalls++; + return Task.FromException(new InvalidOperationException("synthetic later failure")); + }); + Task initialization = provider.Initialize("{\"data-source\":{\"database-type\":\"mssql\",\"connection-string\":\"Server=synthetic.invalid\"}}", + schema: null, accessToken: null); + Type? errorType = null; + try + { + await initialization; + } + catch (Exception exception) + { + errorType = exception.GetType(); + } + + Assert.AreEqual(behavior == "synchronous_throw" ? 0 : 1, laterCalls); + Assert.IsNull(TelemetryFailureContext.Current); + await session.StopAsync(); + Assert.AreEqual(enabled ? 1 : 0, exporter.Events.Count(record => record.Name == "dab.engine.configuration_change_failed")); + return (errorType, initialization.Status); + } + } + + [TestMethod] + public async Task V2ParseFailureDoesNotAcceptAPreviouslyRejectedCandidate() + { + Capture exporter = new(); + using EngineTelemetrySession session = CreateSession(exporter); + using FileSystemRuntimeConfigLoader loader = new(new MockFileSystem()); + using RuntimeConfigProvider provider = new(loader) { ProductTelemetry = session }; + bool acceptHandler = false; + provider.RuntimeConfigLoadedHandlers.Add((_, _) => Task.FromResult(acceptHandler)); + const string json = "{\"data-source\":{\"database-type\":\"mssql\",\"connection-string\":\"Server=synthetic.invalid\"}}"; + Assert.IsFalse(await provider.Initialize(json, schema: null, accessToken: null)); + Assert.IsTrue(provider.TryGetLoadedConfig(out _), "Characterize the existing retained candidate, without changing serving behavior."); + acceptHandler = true; + Assert.IsTrue(await provider.Initialize("{", schema: null, accessToken: null), + "Preserve the legacy V2 Boolean even though this attempt did not parse a new candidate."); + session.MarkHostReady(); + Assert.IsFalse(session.IsReady); + await session.StopAsync(); + EngineTelemetryEvent[] failures = exporter.Events.Where(record => record.Name == "dab.engine.configuration_change_failed").ToArray(); + CollectionAssert.AreEqual(new[] { "initialization", "parsing" }, failures.Select(record => record.Properties["failure_stage"]).ToArray()); + Assert.IsTrue(failures.All(record => record.ConfigurationEpoch == 0)); + Assert.IsFalse(exporter.Events.Any(record => record.Name is "dab.engine.ready" or "dab.engine.configuration_changed")); + } + + [TestMethod] + public void ChangeTokenUsesSignalingAttemptWithoutReplacingOtherCapturedContext() + { + AsyncLocal unrelated = new(); + TelemetryFailureContext registration = new(); + TelemetryFailureContext signal = new(); + DabChangeToken token = new(); + IDisposable callback; + using (TelemetryFailureContext.Enter(registration)) + { + unrelated.Value = "registration"; + callback = token.RegisterChangeCallback(_ => + { + Assert.AreSame(signal, TelemetryFailureContext.Current); + Assert.AreEqual("registration", unrelated.Value); + TelemetryFailureContext.Current!.RecordFailure(TelemetryFailureStage.Metadata); + }, null); + } + + using (callback) + using (TelemetryFailureContext.Enter(signal)) + { + unrelated.Value = "signal"; + token.SignalChange(); + Assert.AreSame(signal, TelemetryFailureContext.Current); + Assert.AreEqual("signal", unrelated.Value); + } + + Assert.IsNull(TelemetryFailureContext.Current); + Assert.AreEqual(TelemetryFailureStage.Unknown, registration.FailureStage); + Assert.AreEqual(TelemetryFailureStage.Metadata, signal.FailureStage); + } + + [TestMethod] + public void UnobservedChangeDoesNotReuseRegistrationFailureContext() + { + TelemetryFailureContext registration = new(); + DabChangeToken token = new(); + IDisposable callback; + using (TelemetryFailureContext.Enter(registration)) + { + callback = token.RegisterChangeCallback(_ => Assert.IsNull(TelemetryFailureContext.Current), null); + } + + using (callback) + { + token.SignalChange(); + } + + Assert.AreEqual(TelemetryFailureStage.Unknown, registration.FailureStage); + } + + [TestMethod] + public async Task RepeatedSignalsCannotReplaceTheFirstSignalsFailureContext() + { + TelemetryFailureContext first = new(); + TelemetryFailureContext second = new(); + DabChangeToken token = new(); + TaskCompletionSource entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + using IDisposable last = token.RegisterChangeCallback(_ => + { + Assert.AreSame(first, TelemetryFailureContext.Current); + TelemetryFailureContext.Current!.RecordFailure(TelemetryFailureStage.Validation); + }, null); + using IDisposable blocker = token.RegisterChangeCallback(_ => + { + entered.TrySetResult(); + release.Task.WaitAsync(_timeout).GetAwaiter().GetResult(); + }, null); + Task signaling = Task.Run(() => + { + using IDisposable? scope = TelemetryFailureContext.Enter(first); + token.SignalChange(); + }); + try + { + await entered.Task.WaitAsync(_timeout); + using (TelemetryFailureContext.Enter(second)) + { + token.SignalChange(); + } + } + finally + { + release.TrySetResult(); + await signaling.WaitAsync(_timeout); + } + + Assert.AreEqual(TelemetryFailureStage.Validation, first.FailureStage); + Assert.AreEqual(TelemetryFailureStage.Unknown, second.FailureStage); + } + + [TestMethod] + public async Task ConcurrentLateConfigurationAttemptsKeepTheirOwnFirstFailure() + { + Capture exporter = new(); + using EngineTelemetrySession session = CreateSession(exporter); + using FileSystemRuntimeConfigLoader loader = new(new MockFileSystem()); + using RuntimeConfigProvider provider = new(loader) { ProductTelemetry = session }; + TaskCompletionSource bothEntered = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource firstFailed = new(TaskCreationOptions.RunContinuationsAsynchronously); + int entered = 0; + provider.RuntimeConfigLoadedHandlers.Add(async (_, _) => + { + TelemetryFailureContext context = TelemetryFailureContext.Current!; + int ordinal = Interlocked.Increment(ref entered); + if (ordinal == 2) + { + bothEntered.TrySetResult(); + } + + await bothEntered.Task.WaitAsync(_timeout); + if (ordinal == 1) + { + context.RecordFailure(TelemetryFailureStage.Metadata); + firstFailed.TrySetResult(); + } + else + { + await firstFailed.Task.WaitAsync(_timeout); + context.RecordFailure(TelemetryFailureStage.Validation); + } + + // The generic handler's later fallback must not replace this specific failure. + return false; + }); + const string json = "{\"data-source\":{\"database-type\":\"mssql\",\"connection-string\":\"Server=synthetic.invalid;Integrated Security=true\"},\"entities\":{}}"; + Task first = provider.Initialize(json, schema: null, accessToken: null); + Task second = provider.Initialize(json, schema: null, accessToken: null); + CollectionAssert.AreEqual(new[] { false, false }, await Task.WhenAll(first, second).WaitAsync(_timeout)); + Assert.IsNull(TelemetryFailureContext.Current); + await session.StopAsync(); + EngineTelemetryEvent[] failures = exporter.Events.Where(record => record.Name == "dab.engine.configuration_change_failed").ToArray(); + CollectionAssert.AreEquivalent(new[] { "metadata", "validation" }, failures.Select(record => record.Properties["failure_stage"]).ToArray()); + Assert.IsTrue(failures.All(record => record.ConfigurationEpoch == 0)); + Assert.IsFalse(exporter.Events.Any(record => record.Name == "dab.engine.ready")); + } + + [TestMethod] + public async Task ConcurrentCallbacksCannotReplaceTheFirstRecordedFailure() + { + TelemetryFailureContext context = new(); + context.RecordFailure(TelemetryFailureStage.Metadata); + await Task.WhenAll(Enumerable.Range(0, 32).Select(_ => Task.Run(() => context.RecordFailure(TelemetryFailureStage.Serving)))); + Assert.AreEqual(TelemetryFailureStage.Metadata, context.FailureStage); + } + + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task RealValidatorClassifiesCollectedMetadataErrorsWithoutOverwritingPriorValidation(bool earlierValidationError) + { + MockFileSystem files = new(); + string schemaPath = files.Path.Combine(files.Directory.GetCurrentDirectory(), "synthetic-metadata.graphql"); + files.AddFile(schemaPath, new MockFileData("type Book @model(name: \"Book\") { id: ID! }")); + Entity entity = new(new("books", EntitySourceType.Table, null, null), + new("Book", "Books"), Fields: null, new(Enabled: false), + [new("anonymous", [new(EntityActionOperation.Read, null, null)])], Mappings: null, + Relationships: new() + { + ["related"] = new(Cardinality.One, "Book", [], [], null, [], []) + }); + RuntimeConfig config = new(Schema: RuntimeConfig.DEFAULT_CONFIG_SCHEMA_LINK, + DataSource: new(DatabaseType.CosmosDB_NoSQL, "unused", new() + { + ["database"] = "synthetic", + ["container"] = "books", + ["schema"] = schemaPath + }), + Entities: new(new Dictionary { ["Book"] = entity }), + Runtime: new(new(Enabled: false), new(Path: earlierValidationError ? "graphql" : "/graphql"), Mcp: null, + Host: new(null, null, HostMode.Development))); + using FileSystemRuntimeConfigLoader loader = new(files) { RuntimeConfig = config }; + using RuntimeConfigProvider provider = new(loader); + RuntimeConfigValidator validator = new(provider, files, NullLogger.Instance, isValidateOnly: true); + TelemetryFailureContext failure = new(); + using (TelemetryFailureContext.Enter(failure)) + { + Assert.IsFalse(await validator.TryValidateConfig("synthetic-config.json", NullLoggerFactory.Instance)); + } + + Assert.AreEqual(earlierValidationError ? 3 : 2, validator.ConfigValidationExceptions.Count, + "The two missing inferred relationship objects must be collected, not thrown or skipped."); + Assert.AreEqual(2, validator.ConfigValidationExceptions.Count(exception => exception.Message.StartsWith("Could not infer database object", StringComparison.Ordinal))); + Assert.AreEqual(earlierValidationError ? TelemetryFailureStage.Validation : TelemetryFailureStage.Metadata, failure.FailureStage); + } + + [DataTestMethod] + [DataRow(false, false)] + [DataRow(false, true)] + [DataRow(true, false)] + [DataRow(true, true)] + public async Task NestedDisabledAttemptsDoNotContaminateAnEnabledParent(bool parentV2, bool childV2) + { + const string json = "{\"data-source\":{\"database-type\":\"mssql\",\"connection-string\":\"Server=synthetic.invalid\"}}"; + Capture exporter = new(); + using EngineTelemetrySession session = CreateSession(exporter); + session.MarkHostReady(); + using FileSystemRuntimeConfigLoader loader = new(new MockFileSystem()); + using RuntimeConfigProvider parent = new(loader) { ProductTelemetry = session }; + parent.RuntimeConfigLoadedHandlers.Add(async (_, _) => + { + TelemetryFailureContext? outer = TelemetryFailureContext.Current; + Assert.IsNotNull(outer); + using EngineTelemetrySession disabled = EngineTelemetrySession.Create(); + using FileSystemRuntimeConfigLoader childLoader = new(new MockFileSystem()); + using RuntimeConfigProvider child = new(childLoader) { ProductTelemetry = disabled }; + await Initialize(child, childV2, "{"); + Assert.AreSame(outer, TelemetryFailureContext.Current); + Assert.IsFalse(child.TryGetLoadedConfig(out _)); + child.RuntimeConfigLoadedHandlers.Add(async (_, _) => + { + Assert.IsNull(TelemetryFailureContext.Current); + await Task.Yield(); + Assert.IsNull(TelemetryFailureContext.Current); + return true; + }); + Assert.IsTrue(await Initialize(child, childV2, json)); + Assert.AreSame(outer, TelemetryFailureContext.Current); + return true; + }); + + Assert.IsTrue(await Initialize(parent, parentV2, json)); + Assert.IsTrue(session.IsReady, "A handled failure in a different disabled provider is not this attempt's failure."); + Assert.IsNull(TelemetryFailureContext.Current); + await session.StopAsync(); + Assert.AreEqual(1, exporter.Events.Count(record => record.Name == "dab.engine.ready")); + Assert.IsFalse(exporter.Events.Any(record => record.Name == "dab.engine.configuration_change_failed")); + + static Task Initialize(RuntimeConfigProvider provider, bool versionTwo, string configuration) => versionTwo + ? provider.Initialize(configuration, schema: null, accessToken: null) + : provider.Initialize(configuration, graphQLSchema: null, connectionString: "Server=synthetic.invalid", + accessToken: null, replacementSettings: null); + } + + [TestMethod] + public async Task DisabledInitializationHasNoFailureContext() + { + using EngineTelemetrySession session = EngineTelemetrySession.Create(); + using FileSystemRuntimeConfigLoader loader = new(new MockFileSystem()); + using RuntimeConfigProvider provider = new(loader) { ProductTelemetry = session }; + provider.RuntimeConfigLoadedHandlers.Add((_, _) => + { + Assert.IsNull(TelemetryFailureContext.Current); + return Task.FromResult(false); + }); + Assert.IsFalse(await provider.Initialize("{\"data-source\":{\"database-type\":\"mssql\",\"connection-string\":\"Server=synthetic.invalid\"}}", + schema: null, accessToken: null)); + Assert.IsNull(TelemetryFailureContext.Current); + } + + private static EngineTelemetrySession CreateSession(Capture exporter) => EngineTelemetrySession.Create( + () => exporter, enableSyntheticCollection: true, readEnvironmentVariable: _ => null, + showNotice: () => { }, resolveIdentity: _ => new(Guid.NewGuid(), "ephemeral"), startTimer: false); + + private sealed class Capture : IEngineTelemetryExporter + { + internal ConcurrentQueue Events { get; } = new(); + public ValueTask ExportAsync(EngineTelemetryEvent record, CancellationToken cancellationToken) + { + Events.Enqueue(record); + return ValueTask.FromResult(true); + } + + public void Dispose() { } + } +} diff --git a/src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs index b5021fb103..3731f202ae 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs @@ -15,6 +15,7 @@ using System.Threading.Tasks; using Azure.DataApiBuilder.Config; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Resolvers; using Azure.DataApiBuilder.Core.Resolvers.Factories; @@ -75,6 +76,84 @@ public class EngineTelemetryReloadTests LOG_LEVEL_INITIALIZER_ON_CONFIG_CHANGE ]; + [DataTestMethod] + [DataRow("validation", "validation")] + [DataRow("metadata", "metadata")] + [DataRow("serving", "serving")] + public async Task InitialWebStartupReportsTheActualFailureStage(string boundary, string expectedStage) + { + string directory = Path.Combine(Path.GetTempPath(), "dab-telemetry-stage-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + string path = Path.Combine(directory, "dab-config.json"); + CapturingExporter exporter = new(); + using EngineTelemetrySession session = CreateSession(exporter); + Mock metadata = new(MockBehavior.Strict); + metadata.Setup(factory => factory.InitializeAsync()).Returns(boundary == "metadata" + ? Task.FromException(new InvalidOperationException(SENTINEL)) + : Task.CompletedTask); + IHost? host = null; + try + { + string json = CreateConfigJson(); + if (boundary == "validation") + { + json = json.Replace("\"path\": \"/api\"", "\"path\": \"invalid path\"", StringComparison.Ordinal); + } + + await File.WriteAllTextAsync(path, json); + host = new HostBuilder() + .UseEnvironment(Environments.Production) + .ConfigureAppConfiguration((_, builder) => builder.AddInMemoryCollection(new Dictionary + { + ["ConfigFileName"] = path, + ["CONNSTRING"] = CONNECTION_STRING + })) + .ConfigureLogging(logging => logging.ClearProviders()) + .ConfigureWebHost(web => web + .UseSetting(WebHostDefaults.ApplicationKey, typeof(Startup).Assembly.GetName().Name) + .UseTestServer() + .UseStartup(context => new Startup(context.Configuration, NullLogger.Instance) { ProductTelemetry = session }) + .ConfigureTestServices(services => + { + foreach (ServiceDescriptor descriptor in services.Where(descriptor => + descriptor.ServiceType.IsConstructedGenericType && + descriptor.ServiceType.GetGenericTypeDefinition() == typeof(ILogger<>)).ToArray()) + { + services.Remove(descriptor); + } + + services.Replace(ServiceDescriptor.Singleton(new DynamicLogLevelProvider())); + services.Replace(ServiceDescriptor.Singleton(metadata.Object)); + if (boundary == "serving") + { + services.Replace(ServiceDescriptor.Singleton(_ => throw new InvalidOperationException(SENTINEL))); + } + })).Build(); + using CancellationTokenSource timeout = new(_timeout); + try + { + await host.StartAsync(timeout.Token); + } + catch (OperationCanceledException) when (!timeout.IsCancellationRequested) + { + // Real Startup requests host shutdown after its initialization failure. + } + + metadata.Verify(factory => factory.InitializeAsync(), boundary == "validation" ? Times.Never() : Times.Once()); + Assert.IsFalse(session.IsReady); + EngineTelemetryEvent[] records = await DrainAsync(session, exporter); + EngineTelemetryEvent failure = records.Single(record => record.Name == "dab.engine.startup_failed"); + Assert.AreEqual(expectedStage, failure.Properties["failure_stage"]); + Assert.AreEqual("initialization", failure.Properties["failure_category"]); + Assert.IsFalse(records.Any(record => record.Name == READY || record.Name == CHANGED)); + } + finally + { + host?.Dispose(); + Directory.Delete(directory, recursive: true); + } + } + [TestMethod] public async Task FileReloadAcceptsReplacementOnlyAfterEverySynchronousSubscriberReturns() { @@ -110,13 +189,22 @@ public async Task FileReloadAcceptsReplacementOnlyAfterEverySynchronousSubscribe [DataTestMethod] [DataRow("parse")] + [DataRow("validation")] [DataRow(METADATA_PROVIDER_FACTORY_ON_CONFIG_CHANGED)] [DataRow(LOG_LEVEL_INITIALIZER_ON_CONFIG_CHANGE)] public async Task FileReloadRejectsParseOrSubscriberFailureWithoutAdvancingTelemetryAndCanRecover(string failurePoint) { using ReloadFixture fixture = new(); bool reject = true; - if (failurePoint != "parse") + using IDisposable? validation = failurePoint == "validation" + ? ChangeToken.OnChange(fixture.Loader.GetChangeToken, () => + { + if (reject) + { + throw new InvalidOperationException(SENTINEL); + } + }) : null; + if (failurePoint is not ("parse" or "validation")) { fixture.Handler.Subscribe(failurePoint, (_, _) => { @@ -140,6 +228,10 @@ public async Task FileReloadRejectsParseOrSubscriberFailureWithoutAdvancingTelem Assert.IsTrue(fixture.Loader.IsParseErrorEmitted); Assert.AreSame(fixture.InitialConfig, fixture.Loader.RuntimeConfig); } + else if (failurePoint == "validation") + { + CollectionAssert.AreEqual(new[] { "change_token", "rejected" }, fixture.Trace); + } else { CollectionAssert.AreEqual(new[] { "change_token" } @@ -163,6 +255,11 @@ public async Task FileReloadRejectsParseOrSubscriberFailureWithoutAdvancingTelem EngineTelemetryEvent failure = records.Single(record => record.Name == REJECTED); Assert.AreEqual(1L, failure.ConfigurationEpoch); Assert.AreEqual("configuration", failure.Properties["failure_category"]); + Assert.IsTrue(failure.Properties.ContainsKey("failure_stage")); + Assert.AreEqual(failurePoint == "parse" ? "parsing" : + failurePoint == "validation" ? "validation" : + failurePoint == METADATA_PROVIDER_FACTORY_ON_CONFIG_CHANGED ? "metadata" : "serving", + failure.Properties["failure_stage"]); Assert.IsFalse(failure.Properties.ContainsKey("snapshot_schema")); Assert.AreEqual(2L, records.Single(record => record.Name == CHANGED).ConfigurationEpoch); } @@ -173,11 +270,11 @@ public async Task FileReloadRejectsParseOrSubscriberFailureWithoutAdvancingTelem public async Task ThrowingTelemetryObserverDoesNotEscapeOrTurnAcceptanceIntoRejection(bool invalidJson) { using ReloadFixture fixture = new(); - Action? forward = fixture.Loader.TelemetryReloadCompleted; + Action? forward = fixture.Loader.TelemetryReloadCompleted; Assert.IsNotNull(forward); - fixture.Loader.TelemetryReloadCompleted = (config, accepted) => + fixture.Loader.TelemetryReloadCompleted = (config, accepted, stage) => { - forward(config, accepted); + forward(config, accepted, stage); throw new InvalidOperationException(SENTINEL); }; @@ -218,6 +315,36 @@ public async Task LateConfigurationParseOrMergeRejectionIsReportedExactlyOnce(bo EngineTelemetryEvent[] records = await DrainAsync(session, exporter); CollectionAssert.AreEqual(new[] { PROCESS_STARTED, REJECTED, STOPPED }, records.Select(record => record.Name).ToArray()); Assert.AreEqual(0L, records.Single(record => record.Name == REJECTED).ConfigurationEpoch); + Assert.IsTrue(records.Single(record => record.Name == REJECTED).Properties.ContainsKey("failure_stage")); + Assert.AreEqual("parsing", records.Single(record => record.Name == REJECTED).Properties["failure_stage"]); + } + + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task LateConfigurationValidationFailureIsDistinctAndCanRecover(bool versionTwo) + { + CapturingExporter exporter = new(); + using EngineTelemetrySession session = CreateSession(exporter); + Mock executor = new(MockBehavior.Strict); + await using LateConfigServer server = CreateLateConfigServer(session, CreateQueryManager(executor).Object); + RuntimeConfigProvider provider = AssertUnconfiguredHost(server.Server, session); + string invalid = CreateConfigJson().Replace("\"path\": \"/api\"", "\"path\": \"invalid path\"", StringComparison.Ordinal); + bool accepted = versionTwo + ? await provider.Initialize(invalid, schema: null, accessToken: null) + : await provider.Initialize(invalid, graphQLSchema: null, connectionString: CONNECTION_STRING, accessToken: null, replacementSettings: null); + Assert.IsFalse(accepted); + Assert.IsFalse(session.IsReady); + AssertConfiguration(session, config: null, epoch: 0); + Assert.IsTrue(await InitializeAsync(provider, versionTwo)); + Assert.IsTrue(session.IsReady); + EngineTelemetryEvent[] records = await DrainAsync(session, exporter); + EngineTelemetryEvent rejection = records.Single(record => record.Name == REJECTED); + Assert.AreEqual("validation", rejection.Properties["failure_stage"]); + Assert.AreEqual("configuration", rejection.Properties["failure_category"]); + Assert.AreEqual(0L, rejection.ConfigurationEpoch); + Assert.AreEqual(1L, records.Single(record => record.Name == READY).ConfigurationEpoch); + Assert.IsFalse(records.Any(record => record.Name == "dab.engine.startup_failed")); } [DataTestMethod] @@ -259,6 +386,10 @@ public async Task LateConfigurationWaitsForAllHandlersBeforeTelemetryAcceptance( CollectionAssert.AreEqual(new[] { PROCESS_STARTED, accepted ? READY : REJECTED, STOPPED }, records.Select(record => record.Name).ToArray()); Assert.AreEqual(accepted ? 1L : 0L, records[1].ConfigurationEpoch); + if (!accepted) + { + Assert.AreEqual("initialization", records[1].Properties["failure_stage"], "Custom handler rejection has no more specific known boundary."); + } } [DataTestMethod] @@ -355,6 +486,8 @@ public async Task StartupAwaitsAndRejectsLateMetadataFailureWithoutReportingTerm EngineTelemetryEvent rejected = records.Single(record => record.Name == REJECTED); Assert.AreEqual(0L, rejected.ConfigurationEpoch); Assert.AreEqual("configuration", rejected.Properties["failure_category"]); + Assert.IsTrue(rejected.Properties.ContainsKey("failure_stage")); + Assert.AreEqual("metadata", rejected.Properties["failure_stage"]); Assert.IsFalse(rejected.Properties.ContainsKey("dab_api_id")); Assert.IsFalse(rejected.Properties.ContainsKey("snapshot_schema")); } @@ -532,6 +665,7 @@ public ReloadFixture() Assert.IsNotNull(initial); InitialConfig = initial; Session = CreateSession(Exporter); + Loader.TelemetryCaptureEnabled = () => Session.IsEnabled; Session.AcceptConfiguration(initial); Session.MarkHostReady(); Assert.IsTrue(Session.IsReady); @@ -544,7 +678,7 @@ public ReloadFixture() // Mirror Startup's forwarding delegate explicitly. This verifies the loader's // real notification boundary, not Startup.ConfigureServices' delegate wiring. // Capture observations for assertions outside Notify's exception-swallowing guard. - Loader.TelemetryReloadCompleted = (config, accepted) => + Loader.TelemetryReloadCompleted = (config, accepted, stage) => { Completions.Add((config, accepted)); Trace.Add(accepted ? "accepted" : "rejected"); @@ -554,7 +688,7 @@ public ReloadFixture() } else { - Session.ConfigurationChangeFailed(); + Session.ConfigurationChangeFailed(stage); } }; } diff --git a/src/Service.Tests/Telemetry/EngineTelemetrySessionTests.cs b/src/Service.Tests/Telemetry/EngineTelemetrySessionTests.cs index 565e10e162..41a36b40f7 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetrySessionTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetrySessionTests.cs @@ -13,6 +13,7 @@ using System.Threading; using System.Threading.Tasks; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Core.Telemetry.Product; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -243,7 +244,7 @@ public async Task ReadinessRequiresBothHostAndAcceptedConfigurationAndIsEmittedO Assert.IsTrue(session.IsReady); session.MarkHostReady(); session.AcceptConfiguration(config); - session.StartupFailed("metadata"); + session.StartupFailed(TelemetryFailureStage.Metadata); EngineTelemetryEvent[] records = await DrainAsync(session, exporter); CollectionAssert.AreEqual(new[] { PROCESS_STARTED, READY, STOPPED }, records.Select(record => record.Name).ToArray()); @@ -363,18 +364,21 @@ public async Task RejectedConfigurationReportsFailureAgainstCurrentEpochWithoutA } [DataTestMethod] - [DataRow("initialization", "initialization")] - [DataRow("configuration", "configuration")] - [DataRow("metadata", "metadata")] - [DataRow("serving", "serving")] - [DataRow(SENTINEL, "unknown")] - public async Task StartupFailureIsOnceSanitizedAndPreventsReadiness(string stage, string expectedStage) + [DataRow((int)TelemetryFailureStage.Initialization, "initialization")] + [DataRow((int)TelemetryFailureStage.Configuration, "configuration")] + [DataRow((int)TelemetryFailureStage.Parsing, "parsing")] + [DataRow((int)TelemetryFailureStage.Validation, "validation")] + [DataRow((int)TelemetryFailureStage.Metadata, "metadata")] + [DataRow((int)TelemetryFailureStage.Serving, "serving")] + [DataRow(-1, "unknown")] + [DataRow(int.MaxValue, "unknown")] + public async Task StartupFailureIsOnceSanitizedAndPreventsReadiness(int stage, string expectedStage) { ManualTimeProvider clock = new(); CapturingExporter exporter = new(); using EngineTelemetrySession session = CreateSession(exporter, clock); - session.StartupFailed(stage); - session.StartupFailed("serving"); + session.StartupFailed((TelemetryFailureStage)stage); + session.StartupFailed(TelemetryFailureStage.Serving); session.MarkHostReady(); session.AcceptConfiguration(CreateConfig()); Assert.IsFalse(session.IsReady); @@ -1428,7 +1432,7 @@ private static async Task ExerciseDisabledSessionAsync(EngineTelemetrySession se session.AcceptConfiguration(CreateConfig()); session.MarkHostReady(); session.ConfigurationChangeFailed(); - session.StartupFailed(SENTINEL); + session.StartupFailed((TelemetryFailureStage)int.MaxValue); session.Tick(); using EngineTelemetryRequestScope request = session.BeginRequest(EngineTelemetryApi.Rest, EngineTelemetryTransport.Http, EngineTelemetryRole.Anonymous); request.MarkEligible(); diff --git a/src/Service.Tests/UnitTests/McpStdioHelperTests.cs b/src/Service.Tests/UnitTests/McpStdioHelperTests.cs index 3963369611..171261c8a0 100644 --- a/src/Service.Tests/UnitTests/McpStdioHelperTests.cs +++ b/src/Service.Tests/UnitTests/McpStdioHelperTests.cs @@ -7,16 +7,20 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions.TestingHelpers; using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; +using Azure.DataApiBuilder.Config; using Azure.DataApiBuilder.Config.DatabasePrimitives; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Services; using Azure.DataApiBuilder.Core.Services.MetadataProviders; using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Mcp.Core; +using Azure.DataApiBuilder.Mcp.Model; using Azure.DataApiBuilder.Service.Exceptions; using Azure.DataApiBuilder.Service.Utilities; using Microsoft.Extensions.DependencyInjection; @@ -189,9 +193,60 @@ public void StdioCancellationRecordsStartupFailureOnlyBeforeReadiness(bool ready Assert.AreEqual(1, host.DisposeCallCount); Assert.AreEqual(ready ? 0 : 1, exporter.Events.Count(record => record.Name == "dab.engine.startup_failed"), "The failure must be recorded before the helper stops and disables its session."); + if (!ready) + { + Assert.AreEqual("metadata", exporter.Events.Single(record => record.Name == "dab.engine.startup_failed").Properties["failure_stage"]); + } + Assert.AreEqual("dab.engine.stopped", exporter.Events.Last().Name); } + [DataTestMethod] + [TestCategory("EngineTelemetry")] + [DataRow(true, "metadata")] + [DataRow(false, "serving")] + public void StdioReportsMetadataAndServingFailuresSeparately(bool metadataFails, string expectedStage) + { + CapturingProductExporter exporter = new(); + using EngineTelemetrySession telemetry = EngineTelemetrySession.Create(() => exporter, enableSyntheticCollection: true, + readEnvironmentVariable: _ => null, showNotice: () => { }, startTimer: false); + TestMetadataProviderFactory metadata = new() { InitializeAsyncException = metadataFails ? new InvalidOperationException("synthetic private failure") : null }; + using FileSystemRuntimeConfigLoader loader = new(new MockFileSystem()) + { + RuntimeConfig = new(null, new(DatabaseType.MSSQL, string.Empty), new(new Dictionary())) + }; + using RuntimeConfigProvider provider = new(loader) { ProductTelemetry = telemetry }; + TestMcpStdioServer stdio = new(); + using ServiceProvider services = new ServiceCollection() + .AddSingleton(telemetry) + .AddSingleton(provider) + .AddSingleton(metadata) + .AddSingleton() + .AddSingleton(new TestApplicationLifetime()) + .AddSingleton(stdio) + .AddSingleton(_ => throw new InvalidOperationException("synthetic private failure")) + .BuildServiceProvider(); + TextWriter originalError = Console.Error; + using StringWriter capture = new(); + try + { + Console.SetError(capture); + Assert.IsFalse(McpStdioHelper.RunMcpStdioHost(new TestHost(services))); + } + finally + { + Console.SetError(originalError); + } + + EngineTelemetryEvent failure = exporter.Events.Single(record => record.Name == "dab.engine.startup_failed"); + Assert.AreEqual(expectedStage, failure.Properties["failure_stage"]); + Assert.AreEqual("initialization", failure.Properties["failure_category"]); + Assert.AreEqual(1, metadata.InitializeAsyncCallCount); + Assert.AreEqual(0, stdio.RunAsyncCallCount, "Tool registration must fail before the ready stdio loop begins."); + Assert.IsFalse(exporter.Events.Any(record => record.Name == "dab.engine.ready")); + Assert.IsFalse(failure.Properties.Values.Any(value => value.Contains("synthetic private failure", StringComparison.Ordinal))); + } + private static ServiceProvider BuildServices( TestMcpStdioServer stdioServer, TestMetadataProviderFactory metadataProviderFactory, diff --git a/src/Service/Controllers/ConfigurationController.cs b/src/Service/Controllers/ConfigurationController.cs index cbfed45bb7..cc5f3188c2 100644 --- a/src/Service/Controllers/ConfigurationController.cs +++ b/src/Service/Controllers/ConfigurationController.cs @@ -4,6 +4,7 @@ using System; using System.Threading.Tasks; using Azure.DataApiBuilder.Config; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Models; using Microsoft.AspNetCore.Mvc; @@ -66,7 +67,7 @@ public async Task Index([FromBody] ConfigurationPostParametersV2 c if (!initializationStarted) { // The provider owns all later failures; malformed merge input never reaches it. - _configurationProvider.ProductTelemetry?.ConfigurationChangeFailed(); + _configurationProvider.ProductTelemetry?.ConfigurationChangeFailed(TelemetryFailureStage.Parsing); } _logger.LogError( diff --git a/src/Service/Program.cs b/src/Service/Program.cs index 9108fcedd3..cbdcebe39c 100644 --- a/src/Service/Program.cs +++ b/src/Service/Program.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Azure.DataApiBuilder.Config; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Core.Telemetry; using Azure.DataApiBuilder.Core.Telemetry.Product; using Azure.DataApiBuilder.Mcp.Core; @@ -101,7 +102,7 @@ internal static bool StartEngineCore(string[] args, bool runMcpStdio, string? mc // other startup failures. Direct StartEngine callers retain their prior path. if (validateUrls && !ValidateAspNetCoreUrls()) { - productTelemetry.StartupFailed("configuration"); + productTelemetry.StartupFailed(TelemetryFailureStage.Configuration); Console.Error.WriteLine("Invalid ASPNETCORE_URLS format. e.g.: ASPNETCORE_URLS=\"http://localhost:5000;https://localhost:5001\""); return false; } @@ -137,7 +138,7 @@ internal static bool StartEngineCore(string[] args, bool runMcpStdio, string? mc bool completed = McpStdioHelper.RunMcpStdioHost(host); if (!completed) { - productTelemetry.StartupFailed("metadata"); + productTelemetry.StartupFailed(TelemetryFailureStage.Metadata); } return completed; diff --git a/src/Service/Startup.cs b/src/Service/Startup.cs index 59f4e0e1d4..d75297e143 100644 --- a/src/Service/Startup.cs +++ b/src/Service/Startup.cs @@ -15,6 +15,7 @@ using Azure.DataApiBuilder.Config.Converters; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Config.ObjectModel.Embeddings; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Config.Utilities; using Azure.DataApiBuilder.Core.AuthenticationHelpers; using Azure.DataApiBuilder.Core.AuthenticationHelpers.AuthenticationSimulator; @@ -141,7 +142,7 @@ public void ConfigureServices(IServiceCollection services) services.AddHostedService(sp => sp.GetRequiredService()); RuntimeConfigProvider configProvider = new(configLoader) { ProductTelemetry = ProductTelemetry }; _configProvider = configProvider; - configLoader.TelemetryReloadCompleted = (acceptedConfig, accepted) => + configLoader.TelemetryReloadCompleted = (acceptedConfig, accepted, failureStage) => { if (accepted && acceptedConfig is not null) { @@ -149,7 +150,7 @@ public void ConfigureServices(IServiceCollection services) } else { - ProductTelemetry?.ConfigurationChangeFailed(); + ProductTelemetry?.ConfigurationChangeFailed(failureStage); } }; @@ -1465,6 +1466,7 @@ private static void SetAppServiceAuthentication(IServiceCollection services) /// Indicates if the runtime is ready to accept requests. private async Task PerformOnConfigChangeAsync(IApplicationBuilder app) { + TelemetryFailureStage stage = TelemetryFailureStage.Configuration; try { RuntimeConfigProvider runtimeConfigProvider = app.ApplicationServices.GetService()!; @@ -1474,8 +1476,10 @@ private async Task PerformOnConfigChangeAsync(IApplicationBuilder app) // Now that the configuration has been set, perform validation of the runtime config // itself. + stage = TelemetryFailureStage.Validation; runtimeConfigValidator.ValidateConfigProperties(); + stage = TelemetryFailureStage.Metadata; IMetadataProviderFactory sqlMetadataProviderFactory = app.ApplicationServices.GetRequiredService(); await sqlMetadataProviderFactory.InitializeAsync(); @@ -1486,6 +1490,7 @@ private async Task PerformOnConfigChangeAsync(IApplicationBuilder app) // In their constructors, those services consequentially inject // other required services, triggering instantiation. Such recursive nature of DI and // service instantiation results in the activation of all required services. + stage = TelemetryFailureStage.Serving; GraphQLSchemaCreator graphQLSchemaCreator = app.ApplicationServices.GetRequiredService(); @@ -1500,10 +1505,13 @@ private async Task PerformOnConfigChangeAsync(IApplicationBuilder app) if (runtimeConfig.IsDevelopmentMode()) { // Running only in developer mode to ensure fast and smooth startup in production. + stage = TelemetryFailureStage.Validation; runtimeConfigValidator.ValidateRelationshipConfigCorrectness(runtimeConfig); + stage = TelemetryFailureStage.Metadata; runtimeConfigValidator.ValidateRelationships(runtimeConfig, sqlMetadataProviderFactory!); } + stage = TelemetryFailureStage.Serving; // OpenAPI document creation is only attempted for REST supporting database types. // CosmosDB is not supported for OpenAPI document creation. if (!runtimeConfig.CosmosDataSourceUsed) @@ -1537,17 +1545,20 @@ private async Task PerformOnConfigChangeAsync(IApplicationBuilder app) } catch (Exception ex) { + // Annotate before converting the failure to false. The provider's outer + // observer owns late-config reporting and can distinguish concurrent attempts. + TelemetryFailureContext.Current?.RecordFailure(stage); // RuntimeConfigProvider owns late-configuration failure reporting, including // parse/merge and post-parse initialization failures, exactly once per attempt. if (_configProvider?.IsLateConfigured != true) { if (ProductTelemetry?.IsReady == true) { - ProductTelemetry.ConfigurationChangeFailed(); + ProductTelemetry.ConfigurationChangeFailed(stage); } else { - ProductTelemetry?.StartupFailed("configuration"); + ProductTelemetry?.StartupFailed(stage); } } diff --git a/src/Service/Utilities/McpStdioHelper.cs b/src/Service/Utilities/McpStdioHelper.cs index 83fc54f1d3..197477df1d 100644 --- a/src/Service/Utilities/McpStdioHelper.cs +++ b/src/Service/Utilities/McpStdioHelper.cs @@ -6,6 +6,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Text; +using Azure.DataApiBuilder.Config.Telemetry; using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Services.MetadataProviders; using Azure.DataApiBuilder.Core.Telemetry.Product; @@ -94,6 +95,7 @@ public static void ConfigureMcpStdio(IConfigurationBuilder builder, string? mcpR /// reported, which Program.Main surfaces as a non-zero exit code. public static bool RunMcpStdioHost(IHost host) { + TelemetryFailureStage stage = TelemetryFailureStage.Metadata; try { // Stdio mode never calls host.Run(), so Startup.Configure -- and with it @@ -106,6 +108,7 @@ public static bool RunMcpStdioHost(IHost host) host.Services.GetRequiredService(); metadataProviderFactory.InitializeAsync().GetAwaiter().GetResult(); + stage = TelemetryFailureStage.Serving; McpToolRegistry registry = host.Services.GetRequiredService(); IEnumerable tools = @@ -136,12 +139,12 @@ public static bool RunMcpStdioHost(IHost host) // Record pre-ready cancellation before finally stops/disables the session. // StartupFailed is a no-op once ready; normal loop cancellation is not a // startup failure. Preserve propagation to Program's existing handler. - host.Services.GetService()?.StartupFailed("metadata"); + host.Services.GetService()?.StartupFailed(stage); throw; } catch (Exception ex) { - host.Services.GetService()?.StartupFailed("metadata"); + host.Services.GetService()?.StartupFailed(stage); // Mirrors Startup.PerformOnConfigChangeAsync: report and return false instead of letting // the exception escape a method whose contract is a bool, and Program.Main turns that // false into ExitCode -1. Cancellation is left to Program.StartEngine's own handler. From e85f3338d5b6ff543b0cd6c5686c53eeb0899e8a Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Fri, 25 Sep 2026 01:02:50 -0700 Subject: [PATCH 3/6] Fix conditional GraphQL telemetry and hosting classification --- docs/telemetry.md | 13 + .../Product/EngineTelemetryContext.cs | 37 ++- .../Telemetry/EngineTelemetryContextTests.cs | 225 ++++++++++++++++++ .../EngineTelemetryGraphQLIntegrationTests.cs | 222 ++++++++++++++++- .../Telemetry/EngineTelemetryLocalDbTests.cs | 110 +++++++-- .../Telemetry/EngineTelemetryProtocolTests.cs | 4 +- .../EngineTelemetryGraphQLListener.cs | 122 +++++++--- 7 files changed, 684 insertions(+), 49 deletions(-) create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryContextTests.cs diff --git a/docs/telemetry.md b/docs/telemetry.md index d9807033f0..37a2c796d6 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -58,6 +58,17 @@ All events include schema version, random event ID, engine session ID, sequence, Context includes DAB version; coarse OS family/version; process architecture; normalized .NET version; execution mode; categorical launcher/hosting/container detection; and distribution/channel/packaging labels. Known CLI/service entry assemblies select a categorical launcher; other entry points remain unknown. Distribution/channel/packaging remain `unknown` without reliable build provenance; synthetic opt-in does not prove source packaging. No network discovery or raw environment value is recorded. +Hosting is a best-effort observation captured once per enabled run, not platform attestation. Only nonblank presence of these documented built-in signals is considered: + +| `hosting` category | Required signals | +| --- | --- | +| `azure_container_apps` | Both `CONTAINER_APP_NAME` and `CONTAINER_APP_REVISION`, or both `CONTAINER_APP_JOB_NAME` and `CONTAINER_APP_JOB_EXECUTION_NAME` ([Container Apps built-ins](https://learn.microsoft.com/azure/container-apps/environment-variables#built-in-environment-variables)). | +| `kubernetes` | Both `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT_HTTPS`, with no nonblank Container Apps marker above ([Kubernetes in-pod signals](https://kubernetes.io/docs/tasks/run-application/access-api-from-pod/#directly-accessing-the-rest-api)). | +| `generic_container` | `DOTNET_RUNNING_IN_CONTAINER=true` without sufficient evidence for a specific platform. | +| `unknown` | Insufficient evidence, or an explicit `DOTNET_RUNNING_IN_CONTAINER=false` conflicting with platform inference. | + +Complete Container Apps evidence takes precedence over Kubernetes evidence. Partial Container Apps evidence blocks a Kubernetes-specific inference; the generic/unknown fallback remains. `container` independently reflects the parsed .NET flag (`enabled`, `disabled` or `unknown`), so hosting can be known when that flag is absent. No names, revisions, job identifiers, addresses or ports are retained or exported. Disabled sessions do not inspect hosting signals; reloads do not refresh them. + Configuration events contain `snapshot_schema=configuration-v1`, configuration delivery, source-provider categories and bucketed counts/limits. Fixed feature families are: | Family | Settings | @@ -81,6 +92,8 @@ The mapped embedding HTTP endpoint is included as REST request traffic even when Readiness requires accepted usable configuration and host/tool readiness. First-served and first-success are independently once per run. Discovery, health, documentation, introspection-only GraphQL, and MCP protocol-control/metadata traffic are excluded. HTTP 200 with GraphQL errors or an MCP tool error is not logical success. Variable-batch GraphQL results count independently. The current incremental/streaming GraphQL path reports `unknown` rather than inventing success from an unfinished stream. +GraphQL data eligibility uses the executor's compiled root selections and coerced variables, including variable defaults and conditional fields/fragments. A successful execution whose data selections are all excluded by `@skip`/`@include` does not count as data usage. Decisions are request-local, including each variable-batch member; cached compiled operations do not retain telemetry eligibility. Failed validation/coercion of a selected data operation can still count as a failed attempt when effective selections are unavailable, never as successful usage. Eligible cached and empty reads count without requiring a database attempt or returned row. Each completed batch member uses its own result outcome, not another member's error. + The dedicated internal health client marks its self-probes with an in-memory per-session value so REST/GraphQL health queries cannot establish usage milestones or add usage counts. The marker is neither stored nor exported and does not grant authorization. Ordinary requests, including callers supplying an unrelated marker, remain eligible. System roles are classified case-insensitively, matching authentication behavior. MCP HTTP completion is observed per tool response, including legacy SSE sessions. The SDK's nonserialized message context carries only an opaque completion holder; an outgoing filter and byte-opaque stream observer check write/flush completion without inspecting payloads or storing request IDs. A completed tool response does not wait for session disconnection. A send with no observable write/flush remains `unknown`, and observed write failures remain failures even if the SDK absorbs the exception. This proves server-side flush, not client receipt. diff --git a/src/Core/Telemetry/Product/EngineTelemetryContext.cs b/src/Core/Telemetry/Product/EngineTelemetryContext.cs index 31c85fa42b..c3e4a4c7e7 100644 --- a/src/Core/Telemetry/Product/EngineTelemetryContext.cs +++ b/src/Core/Telemetry/Product/EngineTelemetryContext.cs @@ -30,7 +30,7 @@ internal static ImmutableDictionary Create(string executionMode, .Add("dotnet_version", $"{Environment.Version.Major}.{Environment.Version.Minor}.{Environment.Version.Build}") .Add("execution_mode", executionMode is "web" or "mcp_stdio" or "embedded" ? executionMode : "unknown") .Add("launcher", launcher) - .Add("hosting", container == true ? "generic_container" : "unknown") + .Add("hosting", GetHosting(container, readEnvironmentVariable)) .Add("container", container switch { true => "enabled", false => "disabled", _ => "unknown" }) // Test mode is not evidence of packaging or a release channel. .Add("distribution", "unknown") @@ -38,6 +38,41 @@ internal static ImmutableDictionary Create(string executionMode, .Add("packaging", "unknown"); } + private static string GetHosting(bool? container, Func readEnvironmentVariable) + { + // An explicit negative container flag conflicts with an orchestrator inference. + // Keep it as a separate observation and do not guess a hosting platform. + if (container == false) + { + return "unknown"; + } + + // Fixed allowlist; reduce values immediately to presence. Do not retain names, + // addresses, ports, credentials, or infer a region from any of these values. + bool app = Present("CONTAINER_APP_NAME"); + bool revision = Present("CONTAINER_APP_REVISION"); + bool job = Present("CONTAINER_APP_JOB_NAME"); + bool execution = Present("CONTAINER_APP_JOB_EXECUTION_NAME"); + bool kubernetesHost = Present("KUBERNETES_SERVICE_HOST"); + bool kubernetesPort = Present("KUBERNETES_SERVICE_PORT_HTTPS"); + if ((app && revision) || (job && execution)) + { + // The more specific managed-platform evidence wins over Kubernetes signals. + return "azure_container_apps"; + } + + // Partial ACA evidence is insufficient to name that platform, but also prevents + // falling through to a less-specific orchestrator classification. + if (!(app || revision || job || execution) && kubernetesHost && kubernetesPort) + { + return "kubernetes"; + } + + return container == true ? "generic_container" : "unknown"; + + bool Present(string name) => !string.IsNullOrWhiteSpace(readEnvironmentVariable(name)); + } + private static string GetOperatingSystem() { if (OperatingSystem.IsWindows()) diff --git a/src/Service.Tests/Telemetry/EngineTelemetryContextTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryContextTests.cs new file mode 100644 index 0000000000..e093852d6b --- /dev/null +++ b/src/Service.Tests/Telemetry/EngineTelemetryContextTests.cs @@ -0,0 +1,225 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Core.Telemetry.Product; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Azure.DataApiBuilder.Service.Tests.Telemetry; + +[TestClass] +[TestCategory("EngineTelemetry")] +public class EngineTelemetryContextTests +{ + private const string PRIVATE_VALUE = "PRIVATE_PLATFORM_VALUE_a165fe"; + + [DataTestMethod] + [DataRow("true", "app", "azure_container_apps", "enabled")] + [DataRow("true", "job", "azure_container_apps", "enabled")] + [DataRow("true", "kubernetes", "kubernetes", "enabled")] + [DataRow("true", "app+kubernetes", "azure_container_apps", "enabled")] + [DataRow("true", "job+kubernetes", "azure_container_apps", "enabled")] + [DataRow("true", "app+job+kubernetes", "azure_container_apps", "enabled")] + [DataRow("true", "app+partial-kubernetes", "azure_container_apps", "enabled")] + [DataRow("true", "job+partial-app", "azure_container_apps", "enabled")] + [DataRow(null, "app", "azure_container_apps", "unknown")] + [DataRow(null, "job", "azure_container_apps", "unknown")] + [DataRow(null, "kubernetes", "kubernetes", "unknown")] + [DataRow("true", "none", "generic_container", "enabled")] + [DataRow(null, "none", "unknown", "unknown")] + [DataRow("false", "app+kubernetes", "unknown", "disabled")] + [DataRow("true", "partial-app+kubernetes", "generic_container", "enabled")] + [DataRow(null, "partial-app+kubernetes", "unknown", "unknown")] + [DataRow("true", "partial-revision+kubernetes", "generic_container", "enabled")] + [DataRow("true", "partial-job+kubernetes", "generic_container", "enabled")] + [DataRow("true", "partial-execution+kubernetes", "generic_container", "enabled")] + [DataRow(null, "partial-app+partial-execution", "unknown", "unknown")] + [DataRow("true", "partial-kubernetes", "generic_container", "enabled")] + [DataRow("invalid", "partial-kubernetes", "unknown", "unknown")] + [DataRow("invalid", "kubernetes", "kubernetes", "unknown")] + public void HostingUsesOnlyDocumentedPresenceWithConservativePrecedence(string? container, string signals, string hosting, string containerState) + { + Dictionary environment = new() { ["DOTNET_RUNNING_IN_CONTAINER"] = container }; + foreach (string signal in signals.Split('+')) + { + switch (signal) + { + case "app": + environment["CONTAINER_APP_NAME"] = PRIVATE_VALUE + "_app"; + environment["CONTAINER_APP_REVISION"] = PRIVATE_VALUE + "_revision"; + break; + case "job": + environment["CONTAINER_APP_JOB_NAME"] = PRIVATE_VALUE + "_job"; + environment["CONTAINER_APP_JOB_EXECUTION_NAME"] = PRIVATE_VALUE + "_execution"; + break; + case "kubernetes": + environment["KUBERNETES_SERVICE_HOST"] = PRIVATE_VALUE + "_address"; + environment["KUBERNETES_SERVICE_PORT_HTTPS"] = PRIVATE_VALUE + "_port"; + break; + case "partial-app": + environment["CONTAINER_APP_NAME"] = PRIVATE_VALUE; + break; + case "partial-revision": + environment["CONTAINER_APP_REVISION"] = PRIVATE_VALUE; + break; + case "partial-job": + environment["CONTAINER_APP_JOB_NAME"] = PRIVATE_VALUE; + break; + case "partial-execution": + environment["CONTAINER_APP_JOB_EXECUTION_NAME"] = PRIVATE_VALUE; + break; + case "partial-kubernetes": + environment["KUBERNETES_SERVICE_HOST"] = PRIVATE_VALUE; + break; + } + } + + List reads = new(); + var context = EngineTelemetryContext.Create("web", name => + { + reads.Add(name); + return environment.GetValueOrDefault(name); + }); + Assert.AreEqual(hosting, context["hosting"]); + Assert.AreEqual(containerState, context["container"]); + Assert.AreEqual(12, context.Count, "Detection must not add identifiers or extra context fields."); + Assert.IsFalse(JsonSerializer.Serialize(context).Contains(PRIVATE_VALUE, StringComparison.Ordinal)); + string[] allowed = ["DOTNET_RUNNING_IN_CONTAINER", "CONTAINER_APP_NAME", "CONTAINER_APP_REVISION", "CONTAINER_APP_JOB_NAME", + "CONTAINER_APP_JOB_EXECUTION_NAME", "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT_HTTPS"]; + Assert.IsTrue(reads.All(allowed.Contains)); + Assert.AreEqual(reads.Count, reads.Distinct().Count(), "Read each allowed input at most once."); + Assert.AreEqual(container == "false" ? 1 : 7, reads.Count, "An explicit negative container flag must skip platform probes."); + } + + [DataTestMethod] + [DataRow(null)] + [DataRow("")] + [DataRow(" \t\r\n ")] + public void EmptyPlatformMarkersCannotEstablishHosting(string? absent) + { + var context = EngineTelemetryContext.Create("web", name => name == "DOTNET_RUNNING_IN_CONTAINER" ? "true" : absent); + Assert.AreEqual("generic_container", context["hosting"]); + } + + [DataTestMethod] + [DataRow(false, "kubernetes")] + [DataRow(true, "generic_container")] + public void BlankMarkersDoNotCompleteOrConflictWithPlatformEvidence(bool partialApp, string expected) + { + var context = EngineTelemetryContext.Create("web", name => name switch + { + "DOTNET_RUNNING_IN_CONTAINER" => "true", + "KUBERNETES_SERVICE_HOST" or "KUBERNETES_SERVICE_PORT_HTTPS" => PRIVATE_VALUE, + "CONTAINER_APP_NAME" when partialApp => PRIVATE_VALUE, + _ => " \t\r\n " + }); + Assert.AreEqual(expected, context["hosting"]); + } + + [DataTestMethod] + [DataRow("CONTAINER_APP_NAME")] + [DataRow("CONTAINER_APP_REVISION")] + [DataRow("CONTAINER_APP_JOB_NAME")] + [DataRow("CONTAINER_APP_JOB_EXECUTION_NAME")] + [DataRow("KUBERNETES_SERVICE_HOST")] + [DataRow("KUBERNETES_SERVICE_PORT_HTTPS")] + public void OneMarkerAloneIsInsufficient(string onlyMarker) + { + var context = EngineTelemetryContext.Create("web", name => name == onlyMarker ? PRIVATE_VALUE : null); + Assert.AreEqual("unknown", context["hosting"]); + Assert.AreEqual("unknown", context["container"]); + } + + [DataTestMethod] + [DataRow(false, false, 0)] + [DataRow(true, true, 1)] + [DataRow(true, false, 8)] + public async Task SessionGatesPlatformReadsAndNeverRetainsTheirValues(bool enabled, bool optOut, int expectedReads) + { + string[] allowed = ["DAB_TELEMETRY_OPT_OUT", "DOTNET_RUNNING_IN_CONTAINER", "CONTAINER_APP_NAME", "CONTAINER_APP_REVISION", + "CONTAINER_APP_JOB_NAME", "CONTAINER_APP_JOB_EXECUTION_NAME", "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT_HTTPS"]; + List reads = new(); + Capture exporter = new(); + using EngineTelemetrySession session = EngineTelemetrySession.Create(() => exporter, + enableSyntheticCollection: enabled, readEnvironmentVariable: name => + { + Assert.IsTrue(allowed.Contains(name), "Do not probe any environment variable outside the published allowlist."); + reads.Add(name); + return name switch + { + "DAB_TELEMETRY_OPT_OUT" => optOut ? "1" : null, + "DOTNET_RUNNING_IN_CONTAINER" => "true", + "CONTAINER_APP_NAME" or "CONTAINER_APP_REVISION" => PRIVATE_VALUE, + _ => null + }; + }, showNotice: () => { }, resolveIdentity: _ => new(Guid.NewGuid(), "ephemeral"), startTimer: false); + Assert.AreEqual(enabled && !optOut, session.IsEnabled, "A swallowed detector exception must not make the test pass by disabling collection."); + session.AcceptConfiguration(new(null, new(DatabaseType.MSSQL, string.Empty), new(new Dictionary()))); + session.MarkHostReady(); + Assert.AreEqual(enabled && !optOut, session.IsReady); + session.AcceptConfiguration(new(null, new(DatabaseType.PostgreSQL, string.Empty), new(new Dictionary())), "hot_reload"); + Assert.AreEqual(enabled && !optOut, session.IsEnabled); + Assert.AreEqual(enabled && !optOut, session.IsReady); + await session.StopAsync(); + Assert.AreEqual(expectedReads, reads.Count); + Assert.AreEqual(reads.Count, reads.Distinct().Count(), "Immutable run context must not reread platform signals on reload or shutdown."); + if (enabled && !optOut) + { + foreach (string name in new[] { "dab.engine.ready", "dab.engine.configuration_changed", "dab.engine.stopped" }) + { + Assert.AreEqual(1, exporter.Events.Count(record => record.Name == name), "The lifecycle path must actually run."); + } + + Assert.IsTrue(exporter.Events.All(record => record.Properties["hosting"] == "azure_container_apps")); + Assert.IsFalse(JsonSerializer.Serialize(exporter.Events).Contains(PRIVATE_VALUE, StringComparison.Ordinal)); + } + else + { + Assert.AreEqual(0, exporter.Events.Count); + } + } + + [DataTestMethod] + [DataRow("CONTAINER_APP_NAME")] + [DataRow("CONTAINER_APP_REVISION")] + [DataRow("CONTAINER_APP_JOB_NAME")] + [DataRow("CONTAINER_APP_JOB_EXECUTION_NAME")] + [DataRow("KUBERNETES_SERVICE_HOST")] + [DataRow("KUBERNETES_SERVICE_PORT_HTTPS")] + public async Task UnavailablePlatformSignalDisablesOptionalCollectionWithoutExportingExceptions(string failedRead) + { + int exporterCreations = 0; + Capture exporter = new(); + using EngineTelemetrySession session = EngineTelemetrySession.Create(() => + { + Interlocked.Increment(ref exporterCreations); + return exporter; + }, enableSyntheticCollection: true, readEnvironmentVariable: name => name == failedRead + ? throw new InvalidOperationException(PRIVATE_VALUE) : null, showNotice: () => { }, startTimer: false); + Assert.IsFalse(session.IsEnabled); + await session.StopAsync(); + Assert.AreEqual(0, exporterCreations); + Assert.AreEqual(0, exporter.Events.Count); + } + + private sealed class Capture : IEngineTelemetryExporter + { + internal ConcurrentQueue Events { get; } = new(); + public ValueTask ExportAsync(EngineTelemetryEvent record, CancellationToken cancellationToken) + { + Events.Enqueue(record); + return ValueTask.FromResult(true); + } + + public void Dispose() { } + } +} diff --git a/src/Service.Tests/Telemetry/EngineTelemetryGraphQLIntegrationTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryGraphQLIntegrationTests.cs index c66ee896fc..8aa92ea7da 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetryGraphQLIntegrationTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetryGraphQLIntegrationTests.cs @@ -95,6 +95,13 @@ public async Task ValidationFailureCountsWithoutExecutingResolvers() [DataRow("query echo { ...Info } fragment Info on Query { __type(name: \"echo\") { name } }", "echo")] [DataRow("query Data { echo(value: \"unused\") } query Discovery { __typename }", "Discovery")] [DataRow("{ __typename } # echo(value: \"not executed\")", null)] + [DataRow("{ __typename echo(value: \"excluded\") @skip(if: true) }", null)] + [DataRow("{ __typename echo(value: \"excluded\") @include(if: false) }", null)] + [DataRow("{ __typename ... on Query @skip(if: true) { echo(value: \"excluded\") } }", null)] + [DataRow("{ __typename ...Data @include(if: false) } fragment Data on Query { echo(value: \"excluded\") }", null)] + [DataRow("{ __typename ...Outer } fragment Outer on Query { ...Data @skip(if: true) } fragment Data on Query { echo(value: \"excluded\") }", null)] + [DataRow("query Q($take: Boolean! = false) { __typename echo(value: \"excluded\") @include(if: $take) }", "Q")] + [DataRow("{ echo(value: \"excluded\") @skip(if: true) }", null)] public async Task IntrospectionIsExcludedRegardlessOfAliasesNamesAndSourceText(string document, string? operationName) { await using Fixture fixture = new(); @@ -109,6 +116,196 @@ public async Task IntrospectionIsExcludedRegardlessOfAliasesNamesAndSourceText(s Assert.AreEqual(0, records.Count(record => record.Name == "dab.engine.first_request_served")); } + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task ConditionalVariableBatchCountsOnlyEffectiveDataMembers(bool http) + { + await using Fixture fixture = new(); + (DefaultHttpContext context, ResponseCallbacks response) = CreateHttpContext(); + const string query = "query Q($take: Boolean!, $fail: Boolean!) { __typename ...Data @include(if: $take) } fragment Data on Query { echo(value: \"synthetic\") fail @include(if: $fail) }"; + IReadOnlyDictionary[] variables = + [ + new Dictionary { ["take"] = false, ["fail"] = false }, + new Dictionary { ["take"] = true, ["fail"] = false }, + new Dictionary { ["take"] = false, ["fail"] = true }, + new Dictionary { ["take"] = true, ["fail"] = true } + ]; + OperationRequestBuilder builder = OperationRequestBuilder.New().SetDocument(query).SetVariableValues(variables); + if (http) + { + builder.SetGlobalState(nameof(HttpContext), context); + } + + await using IExecutionResult result = await fixture.ExecuteAsync(builder); + OperationResultBatch batch = result.ExpectOperationResultBatch(); + Assert.AreEqual(4, batch.Results.Count); + Assert.AreEqual(2, fixture.ResolverRequests.Count); + Assert.AreEqual(1, batch.Results.Count(value => value.ExpectOperationResult().Errors.Count > 0)); + CollectionAssert.AreEqual(new int?[] { 0, 1, 2, 3 }, batch.Results.Select(value => value.ExpectOperationResult().VariableIndex).ToArray()); + if (http) + { + Assert.AreEqual(2, response.Count, "Excluded variable sets must not register completion callbacks."); + await fixture.AssertNoRequestTelemetryAsync(); + await response.CompleteAsync(); + } + + EngineTelemetryEvent[] records = await fixture.StopAsync(); + AssertRequests(records, success: 1, partialFailure: 1, transport: http ? "http" : "in_process"); + Assert.AreEqual(1, records.Count(value => value.Name == "dab.engine.first_successful_request")); + } + + [TestMethod] + public async Task ConditionalEligibilityIsNotCachedAcrossVariableValues() + { + await using Fixture fixture = new(); + const string document = "query Q($take: Boolean!) { __typename ...Data @include(if: $take) } fragment Data on Query { echo(value: \"synthetic\") }"; + foreach (bool take in new[] { false, true, false }) + { + await using IExecutionResult result = await fixture.ExecuteAsync(OperationRequestBuilder.New().SetDocument(document) + .SetVariableValues(new Dictionary { ["take"] = take })); + Assert.AreEqual(0, result.ExpectOperationResult().Errors.Count); + } + + Assert.AreEqual(1, fixture.ResolverRequests.Count); + AssertRequests(await fixture.StopAsync(), success: 1); + } + + [DataTestMethod] + [DataRow(false, false)] + [DataRow(true, false)] + [DataRow(false, true)] + [DataRow(true, true)] + public async Task MergedConditionalFragmentSelectionsFollowExecutorInclusion(bool first, bool second) + { + const string document = "query Q($a: Boolean!, $b: Boolean!) { __typename ... on Query @include(if: $a) { echo(value: \"same\") } ... on Query @include(if: $b) { echo(value: \"same\") } }"; + await AssertSelectionsMatchExecutorAsync(document, first, second); + } + + [DataTestMethod] + [DataRow(false, false)] + [DataRow(true, false)] + [DataRow(false, true)] + [DataRow(true, true)] + public async Task RepeatedNamedFragmentsMatchTheUninstrumentedExecutor(bool first, bool second) + { + const string document = "query Q($a: Boolean!, $b: Boolean!) { __typename ...Data @include(if: $a) ... on Query @include(if: $b) { ...Data } } fragment Data on Query { echo(value: \"same\") }"; + await AssertSelectionsMatchExecutorAsync(document, first, second); + } + + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task AllExcludedVariableMembersRemainUncounted(bool abortHttp) + { + await using Fixture fixture = new(); + using CancellationTokenSource abort = new(); + (DefaultHttpContext context, ResponseCallbacks response) = CreateHttpContext(abort.Token); + await using IExecutionResult result = await fixture.ExecuteAsync(OperationRequestBuilder.New() + .SetDocument("query Q($take: Boolean! = false) { __typename echo(value: \"synthetic\") @include(if: $take) }") + .SetVariableValues(new IReadOnlyDictionary[] + { + new Dictionary(), + new Dictionary { ["take"] = false } + }).SetGlobalState(nameof(HttpContext), context)); + Assert.AreEqual(2, result.ExpectOperationResultBatch().Results.Count); + Assert.IsTrue(result.ExpectOperationResultBatch().Results.All(member => member.ExpectOperationResult().Errors.Count == 0)); + Assert.AreEqual(0, fixture.ResolverRequests.Count); + Assert.AreEqual(0, response.Count); + if (abortHttp) + { + abort.Cancel(); + } + + await response.CompleteAsync(); + EngineTelemetryEvent[] records = await fixture.StopAsync(); + AssertRequests(records); + Assert.AreEqual(0, Summaries(records, "http_outcome").Length); + Assert.IsFalse(records.Any(record => record.Name is "dab.engine.first_request_served" or "dab.engine.first_successful_request")); + } + + [TestMethod] + public async Task ConcurrentCachedOperationSelectionsRemainRequestLocal() + { + await using Fixture fixture = new(); + const string document = "query Q($take: Boolean! = false) { __typename echo(value: \"synthetic\") @include(if: $take) }"; + // Warm the shared compiled-operation cache with an excluded request first. + await using (IExecutionResult warmup = await fixture.ExecuteAsync(OperationRequestBuilder.New().SetDocument(document))) + { + Assert.AreEqual(0, warmup.ExpectOperationResult().Errors.Count); + } + + IExecutionResult[] results = await Task.WhenAll(Enumerable.Range(0, 16).Select(index => fixture.ExecuteAsync( + OperationRequestBuilder.New().SetDocument(document).SetVariableValues(new Dictionary { ["take"] = index % 2 == 0 })))); + foreach (IExecutionResult result in results) + { + await using (result) + { + Assert.AreEqual(0, result.ExpectOperationResult().Errors.Count); + } + } + + Assert.AreEqual(8, fixture.ResolverRequests.Count); + Assert.IsTrue(fixture.ResolverRequests.All(request => request is { IsEligible: true })); + Assert.IsNull(fixture.Session.CurrentRequest); + AssertRequests(await fixture.StopAsync(), success: 8); + } + + [TestMethod] + public async Task EmptyDataResultsQualifyWithoutDatabaseAttempts() + { + await using Fixture fixture = new(); + await using IExecutionResult result = await fixture.ExecuteAsync(OperationRequestBuilder.New().SetDocument("{ empty }")); + Assert.AreEqual(0, result.ExpectOperationResult().Errors.Count); + EngineTelemetryEvent[] records = await fixture.StopAsync(); + AssertRequests(records, success: 1); + Assert.AreEqual(0, Summaries(records, "database_attempt").Length); + Assert.AreEqual(1, records.Count(record => record.Name == "dab.engine.first_successful_request")); + } + + [TestMethod] + public async Task ExcludedVariableMembersRemainExcludedOnHttpAbort() + { + await using Fixture fixture = new(); + using CancellationTokenSource abort = new(); + (DefaultHttpContext context, ResponseCallbacks response) = CreateHttpContext(abort.Token); + await using IExecutionResult result = await fixture.ExecuteAsync(OperationRequestBuilder.New() + .SetDocument("query Q($take: Boolean!) { __typename echo(value: \"synthetic\") @include(if: $take) }") + .SetVariableValues(new IReadOnlyDictionary[] + { + new Dictionary { ["take"] = false }, + new Dictionary { ["take"] = true }, + new Dictionary { ["take"] = false } + }).SetGlobalState(nameof(HttpContext), context)); + Assert.AreEqual(3, result.ExpectOperationResultBatch().Results.Count); + Assert.AreEqual(1, response.Count); + abort.Cancel(); + await response.CompleteAsync(); + EngineTelemetryEvent[] records = await fixture.StopAsync(); + AssertRequests(records, canceled: 1, transport: "http"); + AssertHttpOutcomes(records, 1, "unknown"); + Assert.IsFalse(records.Any(record => record.Name == "dab.engine.first_successful_request")); + } + + [TestMethod] + public async Task VariableBatchCoercionFailureCountsTheObservedFailureWithoutInventingPeerExecutions() + { + await using Fixture fixture = new(); + IReadOnlyDictionary[] variables = + [ + new Dictionary { ["value"] = "first" }, + new Dictionary { ["value"] = null }, + new Dictionary { ["value"] = "last" } + ]; + await using IExecutionResult result = await fixture.ExecuteAsync(OperationRequestBuilder.New() + .SetDocument("query Q($value: String!) { echo(value: $value) }").SetVariableValues(variables)); + // HC16 rejects the variable batch as a whole during coercion; it does not + // execute otherwise-valid peers or return a per-variable result batch here. + Assert.IsTrue(result.ExpectOperationResult().Errors.Count > 0); + Assert.AreEqual(0, fixture.ResolverRequests.Count); + AssertRequests(await fixture.StopAsync(), failure: 1); + } + [DataTestMethod] [DataRow("query echo { echo(value:")] [DataRow("query Data { __type(name: \"echo\") {")] @@ -303,6 +500,22 @@ public async Task VariableBatchHttpAbortCompletesEachForkOnceWithoutAStatus() Assert.AreEqual(0, records.Count(record => record.Name == "dab.engine.first_successful_request")); } + private static async Task AssertSelectionsMatchExecutorAsync(string document, bool first, bool second) + { + Dictionary variables = new() { ["a"] = first, ["b"] = second }; + await using Fixture baseline = new(enableTelemetry: false); + await using IExecutionResult expected = await baseline.ExecuteAsync(OperationRequestBuilder.New().SetDocument(document).SetVariableValues(variables)); + await using Fixture fixture = new(); + await using IExecutionResult actual = await fixture.ExecuteAsync(OperationRequestBuilder.New().SetDocument(document).SetVariableValues(variables)); + Assert.AreEqual(0, expected.ExpectOperationResult().Errors.Count); + Assert.AreEqual(0, actual.ExpectOperationResult().Errors.Count); + Assert.AreEqual(baseline.ResolverRequests.Count, fixture.ResolverRequests.Count); + Assert.IsTrue(baseline.ResolverRequests.Count is 0 or 1, "Merged occurrences must not be counted as separate requests."); + // Measure HC16's actual compiled merged-selection behavior, not an independent + // interpretation of the two source directives. Instrumentation must not alter it. + AssertRequests(await fixture.StopAsync(), success: baseline.ResolverRequests.Count); + } + private static OperationRequestBuilder VariableRequest(HttpContext? context = null, bool includePartialFailure = true) { // Identical first/third variable sets must still count as separate actual executions. @@ -398,15 +611,15 @@ private sealed class Fixture : IAsyncDisposable internal ConcurrentQueue ResumedRequests { get; } = new(); internal Func? EchoGate { get; set; } - internal Fixture() + internal Fixture(bool enableTelemetry = true) { - Session = EngineTelemetrySession.Create(() => Exporter, enableSyntheticCollection: true, + Session = EngineTelemetrySession.Create(() => Exporter, enableSyntheticCollection: enableTelemetry, clock: Clock, readEnvironmentVariable: _ => null, showNotice: () => { }, resolveIdentity: _ => new(new Guid("c4bd0757-e68a-4af6-ab1c-05e86e08f489"), "ephemeral"), startTimer: false); Session.AcceptConfiguration(InitialConfiguration); Session.MarkHostReady(); - Assert.IsTrue(Session.IsEnabled); - Assert.IsTrue(Session.IsReady); + Assert.AreEqual(enableTelemetry, Session.IsEnabled); + Assert.AreEqual(enableTelemetry, Session.IsReady); // Capture this session explicitly: HC has its own schema service provider and // must not resolve a second product session or a diagnostic-scope stand-in. @@ -415,6 +628,7 @@ internal Fixture() services.AddGraphQL().AddQueryType(descriptor => { descriptor.Name("Query"); + descriptor.Field("empty").Type>().Resolve(_ => Array.Empty()); descriptor.Field("echo").Argument("value", argument => argument.Type>()) .Type().Resolve(async context => { diff --git a/src/Service.Tests/Telemetry/EngineTelemetryLocalDbTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryLocalDbTests.cs index b6f4370e02..44ce0ab1c6 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetryLocalDbTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetryLocalDbTests.cs @@ -64,20 +64,22 @@ public class EngineTelemetryLocalDbTests private const string PRIVATE_FIELD = "LOCALDB_PRIVATE_MISSING_FIELD_65cd20"; private const string PRIVATE_QUERY_NAME = "LOCALDB_PRIVATE_QUERY_65cd20"; private const string DATA_QUERY = "query " + PRIVATE_QUERY_NAME + " { books { items { id title } } }"; + private const string EMPTY_QUERY = "{ books(filter: { id: { eq: -1 } }) { items { id title } } }"; private const string FAILED_QUERY = "{ books { items { " + PRIVATE_FIELD + " } } }"; private const string DISCOVERY_QUERY = "{ __schema { queryType { fields { name } } } }"; private static readonly TimeSpan _timeout = TimeSpan.FromSeconds(30); [DataTestMethod] - [DataRow(false, false, "Books", false)] - [DataRow(true, false, "Books", false)] - [DataRow(false, true, "Books", false)] - [DataRow(false, false, "swagger", false)] - [DataRow(false, false, "swagger/books", false)] - [DataRow(false, false, "mcp/books", false)] - [DataRow(false, false, "Books", true)] + [DataRow(false, false, "Books", false, false)] + [DataRow(true, false, "Books", false, false)] + [DataRow(false, true, "Books", false, false)] + [DataRow(false, false, "swagger", false, false)] + [DataRow(false, false, "swagger/books", false, false)] + [DataRow(false, false, "mcp/books", false, false)] + [DataRow(false, false, "Books", true, false)] + [DataRow(false, false, "Books", false, true)] public async Task RealHttpRequestsEmitMilestonesAndSqlUsageWithoutCustomerData(bool includeEmbeddingEndpoint, bool useAutoentities, - string restEntityPath, bool includeHealthProbes) + string restEntityPath, bool includeHealthProbes, bool includeCachedReads) { if (!OperatingSystem.IsWindows()) { @@ -134,6 +136,12 @@ public async Task RealHttpRequestsEmitMilestonesAndSqlUsageWithoutCustomerData(b Directory.CreateDirectory(directory); JsonNode runtimeConfiguration = JsonNode.Parse(CreateConfig(connectionString))!; runtimeConfiguration["entities"]!["Books"]!["rest"]!["path"] = restEntityPath; + if (includeCachedReads) + { + runtimeConfiguration["runtime"]!["cache"]!["enabled"] = true; + runtimeConfiguration["runtime"]!["cache"]!["ttl-seconds"] = 600; + } + if (restEntityPath == "mcp/books") { runtimeConfiguration["runtime"]!["mcp"]!["path"] = "/api/mcp"; @@ -186,11 +194,13 @@ public async Task RealHttpRequestsEmitMilestonesAndSqlUsageWithoutCustomerData(b EmbeddingProviderHandler embeddingProvider = new(); ResponseCompletionTracker completions = new(); HealthProbeHandler healthProbes = new(completions); + WindowClock clock = new(); session = EngineTelemetrySession.Create( exporterFactory: () => exporter, enableSyntheticCollection: true, configPath: configPath, executionMode: "web", + clock: includeCachedReads ? clock : null, readEnvironmentVariable: _ => null, showNotice: () => { }, resolveIdentity: _ => new(Guid.NewGuid(), "ephemeral"), @@ -283,6 +293,19 @@ public async Task RealHttpRequestsEmitMilestonesAndSqlUsageWithoutCustomerData(b "The real schema must expose the configured Books collection as books."); } + foreach (string excludedQuery in new[] + { + "{ __typename books @skip(if: true) { items { id } } }", + "{ __typename ...Data @include(if: false) } fragment Data on Query { books { items { id } } }" + }) + { + ServedResponse excluded = await completions.SendAsync(client, HttpMethod.Post, "/graphql", excludedQuery); + Assert.AreEqual(HttpStatusCode.OK, excluded.StatusCode); + using JsonDocument body = JsonDocument.Parse(excluded.Body); + Assert.IsFalse(body.RootElement.TryGetProperty("errors", out _)); + Assert.AreEqual(1, body.RootElement.GetProperty("data").EnumerateObject().Count(), "Only introspection should execute."); + } + // Start with a logical GraphQL failure. Legacy application/json negotiation // returns HTTP 200 for validation errors: HTTP success is not logical success. ServedResponse failure = await completions.SendAsync(client, HttpMethod.Post, "/graphql", FAILED_QUERY, PRIVATE_ROLE); @@ -308,6 +331,35 @@ public async Task RealHttpRequestsEmitMilestonesAndSqlUsageWithoutCustomerData(b ServedResponse graphQL = await completions.SendAsync(client, HttpMethod.Post, "/graphql", DATA_QUERY); AssertRow(graphQL, graphQL: true); + if (includeCachedReads) + { + // The full projection is already cached. Remove the owned seed row so + // returning it again additionally proves these reads use the cache. + await using (SqlConnection database = new(connectionString)) + { + await database.OpenAsync(); + using SqlCommand clear = database.CreateCommand(); + clear.CommandText = "DELETE FROM dbo.TelemetryItems;"; + Assert.AreEqual(1, await clear.ExecuteNonQueryAsync()); + } + + // Advance only the telemetry clock, not the cache's expiration clock. + clock.Advance(TimeSpan.FromHours(6)); + session.Tick(); + await exporter.WaitForAsync("dab.engine.heartbeat"); + // These requests are isolated in a new measurement window. Both must + // count as successful usage even though neither executes a SQL command. + for (int attempt = 0; attempt < 2; attempt++) + { + AssertRow(await completions.SendAsync(client, HttpMethod.Post, "/graphql", DATA_QUERY), graphQL: true); + } + } + else + { + // An actual successful empty SQL result still counts as data usage. + AssertEmptyRows(await completions.SendAsync(client, HttpMethod.Post, "/graphql", EMPTY_QUERY)); + } + if (includeEmbeddingEndpoint) { // Use the actual Startup-mapped endpoint, controller and embedding service. @@ -328,7 +380,20 @@ public async Task RealHttpRequestsEmitMilestonesAndSqlUsageWithoutCustomerData(b await session.StopAsync(); await exporter.WaitForAsync(STOPPED); EngineTelemetryEvent[] records = exporter.Records.ToArray(); - AssertUsage(records, embeddingRequests: includeEmbeddingEndpoint ? 2 : 0); + AssertUsage(records, embeddingRequests: includeEmbeddingEndpoint ? 2 : 0, additionalGraphQLReads: includeCachedReads ? 2 : 1); + if (includeCachedReads) + { + string window = clock.GetUtcNow().ToString("O", CultureInfo.InvariantCulture); + EngineTelemetryEvent[] cached = records.Where(record => record.Name == SUMMARY && + record.Properties["window_start"] == window).ToArray(); + AssertOutcomes(Summaries(cached, "request"), count: 2, successes: 2, failures: 0); + AssertOutcomes(Summaries(cached, "operation"), count: 2, successes: 2, failures: 0); + Assert.AreEqual(0L, Count(Summaries(cached, "database_attempt"), "count")); + Assert.IsTrue(Count(Summaries(cached, "cache_lookup").Where(record => + record.Properties["cache_layer"] == "level1" && record.Properties["cache_result"] == "hit"), "count") >= 2, + "Both real GraphQL requests must hit the cache."); + } + if (includeEmbeddingEndpoint) { EngineTelemetryEvent[] cache = Summaries(records, "cache_lookup").ToArray(); @@ -340,7 +405,7 @@ public async Task RealHttpRequestsEmitMilestonesAndSqlUsageWithoutCustomerData(b AssertNoPrivateValues(records, databaseName, connectionString, configPath, "TelemetryItems", "Books", PRIVATE_ROLE, PRIVATE_VALUE, PRIVATE_FIELD, - PRIVATE_QUERY_NAME, DATA_QUERY, FAILED_QUERY, DISCOVERY_QUERY, @"(localdb)\MSSQLLocalDB", session.HealthProbeToken!); + PRIVATE_QUERY_NAME, DATA_QUERY, EMPTY_QUERY, FAILED_QUERY, DISCOVERY_QUERY, @"(localdb)\MSSQLLocalDB", session.HealthProbeToken!); } finally { @@ -790,7 +855,15 @@ private static void AssertRow(ServedResponse response, bool graphQL) "The actual API must return the row from this test's unique database."); } - private static void AssertUsage(EngineTelemetryEvent[] records, int embeddingRequests = 0) + private static void AssertEmptyRows(ServedResponse response) + { + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + using JsonDocument body = JsonDocument.Parse(response.Body); + Assert.IsFalse(body.RootElement.TryGetProperty("errors", out _), response.Body); + Assert.AreEqual(0, body.RootElement.GetProperty("data").GetProperty("books").GetProperty("items").GetArrayLength()); + } + + private static void AssertUsage(EngineTelemetryEvent[] records, int embeddingRequests = 0, int additionalGraphQLReads = 0) { EngineTelemetryEvent started = OnlyEvent(records, "dab.engine.process_started"); EngineTelemetryEvent ready = OnlyEvent(records, READY); @@ -809,17 +882,17 @@ private static void AssertUsage(EngineTelemetryEvent[] records, int embeddingReq Assert.IsTrue(summaries.All(record => record.ConfigurationEpoch == 1)); Assert.IsFalse(summaries.Any(record => record.Properties["family"] == "collection_loss")); EngineTelemetryEvent[] requests = Summaries(records, "request").ToArray(); - Assert.AreEqual(3L + embeddingRequests, Count(requests, "count"), "Health, OpenAPI and introspection must not add data requests."); + Assert.AreEqual(3L + embeddingRequests + additionalGraphQLReads, Count(requests, "count"), "Health, OpenAPI and introspection must not add data requests."); Assert.IsTrue(requests.All(record => record.Properties["transport"] == "http")); AssertOutcomes(requests.Where(record => record.Properties["api"] == "rest" && record.Properties["role_class"] == "anonymous"), count: 1 + embeddingRequests, successes: 1 + embeddingRequests, failures: 0); AssertOutcomes(requests.Where(record => record.Properties["api"] == "graph_ql" && - record.Properties["role_class"] == "anonymous"), count: 1, successes: 1, failures: 0); + record.Properties["role_class"] == "anonymous"), count: 1 + additionalGraphQLReads, successes: 1 + additionalGraphQLReads, failures: 0); AssertOutcomes(requests.Where(record => record.Properties["api"] == "graph_ql" && record.Properties["role_class"] == "custom"), count: 1, successes: 0, failures: 1); EngineTelemetryEvent[] http = Summaries(records, "http_outcome").ToArray(); - Assert.AreEqual(3L + embeddingRequests, Count(http, "count")); + Assert.AreEqual(3L + embeddingRequests + additionalGraphQLReads, Count(http, "count")); Assert.IsTrue(http.All(record => record.Properties["http_status_class"] == "success"), "The GraphQL failure is logical, despite its completed 2xx HTTP response."); foreach (string api in new[] { "rest", "graph_ql" }) @@ -872,6 +945,15 @@ private static void AssertNoPrivateValues(EngineTelemetryEvent[] records, params } } + private sealed class WindowClock : TimeProvider + { + private long _timestamp; + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + public override long GetTimestamp() => Interlocked.Read(ref _timestamp); + public override DateTimeOffset GetUtcNow() => new DateTimeOffset(2026, 9, 24, 0, 0, 0, TimeSpan.Zero).AddTicks(GetTimestamp()); + internal void Advance(TimeSpan duration) => Interlocked.Add(ref _timestamp, duration.Ticks); + } + private sealed class CapturingExporter : IEngineTelemetryExporter { private readonly ConcurrentDictionary> _observed = new(); diff --git a/src/Service.Tests/Telemetry/EngineTelemetryProtocolTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryProtocolTests.cs index 365417e8cb..d31d92edc5 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetryProtocolTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetryProtocolTests.cs @@ -385,9 +385,9 @@ public async Task PipelineFailureIsNotReplacedByTheDefaultHttp200() [DataRow("query Data { books { id } } query Discovery { __typename }", null, false)] [DataRow("query Data { books { id } }", "NotAnOperation", false)] [DataRow("{ ...Cycle } fragment Cycle on Query { ...Cycle __typename }", null, false)] - public void GraphQLDiscoveryClassificationUsesTheSelectedAstNotNamesOrAliases(string document, string? operationName, bool eligible) + public void GraphQLFailedRequestDataIntentUsesTheSelectedAstNotNamesOrAliases(string document, string? operationName, bool eligible) { - Assert.AreEqual(eligible, EngineTelemetryGraphQLListener.IsDataOperation(Utf8GraphQLParser.Parse(document), operationName)); + Assert.AreEqual(eligible, EngineTelemetryGraphQLListener.HasDataIntent(Utf8GraphQLParser.Parse(document), operationName)); } [TestMethod] diff --git a/src/Service/Telemetry/EngineTelemetryGraphQLListener.cs b/src/Service/Telemetry/EngineTelemetryGraphQLListener.cs index caaf8f0bce..fe9c876300 100644 --- a/src/Service/Telemetry/EngineTelemetryGraphQLListener.cs +++ b/src/Service/Telemetry/EngineTelemetryGraphQLListener.cs @@ -9,6 +9,7 @@ using HotChocolate; using HotChocolate.Execution; using HotChocolate.Execution.Instrumentation; +using HotChocolate.Execution.Processing; using HotChocolate.Language; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; @@ -72,11 +73,11 @@ public override IDisposable ExecuteRequest(GraphQLRequestContext context) public override IDisposable ExecuteOperation(GraphQLRequestContext context) { - // The parsed document is now available, including on the operation-cache path. - // Mark before resolving fields; BeginRequest already captured the correct epoch. + // Compiled selections and coerced variables are available, even on operation-cache + // hits. Establish eligibility before resolvers, independently for each variable set. if (TryGetScope(context, out ExecutionScope? scope)) { - scope.MarkEligible(); + scope.CaptureEligibility(); } return EmptyScope; @@ -145,7 +146,7 @@ internal static EngineTelemetryOutcome ClassifyResult( return diagnosticOutcome; } - internal static bool IsDataOperation(DocumentNode? document, string? operationName) + internal static bool HasDataIntent(DocumentNode? document, string? operationName) { if (document is null) { @@ -179,9 +180,9 @@ internal static bool IsDataOperation(DocumentNode? document, string? operationNa return false; } - // Expand root fragments iteratively, with cycle protection even for invalid - // documents. Aliases and an operation called "IntrospectionQuery" do not decide - // eligibility. Nested fields below a data field are not separate requests. + // Failure-only fallback for requests rejected before effective selection exists. + // This identifies attempted data, never proves execution or successful usage. + // Expand root fragments with cycle protection even for invalid documents. Stack pending = new(); HashSet visitedFragments = new(StringComparer.Ordinal); pending.Push(operation.SelectionSet); @@ -230,7 +231,7 @@ private sealed class ExecutionScope : IDisposable { private readonly GraphQLRequestContext _context; private readonly HttpContext? _httpContext; - private bool _eligible; + private bool[]? _effectiveEligibility; private int _disposed; internal ExecutionScope(GraphQLRequestContext context, EngineTelemetryRequestScope request, HttpContext? httpContext) @@ -242,14 +243,48 @@ internal ExecutionScope(GraphQLRequestContext context, EngineTelemetryRequestSco internal EngineTelemetryRequestScope Request { get; } - internal void MarkEligible() + internal void CaptureEligibility() { - // Parse failures may have no document info. Only the parsed AST can establish - // eligibility; the source text or operation name alone cannot do so. - if (!_eligible && IsDataOperation(_context.OperationDocumentInfo?.Document, _context.Request.OperationName)) + if (_effectiveEligibility is not null) { - _eligible = true; - Request.MarkEligible(); + return; + } + + try + { + if (!_context.TryGetOperation(out var operation) || _context.VariableValues.IsDefaultOrEmpty) + { + return; + } + + bool[] eligibility = new bool[_context.VariableValues.Length]; + bool anyData = false; + for (int index = 0; index < eligibility.Length; index++) + { + ulong includeFlags = operation.CreateIncludeFlags(_context.VariableValues[index]); + foreach (Selection selection in operation.RootSelectionSet.Selections) + { + if (!selection.Field.IsIntrospectionField && selection.IsIncluded(includeFlags)) + { + eligibility[index] = true; + anyData = true; + break; + } + } + } + + // Retain only closed decisions, not variable values, names, or compiled + // operations. Never attach request-specific flags to HC's cached operation. + _effectiveEligibility = eligibility; + if (anyData) + { + Request.MarkEligible(); + } + } + catch (Exception) + { + // Optional classification cannot change GraphQL execution. Unavailable + // effective selections cannot establish a successful request milestone. } } @@ -262,24 +297,35 @@ public void Dispose() try { - // Validation failures can finish without invoking ExecuteOperation. - MarkEligible(); - if (_eligible) + // Failures or middleware short circuits can skip ExecuteOperation. + CaptureEligibility(); + if (_context.Result is OperationResultBatch batch) { - if (_context.Result is OperationResultBatch batch) + for (int position = 0; position < batch.Results.Count; position++) { - // Variable batching has one RequestContext, but each result is a - // separate execution. Fork the captured start/configuration rather - // than starting a request in the possibly reloaded current epoch. - // The original scope restores ambient state only; do not count it. - foreach (IExecutionResult result in batch.Results) + IExecutionResult result = batch.Results[position]; + // HC indexes ordinary variable results. Deferred stream results + // have no index, but the pinned SDK preserves variable-set order. + int? index = result is OperationResult operationResult ? operationResult.VariableIndex + : batch.Results.Count == _effectiveEligibility?.Length ? position : null; + if (index is int variableIndex && IsEffectiveData(variableIndex)) { - CompleteResult(Request.ForkForCompletion(), result); + // Result-local errors decide each member's outcome. A shared + // diagnostic failure must not poison error-free batch peers. + CompleteResult(Request.ForkForCompletion(), result, EngineTelemetryOutcome.Unknown); } } - else + } + else + { + EngineTelemetryOutcome outcome = ClassifyResult(_context.Result, Request.Outcome, _context.RequestAborted.IsCancellationRequested); + bool eligible = _effectiveEligibility is { Length: 1 } + ? _effectiveEligibility[0] + : IsFailedDataRequest(outcome); + if (eligible) { - CompleteResult(Request, _context.Result); + Request.MarkEligible(); + CompleteResult(Request, _context.Result, Request.Outcome); } } } @@ -290,10 +336,30 @@ public void Dispose() } } - private void CompleteResult(EngineTelemetryRequestScope request, IExecutionResult? result) + private bool IsEffectiveData(int variableIndex) => _effectiveEligibility is not null && + variableIndex >= 0 && variableIndex < _effectiveEligibility.Length && _effectiveEligibility[variableIndex]; + + private bool IsFailedDataRequest(EngineTelemetryOutcome outcome) + { + if (outcome is not (EngineTelemetryOutcome.Failure or EngineTelemetryOutcome.PartialFailure or EngineTelemetryOutcome.Canceled)) + { + return false; + } + + if (_effectiveEligibility is not null) + { + // A whole batch can fail before producing individual results. Count one + // observed failed request only if at least one effective member was data. + return Array.Exists(_effectiveEligibility, eligible => eligible); + } + + return HasDataIntent(_context.OperationDocumentInfo?.Document, _context.Request.OperationName); + } + + private void CompleteResult(EngineTelemetryRequestScope request, IExecutionResult? result, EngineTelemetryOutcome diagnosticOutcome) { CancellationToken aborted = _context.RequestAborted; - EngineTelemetryOutcome outcome = ClassifyResult(result, Request.Outcome, aborted.IsCancellationRequested); + EngineTelemetryOutcome outcome = ClassifyResult(result, diagnosticOutcome, aborted.IsCancellationRequested); request.SetOutcome(outcome); if (_httpContext is not null) { From ec4b6786010816ebc039dff401bcfc37d599b376 Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Fri, 25 Sep 2026 16:53:11 -0700 Subject: [PATCH 4/6] Fix telemetry test formatting --- .../Telemetry/EngineTelemetryQueryExecutorTests.cs | 3 ++- src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Service.Tests/Telemetry/EngineTelemetryQueryExecutorTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryQueryExecutorTests.cs index 95e8d96c74..1e089679e5 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetryQueryExecutorTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetryQueryExecutorTests.cs @@ -164,7 +164,8 @@ public async Task TokenAwareExecutionKeepsCallerCancellationAndCountsOnlyStarted } }; Mock> executor = new(new MsSqlDbExceptionParser(provider), - NullLogger.Instance, provider, new HttpContextAccessor(), null) { CallBase = true }; + NullLogger.Instance, provider, new HttpContextAccessor(), null) + { CallBase = true }; executor.Setup(value => value.CreateConnection(config.DefaultDataSourceName)).Returns(connection); session.AcceptConfiguration(config); session.MarkHostReady(); diff --git a/src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs index 2b308dbc0b..e9fd0539f0 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetryReloadTests.cs @@ -316,7 +316,8 @@ public async Task RejectedMcpRegistryKeepsTelemetryEpochAndRecoversDespiteNotifi second.Setup(value => value.IsEnabled(It.IsAny())).Returns(true); first.Setup(value => value.GetToolMetadata()).Returns(() => new Tool { - Name = "first_tool", Description = description, + Name = "first_tool", + Description = description, InputSchema = JsonSerializer.SerializeToElement(new { type = "object" }) }); second.Setup(value => value.GetToolMetadata()).Returns(() => new Tool From df0d2bc8ecd61afc91fcf4235d2d6735e5fbb677 Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Fri, 25 Sep 2026 21:37:54 -0700 Subject: [PATCH 5/6] Fix telemetry host discovery and cross-platform CI regressions --- .../Product/EngineTelemetryIdentityStore.cs | 24 ++++++- .../Configuration/ConfigurationTests.cs | 16 +++++ .../EngineTelemetryHostDiscoveryTests.cs | 63 +++++++++++++++++++ .../Telemetry/EngineTelemetryIdentityTests.cs | 53 +++++++++------- .../Telemetry/EngineTelemetryLocalDbTests.cs | 4 +- src/Service/Program.cs | 8 ++- 6 files changed, 136 insertions(+), 32 deletions(-) create mode 100644 src/Service.Tests/Telemetry/EngineTelemetryHostDiscoveryTests.cs diff --git a/src/Core/Telemetry/Product/EngineTelemetryIdentityStore.cs b/src/Core/Telemetry/Product/EngineTelemetryIdentityStore.cs index 0eb7e0e3c5..bdb5062f6f 100644 --- a/src/Core/Telemetry/Product/EngineTelemetryIdentityStore.cs +++ b/src/Core/Telemetry/Product/EngineTelemetryIdentityStore.cs @@ -120,9 +120,21 @@ internal static EngineTelemetryIdentity Resolve(string? configPath, Func /// When updating config during runtime is possible, then For invalid config the Application continues to /// accept request with status code of 503. diff --git a/src/Service.Tests/Telemetry/EngineTelemetryHostDiscoveryTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryHostDiscoveryTests.cs new file mode 100644 index 0000000000..79ad346f54 --- /dev/null +++ b/src/Service.Tests/Telemetry/EngineTelemetryHostDiscoveryTests.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#nullable enable + +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Azure.DataApiBuilder.Service.Tests.Telemetry +{ + [TestClass] + [TestCategory("EngineTelemetry")] + [DoNotParallelize] + public class EngineTelemetryHostDiscoveryTests + { + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task WebApplicationFactoryDiscoversTheEntryPointWithoutAmbiguousHostFactories(bool useStartupMarker) + { + if (useStartupMarker) + { + await AssertHostDiscoveryAsync(); + } + else + { + await AssertHostDiscoveryAsync(); + } + } + + private static async Task AssertHostDiscoveryAsync() where TEntryPoint : class + { + // Preserve the framework's real host discovery. Only replace the application + // startup so this regression needs no database, credentials or telemetry sender. + using WebApplicationFactory application = new(); + using WebApplicationFactory configured = application.WithWebHostBuilder(builder => builder + .UseEnvironment("Production") + .ConfigureAppConfiguration((_, configuration) => configuration.Sources.Clear()) + .ConfigureLogging(logging => logging.ClearProviders()) + .UseStartup()); + using HttpClient client = configured.CreateClient(); + using HttpResponseMessage response = await client.GetAsync("/"); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual("host-discovered", await response.Content.ReadAsStringAsync()); + } + + public sealed class DiscoveryStartup + { + public static void Configure(IApplicationBuilder app) + { + app.Run(context => context.Response.WriteAsync("host-discovered")); + } + } + } +} diff --git a/src/Service.Tests/Telemetry/EngineTelemetryIdentityTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryIdentityTests.cs index 02c3ee9a9c..58bcdd1085 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetryIdentityTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetryIdentityTests.cs @@ -583,31 +583,36 @@ public void AbandonedTemporaryFileIsNeitherAdoptedNorDeleted() [TestMethod] public async Task ConcurrentCreatorsReadOneImmutableWinnerAndCleanTheirOwnTemporaryFiles() { - using TemporaryConfig files = new(); - using Barrier start = new(12); - Task[] creators = Enumerable.Range(0, 12).Select(_ => - Task.Factory.StartNew(() => - { - if (!start.SignalAndWait(TimeSpan.FromSeconds(10))) + // Repeated independent races exercise the native publication path, not a cached + // identity or a process-local lock. Every attempt must preserve one immutable winner. + for (int iteration = 0; iteration < 20; iteration++) + { + using TemporaryConfig files = new(); + using Barrier start = new(12); + Task[] creators = Enumerable.Range(0, 12).Select(_ => + Task.Factory.StartNew(() => { - throw new TimeoutException("Synthetic concurrent start did not complete."); - } - - return ResolveEnabled(files.ConfigPath); - }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)).ToArray(); - - EngineTelemetryIdentity[] identities = await Task.WhenAll(creators).WaitAsync(TimeSpan.FromSeconds(30)); - - Assert.AreEqual(1, identities.Count(identity => identity.Stability == "newly_saved")); - Assert.AreEqual(11, identities.Count(identity => identity.Stability == "reused")); - Assert.AreEqual(1, identities.Select(identity => identity.ApiId).Distinct().Count()); - AssertRandomGuid(identities[0].ApiId); - byte[] winnerBytes = File.ReadAllBytes(files.SidecarPath); - EngineTelemetryIdentity subsequent = ResolveEnabled(files.ConfigPath); - Assert.AreEqual("reused", subsequent.Stability); - Assert.AreEqual(identities[0].ApiId, subsequent.ApiId); - CollectionAssert.AreEqual(winnerBytes, File.ReadAllBytes(files.SidecarPath)); - CollectionAssert.AreEquivalent(new[] { files.ConfigPath, files.SidecarPath }, Directory.GetFiles(files.DirectoryPath)); + if (!start.SignalAndWait(TimeSpan.FromSeconds(10))) + { + throw new TimeoutException("Synthetic concurrent start did not complete."); + } + + return ResolveEnabled(files.ConfigPath); + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)).ToArray(); + + EngineTelemetryIdentity[] identities = await Task.WhenAll(creators).WaitAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(1, identities.Count(identity => identity.Stability == "newly_saved"), $"Iteration {iteration}."); + Assert.AreEqual(11, identities.Count(identity => identity.Stability == "reused"), $"Iteration {iteration}."); + Assert.AreEqual(1, identities.Select(identity => identity.ApiId).Distinct().Count(), $"Iteration {iteration}."); + AssertRandomGuid(identities[0].ApiId); + byte[] winnerBytes = File.ReadAllBytes(files.SidecarPath); + EngineTelemetryIdentity subsequent = ResolveEnabled(files.ConfigPath); + Assert.AreEqual("reused", subsequent.Stability); + Assert.AreEqual(identities[0].ApiId, subsequent.ApiId); + CollectionAssert.AreEqual(winnerBytes, File.ReadAllBytes(files.SidecarPath)); + CollectionAssert.AreEquivalent(new[] { files.ConfigPath, files.SidecarPath }, Directory.GetFiles(files.DirectoryPath)); + } } [TestMethod] diff --git a/src/Service.Tests/Telemetry/EngineTelemetryLocalDbTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryLocalDbTests.cs index 537a48aade..dadc5c9182 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetryLocalDbTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetryLocalDbTests.cs @@ -214,7 +214,7 @@ public async Task RealHttpRequestsEmitMilestonesAndSqlUsageWithoutCustomerData(b resolveIdentity: _ => new(Guid.NewGuid(), "ephemeral"), startTimer: false); - host = Program.CreateHostBuilder( + host = Program.CreateHostBuilderCore( ["--ConfigFileName", configPath, "--no-https-redirect"], runMcpStdio: false, mcpRole: null, productTelemetry: session) .UseEnvironment("Development") @@ -587,7 +587,7 @@ public async Task RealMcpStdioRequestsEmitMilestonesAndSqlUsageWithoutCustomerDa }, startTimer: false); - host = Program.CreateHostBuilder( + host = Program.CreateHostBuilderCore( ["--ConfigFileName", configPath, "--no-https-redirect"], runMcpStdio: true, mcpRole: "anonymous", productTelemetry: session) .UseEnvironment("Development") diff --git a/src/Service/Program.cs b/src/Service/Program.cs index 1e82d4ff40..7d05391dcc 100644 --- a/src/Service/Program.cs +++ b/src/Service/Program.cs @@ -131,7 +131,7 @@ internal static bool StartEngineCore(string[] args, bool runMcpStdio, string? mc } } - using IHost host = CreateHostBuilder(args, runMcpStdio, mcpRole, productTelemetry).Build(); + using IHost host = CreateHostBuilderCore(args, runMcpStdio, mcpRole, productTelemetry).Build(); if (runMcpStdio) { @@ -178,9 +178,11 @@ public static bool StartEngine(string[] args) } public static IHostBuilder CreateHostBuilder(string[] args, bool runMcpStdio, string? mcpRole) - => CreateHostBuilder(args, runMcpStdio, mcpRole, productTelemetry: null); + => CreateHostBuilderCore(args, runMcpStdio, mcpRole, productTelemetry: null); - internal static IHostBuilder CreateHostBuilder(string[] args, bool runMcpStdio, string? mcpRole, EngineTelemetrySession? productTelemetry) + // HostFactoryResolver discovers CreateHostBuilder by name, including nonpublic methods. + // Keep the engine-owned telemetry helper out of that convention-based lookup. + internal static IHostBuilder CreateHostBuilderCore(string[] args, bool runMcpStdio, string? mcpRole, EngineTelemetrySession? productTelemetry) { return Host.CreateDefaultBuilder(args) .ConfigureAppConfiguration(builder => From e5ff5a5fb7d99bb0d6f6353c66d3f4e39df49109 Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Sat, 26 Sep 2026 01:39:02 -0700 Subject: [PATCH 6/6] Classify duplicate MCP role headers as unknown --- .../Utils/McpTelemetryHelper.cs | 11 +- .../Telemetry/EngineTelemetryProtocolTests.cs | 119 ++++++++++++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) diff --git a/src/Azure.DataApiBuilder.Mcp/Utils/McpTelemetryHelper.cs b/src/Azure.DataApiBuilder.Mcp/Utils/McpTelemetryHelper.cs index 070e01cb1a..2ba2311f43 100644 --- a/src/Azure.DataApiBuilder.Mcp/Utils/McpTelemetryHelper.cs +++ b/src/Azure.DataApiBuilder.Mcp/Utils/McpTelemetryHelper.cs @@ -14,6 +14,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Primitives; using ModelContextProtocol.Protocol; using static Azure.DataApiBuilder.Mcp.Model.McpEnums; @@ -302,11 +303,15 @@ private static EngineTelemetryOutcome ClassifyProductResult(CallToolResult resul httpContext = services.GetService()?.HttpContext; } + // Preserve HTTP header cardinality: joining multiple values would invent a + // custom role instead of marking the ambiguous input as unknown. + StringValues roleHeader = httpContext?.Request.Headers[AuthorizationResolver.CLIENT_ROLE_HEADER] ?? StringValues.Empty; string? role = isStdio ? configuration?.GetValue("MCP:Role") - : httpContext?.Request.Headers[AuthorizationResolver.CLIENT_ROLE_HEADER].ToString(); - EngineTelemetryRole roleClass = EngineTelemetrySession.ClassifyRole( - role, httpContext?.User.Identity?.IsAuthenticated == true); + : roleHeader.Count == 1 ? roleHeader[0] : null; + EngineTelemetryRole roleClass = roleHeader.Count > 1 + ? EngineTelemetryRole.Unknown + : EngineTelemetrySession.ClassifyRole(role, httpContext?.User.Identity?.IsAuthenticated == true); EngineTelemetryTransport transport = EngineTelemetryTransport.InProcess; if (isStdio) { diff --git a/src/Service.Tests/Telemetry/EngineTelemetryProtocolTests.cs b/src/Service.Tests/Telemetry/EngineTelemetryProtocolTests.cs index 7daab3fc86..6a4b31f517 100644 --- a/src/Service.Tests/Telemetry/EngineTelemetryProtocolTests.cs +++ b/src/Service.Tests/Telemetry/EngineTelemetryProtocolTests.cs @@ -17,6 +17,7 @@ using System.Threading.Tasks; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Core.AuthenticationHelpers; +using Azure.DataApiBuilder.Core.Authorization; using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Services; using Azure.DataApiBuilder.Core.Telemetry; @@ -38,6 +39,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Primitives; using Microsoft.VisualStudio.TestTools.UnitTesting; using ModelContextProtocol.Protocol; using Moq; @@ -520,6 +522,112 @@ public void McpLogicalOperationsUseOnlyClosedProductCategories(ToolType type, st Assert.AreEqual((EngineTelemetryOperation)expected, McpTelemetryHelper.ClassifyProductOperation(tool.Object, name)); } + [DataTestMethod] + [DataRow(new string[] { }, false, (int)EngineTelemetryRole.Anonymous)] + [DataRow(new string[] { }, true, (int)EngineTelemetryRole.Authenticated)] + [DataRow(new string[] { "Anonymous" }, false, (int)EngineTelemetryRole.Anonymous)] + [DataRow(new string[] { "AUTHENTICATED" }, true, (int)EngineTelemetryRole.Authenticated)] + [DataRow(new string[] { "" }, true, (int)EngineTelemetryRole.Authenticated)] + [DataRow(new string[] { "synthetic-private-role-a" }, true, (int)EngineTelemetryRole.Custom)] + [DataRow(new string[] { "synthetic-private-role-a,synthetic-private-role-b" }, true, (int)EngineTelemetryRole.Custom)] + [DataRow(new string[] { "synthetic-private-role-a", "synthetic-private-role-b" }, true, (int)EngineTelemetryRole.Unknown)] + [DataRow(new string[] { "synthetic-private-role-a", "synthetic-private-role-a" }, true, (int)EngineTelemetryRole.Unknown)] + [DataRow(new string[] { "anonymous", "authenticated" }, false, (int)EngineTelemetryRole.Unknown)] + [DataRow(new string[] { "", "" }, false, (int)EngineTelemetryRole.Unknown)] + public async Task McpHttpRoleClassificationPreservesHeaderCardinality(string[] roles, bool authenticated, int expected) + { + ConcurrentQueue records = new(); + CapturingExporter exporter = new(records); + using EngineTelemetrySession session = EngineTelemetrySession.Create(() => exporter, + enableSyntheticCollection: true, readEnvironmentVariable: _ => null, showNotice: () => { }, + resolveIdentity: _ => new(Guid.NewGuid(), "ephemeral"), startTimer: false); + session.AcceptConfiguration(CreateConfig()); + session.MarkHostReady(); + (DefaultHttpContext context, ResponseCallbacks response) = CreateHttpContext(); + context.User = new ClaimsPrincipal(new ClaimsIdentity(authenticated ? "synthetic" : null)); + context.Request.Headers[AuthorizationResolver.CLIENT_ROLE_HEADER] = new StringValues(roles); + Mock accessor = new(); + accessor.SetupGet(value => value.HttpContext).Returns(context); + using ServiceProvider services = new ServiceCollection() + .AddSingleton(session) + .AddSingleton(accessor.Object) + .BuildServiceProvider(); + CallToolResult result = new() { Content = [] }; + Mock tool = CreateTool(result); + EngineTelemetryRequestScope? observed = null; + tool.Setup(value => value.ExecuteAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => + { + observed = session.CurrentRequest; + return Task.FromResult(result); + }); + + // Exercise the adapter boundary directly: normal HTTP authorization rejects + // duplicate roles upstream, but an embedding host can supply its own context. + CallToolResult actual = await McpTelemetryHelper.ExecuteWithTelemetryAsync( + tool.Object, "read_records", null, services, CancellationToken.None); + + Assert.AreSame(result, actual); + Assert.IsNotNull(observed); + Assert.AreEqual((EngineTelemetryRole)expected, observed.Role); + Assert.AreEqual(EngineTelemetryHttpMiddleware.ClassifyRequestRole(context), observed.Role); + Assert.IsFalse(observed.IsCompleted); + CollectionAssert.AreEqual(roles, context.Request.Headers[AuthorizationResolver.CLIENT_ROLE_HEADER].ToArray()); + await response.CompleteAsync(); + await session.StopAsync(); + EngineTelemetryEvent request = records.Single(record => record.Name == "dab.engine.usage_summary" && + record.Properties["family"] == "request"); + Assert.AreEqual(((EngineTelemetryRole)expected).ToString().ToLowerInvariant(), request.Properties["role_class"]); + Assert.AreEqual("1", request.Properties["success"]); + Assert.AreEqual("http", request.Properties["transport"]); + Assert.IsFalse(JsonSerializer.Serialize(records.ToArray()).Contains("synthetic-private-role", StringComparison.Ordinal)); + tool.Verify(value => value.ExecuteAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + + [DataTestMethod] + [DataRow("anonymous", (int)EngineTelemetryRole.Anonymous)] + [DataRow("Authenticated", (int)EngineTelemetryRole.Authenticated)] + [DataRow("synthetic-stdio-role", (int)EngineTelemetryRole.Custom)] + public async Task McpStdioRoleUsesConfigurationInsteadOfHttpShimHeaders(string role, int expected) + { + using EngineTelemetrySession session = EngineTelemetrySession.Create(() => new CapturingExporter(new()), + enableSyntheticCollection: true, readEnvironmentVariable: _ => null, showNotice: () => { }, + resolveIdentity: _ => new(Guid.NewGuid(), "ephemeral"), startTimer: false); + session.AcceptConfiguration(CreateConfig()); + session.MarkHostReady(); + (DefaultHttpContext context, _) = CreateHttpContext(); + context.Request.Headers[AuthorizationResolver.CLIENT_ROLE_HEADER] = new StringValues(["unrelated", "duplicate"]); + Mock accessor = new(); + accessor.SetupGet(value => value.HttpContext).Returns(context); + using ServiceProvider services = new ServiceCollection() + .AddSingleton(session) + .AddSingleton(accessor.Object) + .AddSingleton(new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["MCP:StdioMode"] = "true", + ["MCP:Role"] = role + }).Build()) + .BuildServiceProvider(); + CallToolResult result = new() { Content = [] }; + Mock tool = CreateTool(result); + EngineTelemetryRequestScope? observed = null; + tool.Setup(value => value.ExecuteAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => + { + observed = session.CurrentRequest; + return Task.FromResult(result); + }); + + Assert.AreSame(result, await McpTelemetryHelper.ExecuteWithTelemetryAsync( + tool.Object, "read_records", null, services, CancellationToken.None, _ => Task.CompletedTask)); + + Assert.IsNotNull(observed); + Assert.AreEqual((EngineTelemetryRole)expected, observed.Role); + Assert.AreEqual(EngineTelemetryTransport.Stdio, observed.Transport); + Assert.IsTrue(observed.IsCompleted); + accessor.VerifyGet(value => value.HttpContext, Times.Never); + } + [TestMethod] public async Task McpStdioWrapperWaitsForWriterAndLeavesCustomerSpanAtToolExecutionBoundary() { @@ -662,6 +770,17 @@ await Task.WhenAll(Enumerable.Range(0, 3).Select(_ => Task.Run(async () => .AddDiagnosticEventListener(_ => probe); } + private sealed class CapturingExporter(ConcurrentQueue records) : IEngineTelemetryExporter + { + public ValueTask ExportAsync(EngineTelemetryEvent record, CancellationToken cancellationToken) + { + records.Enqueue(record); + return ValueTask.FromResult(true); + } + + public void Dispose() { } + } + private sealed class ResponseCallbacks : IHttpResponseFeature { private readonly ConcurrentStack<(Func Callback, object State)> _completed = new();