diff --git a/docs/design/application-name-telemetry.md b/docs/design/application-name-telemetry.md index d69c4f1c50..1f52cbfce0 100644 --- a/docs/design/application-name-telemetry.md +++ b/docs/design/application-name-telemetry.md @@ -9,13 +9,14 @@ Data API builder (DAB) embeds a compact, anonymous **usage-telemetry token** int The token has the shape: ```text -dab__+|||+ +dab__+|||+ ``` -Example (an MSSQL pool, REST + GraphQL on, Static Web Apps auth): +Example (an MSSQL pool in Linux Container Apps, one data source, database identity not yet known, +REST + GraphQL on, Static Web Apps API authentication): ```text -dab_oss_2.0.0+XXSX||110000M1M000MMMMMWMM|100?111001110?+ +dab_oss_2.0.0+XXSX|L1AC0M|110000M1M000MMMMMWMM|100?111001110?+ ``` It is opt-out (`DAB_TELEMETRY_APPNAME_OPT_OUT=1`), carries no secrets or identifiers, and is purely additive to the existing `Application Name` value. @@ -54,14 +55,15 @@ DAB already appended a plain `dab_oss_` user agent to the `Application ## The token format ```text -dab__+|||+ +dab__+|||+ ``` - `dab_oss_` — the marker for open-source deployments. The hosted scenario uses `dab_hosted_` instead (see [Hosted label](#hosted-label-dab_app_name_env)); `dab_` (`ProductInfo.DAB_MARKER_PREFIX`) is the shared prefix used to locate and decode the token in both cases. - `` — the product version `Major.Minor.Patch`. The telemetry is always based on the product version. - The payload is wrapped in `+ ... +` and has four positional sections: `context`, `general`, `runtime`, and `entity`. -> **Reserved general section.** The tracking issue defines a `general` position but does not yet define any general settings. DAB therefore emits that section as empty (`||`). Reserving it now keeps the runtime and entity positions stable when general settings are added later. +The general section occupies the previously reserved position. Older tokens with an empty general +section (`||`) still decode; runtime and entity positions do not change. Each position in a section is a single character drawn from a small alphabet. The shared sentinel values are: @@ -69,11 +71,11 @@ Each position in a section is a single character drawn from a small alphabet. Th | --- | --- | | `0` | feature present and off / false | | `1` | feature present and on / true | -| `M` | **missing** — the config section that would answer this is absent | +| `M` | **missing** — the relevant config or detection evidence is absent or inconclusive (except OS, where `M` means macOS) | | `X` | **not applicable** — not knowable when the pool opens (per-request fields) | | `?` | **not supported** — the concept is not yet modeled in DAB | -A few positions use field-specific letters instead (Source and Auth provider), described below. +A few positions use field-specific letters instead (Source, general host fields, and Auth provider), described below. ### Context section (4 characters) @@ -88,6 +90,77 @@ Identifies *what kind of connection* this is. Only `Source` is knowable when a p **Source map:** `MSSQL -> S`, `DWSQL -> D`, `PostgreSQL -> P`, `MySQL -> M`, `Cosmos -> C`, and `X` when there is no live data source (for example the CLI, which has no open connection). +### General section (6 characters) + +The first four fields describe the process host. Multiple data sources describes the fully parsed, +merged deployment. Managed identity describes **this pool's data source**, not the API authentication +provider, a different database, or another dependency such as Key Vault. + +| Pos | Field | Encoding | +| --- | --- | --- | +| 1 | Operating system | `W` = Windows, `L` = Linux, `M` = macOS, `O` = Other, `U` = Unknown | +| 2 | Running in container | `0` = No, `1` = Yes, `M` = Missing/unknown | +| 3 | Hosting environment | `L` = Local, `A` = Azure, `W` = AWS, `G` = GCP, `O` = Other, `M` = Missing/unknown | +| 4 | Azure hosting service | `C` = Container Apps, `K` = AKS, `S` = App Service, `I` = ACI, `O` = Other, `N` = Not Azure, `M` = Missing/unknown | +| 5 | Multiple data sources | `0` = One, `1` = More than one, `M` = No parsed data sources | +| 6 | Managed identity | `0` = Explicit non-MI database authentication, `1` = Explicit MI database authentication, `M` = Missing/unknown | + +**Detection is offline.** It never calls cloud metadata endpoints, inspects host files, or requests an +access token. Values are sampled when the telemetry token is computed and are not cached across config +reloads. Only categorical codes are retained; environment variable contents never enter the token. + +- **OS:** uses `OperatingSystem.IsWindows()`, `IsLinux()`, and `IsMacOS()`; other runtime platforms emit `O`. + The decoder also recognizes `U` for unknown OS values. +- **Container:** reads `DOTNET_RUNNING_IN_CONTAINER` or `DOTNET_RUNNING_IN_CONTAINERS`, accepting + `true`/`false` or `1`/`0`, case-insensitively. Either valid flag can provide the answer. Contradictory + valid flags, or no valid flags, emit `M`; absence is not proof that the process is outside a container. +- **Cloud / Azure service:** best-effort runtime signals are listed below. Conflicting clouds emit `M`. + With no cloud signals, generic `KUBERNETES_SERVICE_HOST` or Knative's `K_SERVICE` emits Other; otherwise the fallback is Local. + These are deployment hints, not guarantees or security checks. Local developer SDK credentials, + region/project settings, and `DAB_APP_NAME_ENV` do not identify the hosting cloud. +- **Multiple sources:** counts parsed data-source configurations, including child configurations. + Missing file references are not counted. A root config with one child source is still single-source. +- **MI:** SQL Server/DWSQL `Authentication=Active Directory Managed Identity` or `Active Directory MSI` + emits `1`. Explicit SQL credentials, integrated security, other explicit user/service-principal modes, + PostgreSQL/MySQL passwords, or a Cosmos account key emit `0`. Providers' connection-string builders + resolve authentication/password aliases using their effective last-value-wins semantics. + An OBO data source with MI or unknown startup/metadata credentials emits `M`: its single source token + covers both metadata and delegated-user request pools. Known non-MI credentials plus OBO emit `0`. + `Active Directory Default`, workload identity, unspecified + credential chains, external tokens, unresolved references, and unavailable authentication evidence + emit `M`. An available `IDENTITY_ENDPOINT` or `AZURE_CLIENT_ID` does **not** prove MI was used. + The effective connection string (including a hosted or file-load override) is inspected; no credential + selection is instrumented or inferred. The CLI uses the default data source when there is no live one. + +| Runtime signal (nonblank) | Cloud / service hint | +| --- | --- | +| `CONTAINER_APP_NAME`, `CONTAINER_APP_REVISION`, `CONTAINER_APP_JOB_NAME`, `CONTAINER_APP_JOB_EXECUTION_NAME` | Azure / Container Apps | +| `WEBSITE_SITE_NAME`, `WEBSITE_INSTANCE_ID` | Azure / App Service | +| `AWS_EXECUTION_ENV`, `AWS_LAMBDA_FUNCTION_NAME`, `ECS_CONTAINER_METADATA_URI`, `ECS_CONTAINER_METADATA_URI_V4` | AWS | +| `CLOUD_RUN_JOB`, `CLOUD_RUN_WORKER_POOL`, `GAE_ENV` | GCP | + +`K_SERVICE` is shared by Cloud Run and non-GCP Knative deployments. For Cloud Run services without a +provider-specific signal, use `DAB_HOSTING_ENVIRONMENT=GCP`; `K_SERVICE` alone never asserts GCP. + +#### Explicit hosting overrides + +`DAB_HOSTING_ENVIRONMENT` accepts the hosting letters above or `Local`, `Azure`, `AWS`, `GCP`, +`Other`, `Missing`. `DAB_AZURE_HOSTING_SERVICE` accepts the service letters or `ContainerApps`, +`AKS`, `AppService`, `ACI`/`ContainerInstances`, `Other`, `NotAzure`, `Missing`. Spaced service names +(`Container Apps`, `App Service`, `Container Instances`, `Not Azure`) also work. Values are trimmed +and case-insensitive. A blank value is absent; an invalid nonblank value is Missing. + +Overrides take precedence over detection. A specific Azure service implies Azure unless an explicit +hosting override says otherwise. `NotAzure` suppresses automatic Azure hints but permits AWS/GCP +detection or the Local/Other fallback. An explicit non-Azure hosting override always emits `N` for the Azure service; +an explicit unknown hosting override emits `M`. Automatic host inference never replaces an explicit +service override (`M` stays Missing, `N` stays Not Azure). Explicit Azure hosting with `NotAzure` service is contradictory and emits +`M` for the service. Azure without an identifiable service also emits `M`, rather than guessing Other. + +For AKS and ACI, set `DAB_AZURE_HOSTING_SERVICE=AKS` or `ACI`. Generic Kubernetes or the presence of +Azure credentials alone is not sufficient to identify either service. Overrides label telemetry only; +they do not change authentication, credentials, hosting, or connection behavior. + ### Runtime section (20 characters) A fingerprint of the **global** `runtime` configuration. Each position is `0` / `1` / `M` unless noted. @@ -150,7 +223,7 @@ A single class, `ApplicationNameTelemetry` (in `Azure.DataApiBuilder.Config.Tele - `BuildApplicationNameSegment(config, liveDataSource)` produces what is actually embedded and honors the opt-out switch. - `Decode(applicationName)` turns a token back into human-readable lines and is tolerant of truncation, a missing trailing delimiter, an absent payload, and extra (newer) flags. -Encode and decode are driven by **one ordered list of settings per section** (`_contextSettings`, `_runtimeSettings`, `_entitySettings`). Each setting knows how to encode itself and how to describe a decoded character, so the two directions can never drift apart, and adding a flag is a one-line, append-only change. +Encode and decode are driven by **one ordered list of settings per section** (`_contextSettings`, `_generalSettings`, `_runtimeSettings`, `_entitySettings`). Each setting knows how to encode itself and how to describe a decoded character, so the two directions can never drift apart, and adding a flag is a one-line, append-only change. `ApplicationNameTelemetryEnvironment` captures the four categorical host values once per token and accepts an isolated environment reader for deterministic tests. ### Where and when the token is embedded @@ -171,6 +244,12 @@ In a multi-database deployment each data source is its own pool, so the token is Embedding is idempotent: the engine-specific helpers parse the existing `Application Name` and **skip** if it already contains the shared `dab_` marker. This prevents duplicate OSS or hosted payloads if embedding runs more than once. A user-supplied `Application Name` without that marker is preserved and the token is appended after a comma. +SQL Server's client rejects application names longer than 128 UTF-16 code units before any connection +is opened. Composition therefore fits only the DAB-owned suffix into the remaining budget, without +altering the original custom name or an existing isolation prefix. If no space remains, the original +name is retained without a token. A partial token decodes only the fields that fit. The OBO executor +continues to preserve its entire per-user hash first when applying its own 128-character limit. + ### Opt-out (`DAB_TELEMETRY_APPNAME_OPT_OUT`) Setting `DAB_TELEMETRY_APPNAME_OPT_OUT=1` reduces the embedded value to **marker and version only** (`dab_oss_` or `dab_hosted_`, no payload). Any other value (or unset) leaves telemetry on. @@ -184,6 +263,11 @@ When `DAB_APP_NAME_ENV` is set (DAB's hosted offering sets it to `dab_hosted`), A new offline command supports inspection without a database: - `dab appname --config ` parses the config and prints the token. Context is emitted as placeholders (no live connection), so the `Source` is `X`. This command performs **no validation and opens no connection** — it is a static inspection tool, and it intentionally always shows the full encoding regardless of the opt-out switch. +- General host fields describe the machine/container running the CLI, not a future deployment target. The MI field uses the config's default data source, or `M` if none is defined. +- Inspection substitutes available local environment variables but never resolves Key Vault references, + including in child configurations. Unresolved authentication evidence is `M`. The inspection loader + reuses DAB's property converters, traverses children without runtime secret resolution or watchers, + and rejects cycles or more than 64 nested config files. It does not modify the runtime provider. - `dab appname --decode ""` prints a human-readable legend, tolerant of truncation. - `-o, --output ` writes the result to a file instead of stdout. @@ -218,7 +302,13 @@ SELECT program_name FROM sys.dm_exec_sessions WHERE program_name LIKE '%dab[_]%'; ``` -- **PostgreSQL** — `pg_stat_activity.application_name` (PostgreSQL truncates this to 63 bytes; the decoder tolerates truncation): +- **PostgreSQL** — `pg_stat_activity.application_name` (63 bytes in a standard build; the decoder tolerates truncation): + + PostgreSQL 16 first clips the UTF-8 setting on a character boundary, then replaces non-printable + bytes with ASCII `\xhh` escapes, and finally clips that expanded value to 63 bytes for statistics. + Non-ASCII custom prefixes therefore leave less telemetry space than their original UTF-8 length + suggests: `用户` uses six UTF-8 bytes but expands to 24 ASCII bytes. Do not assume every Runtime or + Entity field survives. This is server normalization, not a reason for client-side truncation in DAB. ```sql SELECT application_name FROM pg_stat_activity @@ -229,7 +319,12 @@ A captured token can be decoded back to a legend with `dab appname --decode " operating-system:"), decoded); + Assert.IsTrue(decoded.Contains("General > running-in-container:"), decoded); + Assert.IsTrue(decoded.Contains("General > hosting-environment:"), decoded); + Assert.IsTrue(decoded.Contains("General > azure-hosting-service:"), decoded); + Assert.IsTrue(decoded.Contains("General > multiple-data-sources: 0"), decoded); + Assert.IsTrue(decoded.Contains("General > managed-identity: 0"), decoded); Assert.IsTrue(decoded.Contains("runtime.rest.enabled"), decoded); Assert.IsTrue(decoded.Contains("entities.any.table"), decoded); } @@ -209,6 +221,126 @@ public void TestAppNameEncodeSupportsAbsolutePaths() Assert.IsTrue(_fileSystem.File.ReadAllText(outputPath).StartsWith("dab_oss_", StringComparison.Ordinal)); } + /// Inspection must not require a resolvable connection string or create a Key Vault client. + [DataTestMethod] + [DataRow("@env('DAB_GENERAL_REVIEW_UNSET_CONNECTION')")] + [DataRow("not a connection string")] + [DataRow("@akv('database-connection')")] + public void TestAppNameReview_InspectsUnresolvedCredentialsOffline(string connectionString) + { + // The invalid URI fails before any network access if the runtime AKV resolver is invoked. + string configJson = $$""" + { + "data-source": { "database-type": "mssql", "connection-string": "{{connectionString}}" }, + "azure-key-vault": { "endpoint": "not-a-valid-vault-uri" }, + "entities": {} + } + """; + _fileSystem!.File.WriteAllText("appname-review.json", configJson); + + int code = Program.Execute( + new[] { "appname", "--config", "appname-review.json", "--output", "appname-review.txt" }, + _cliLogger!, _fileSystem, _runtimeConfigLoader!); + + Assert.AreEqual(CliReturnCode.SUCCESS, code); + string telemetry = _fileSystem.File.ReadAllText("appname-review.txt"); + string[] sections = telemetry[(telemetry.IndexOf('+') + 1)..^1].Split('|'); + Assert.AreEqual('M', sections[1][5]); + } + + /// Inspection applies the offline policy recursively and preserves global/runtime semantics. + [TestMethod] + public void TestAppNameReview_NestedConfigurationsAreOffline() + { + const string root = """ + { + "data-source-files": ["appname-child.json", "appname-missing.json"], + "runtime": { "rest": { "enabled": false } }, + "entities": {} + } + """; + const string child = """ + { + "data-source": { "database-type": "mssql", "connection-string": "@akv('connection')" }, + "azure-key-vault": { "endpoint": "not-a-vault-uri" }, + "data-source-files": ["appname-grandchild.json"], + "entities": { "Table": { "source": { "object": "table", "type": "table" }, "permissions": [] } } + } + """; + const string grandchild = """ + { + "data-source": { "database-type": "postgresql", "connection-string": "@akv('connection')" }, + "azure-key-vault": { "endpoint": "not-a-vault-uri" }, + "entities": { "View": { "source": { "object": "view", "type": "view" }, "permissions": [] } } + } + """; + _fileSystem!.File.WriteAllText("appname-review.json", root); + _fileSystem.File.WriteAllText("appname-child.json", child); + _fileSystem.File.WriteAllText("appname-grandchild.json", grandchild); + + int code = Program.Execute( + new[] { "appname", "--config", "appname-review.json", "--output", "appname-review.txt" }, + _cliLogger!, _fileSystem, _runtimeConfigLoader!); + + Assert.AreEqual(CliReturnCode.SUCCESS, code); + string telemetry = _fileSystem.File.ReadAllText("appname-review.txt"); + string[] sections = telemetry[(telemetry.IndexOf('+') + 1)..^1].Split('|'); + Assert.AreEqual('1', sections[1][4], "Two loaded sources, not three file references."); + Assert.AreEqual('M', sections[1][5], "The root has no default connection context."); + Assert.AreEqual('0', sections[2][0], "Use the root runtime, not the child's defaults."); + Assert.AreEqual("11", sections[3][..2], "Both descendants' entities must be counted."); + Assert.IsNull(_runtimeConfigLoader!.RuntimeConfig, "Inspection must not configure or start the runtime loader."); + Assert.AreEqual(root, _fileSystem.File.ReadAllText("appname-review.json")); + Assert.AreEqual(child, _fileSystem.File.ReadAllText("appname-child.json")); + } + + /// Bad/cyclic child configs fail predictably without runaway recursive loading. + [DataTestMethod] + [DataRow("{ not json }")] + [DataRow("{\"data-source-files\":[\"appname-review.json\"],\"entities\":{}}")] + public void TestAppNameReview_InvalidOrCyclicChildFails(string child) + { + _fileSystem!.File.WriteAllText("appname-review.json", "{\"data-source-files\":[\"appname-child.json\"],\"entities\":{}}"); + _fileSystem.File.WriteAllText("appname-child.json", child); + + int code = Program.Execute( + new[] { "appname", "--config", "appname-review.json", "--output", "appname-review.txt" }, + _cliLogger!, _fileSystem, _runtimeConfigLoader!); + + Assert.AreEqual(CliReturnCode.GENERAL_ERROR, code); + Assert.IsFalse(_fileSystem.File.Exists("appname-review.txt")); + } + + /// Local environment substitution remains supported without resolving external secrets. + [TestMethod] + public void TestAppNameReview_UsesLocalEnvironmentAuthentication() + { + const string variable = "DAB_GENERAL_REVIEW_CONNECTION"; + string? original = Environment.GetEnvironmentVariable(variable); + try + { + Environment.SetEnvironmentVariable(variable, "Server=unit-test.invalid;Authentication=Active Directory Managed Identity;"); + _fileSystem!.File.WriteAllText("appname-review.json", """ + { + "data-source": { "database-type": "mssql", "connection-string": "@env('DAB_GENERAL_REVIEW_CONNECTION')" }, + "entities": {} + } + """); + + int code = Program.Execute( + new[] { "appname", "--config", "appname-review.json", "--output", "appname-review.txt" }, + _cliLogger!, _fileSystem, _runtimeConfigLoader!); + + Assert.AreEqual(CliReturnCode.SUCCESS, code); + string telemetry = _fileSystem.File.ReadAllText("appname-review.txt"); + Assert.AreEqual('1', telemetry[(telemetry.IndexOf('+') + 1)..^1].Split('|')[1][5]); + } + finally + { + Environment.SetEnvironmentVariable(variable, original); + } + } + /// /// The `appname` encode path returns GENERAL_ERROR (and writes no output file) when the config /// file cannot be found. diff --git a/src/Cli/AppNameConfigLoader.cs b/src/Cli/AppNameConfigLoader.cs new file mode 100644 index 0000000000..4d15ad5ebb --- /dev/null +++ b/src/Cli/AppNameConfigLoader.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics.CodeAnalysis; +using System.IO.Abstractions; +using System.Text.Json; +using System.Text.Json.Nodes; +using Azure.DataApiBuilder.Config; +using Azure.DataApiBuilder.Config.Converters; +using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Service.Exceptions; + +namespace Cli; + +/// +/// Reads a configuration for telemetry inspection without starting the runtime, creating watchers, +/// injecting connection strings, or resolving Key Vault secrets. Uses the normal property converters +/// but explicitly owns child-file traversal, because the runtime model's JSON constructor loads +/// children with the runtime's secret-resolution policy. +/// +internal static class AppNameConfigLoader +{ + internal static bool TryLoadConfig(string path, IFileSystem fileSystem, [NotNullWhen(true)] out RuntimeConfig? config) + { + DeserializationVariableReplacementSettings replacements = new( + doReplaceEnvVar: true, + doReplaceAkvVar: false, + envFailureMode: EnvironmentVariableReplacementFailureMode.Ignore) + { + SkipApplicationNameInjection = true, + }; + + try + { + JsonSerializerOptions options = RuntimeConfigLoader.GetSerializationOptions(replacements); + HashSet ancestors = new(OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); + config = ReadConfig(path, fileSystem, options, ancestors); + return true; + } + catch (Exception ex) when (ex is JsonException or DataApiBuilderException or IOException + or UnauthorizedAccessException or ArgumentException or InvalidOperationException) + { + // Exceptions from deserialization may include credential-bearing values. The command + // reports a generic parse failure instead of exposing those details to the logger. + config = null; + return false; + } + } + + private static RuntimeConfig ReadConfig(string path, IFileSystem fileSystem, JsonSerializerOptions options, HashSet ancestors) + { + const int maxConfigDepth = 64; + string fullPath = fileSystem.Path.GetFullPath(path); + if (ancestors.Count >= maxConfigDepth || !ancestors.Add(fullPath)) + { + throw new JsonException("Cyclic or excessively nested data-source-files."); + } + + try + { + JsonObject document = JsonNode.Parse( + fileSystem.File.ReadAllText(fullPath), + documentOptions: new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip }) as JsonObject + ?? throw new JsonException("Configuration must be a JSON object."); + + DataSourceFiles? files = document["data-source-files"]?.Deserialize(options); + document.Remove("data-source-files"); + RuntimeConfig root = document.Deserialize(options) + ?? throw new JsonException("Configuration is empty."); + + // Removing only the child-file property preserves the standard converters/defaults for + // every other property while preventing constructor-driven runtime secret resolution. + List<(string FileName, RuntimeConfig Config)> children = new(); + foreach (string childPath in files?.SourceFiles ?? Enumerable.Empty()) + { + // Match runtime semantics: file paths are relative to the process working directory, + // and absent optional child files are not counted as configured data sources. + if (fileSystem.File.Exists(childPath)) + { + RuntimeConfig child = ReadConfig(childPath, fileSystem, options, ancestors); + child.IsChildConfig = true; + children.Add((childPath, child)); + } + } + + IEnumerable configs = new[] { root }.Concat(children.Select(child => child.Config)); + Dictionary sources = configs.SelectMany(config => config.GetDataSourceNamesToDataSourcesIterator()) + .ToDictionary(entry => entry.Key, entry => entry.Value); + Dictionary entities = configs.SelectMany(config => config.Entities) + .ToDictionary(entry => entry.Key, entry => entry.Value); + Dictionary autoentities = configs.SelectMany(config => config.Autoentities) + .ToDictionary(entry => entry.Key, entry => entry.Value); + Dictionary entitySources = configs.SelectMany(config => config.Entities + .Where(_ => config.DataSource is not null || config.ChildConfigs.Count > 0) + .Select(entity => new KeyValuePair(entity.Key, config.GetDataSourceNameFromEntityName(entity.Key)))) + .ToDictionary(entry => entry.Key, entry => entry.Value); + + // Use the existing explicit-data-source-map constructor: it performs no file loading. + // This snapshot is for inspection only, never published to the runtime/provider. + RuntimeConfig result = new(root.Schema, root.DataSource!, root.Runtime!, new(entities), + root.DefaultDataSourceName, sources, entitySources, files, root.AzureKeyVault, new(autoentities)); + result.ChildConfigs.AddRange(children); + return result; + } + finally + { + ancestors.Remove(fullPath); + } + } +} diff --git a/src/Cli/Commands/AppNameOptions.cs b/src/Cli/Commands/AppNameOptions.cs index 62b6370a44..4c9ca3a0b3 100644 --- a/src/Cli/Commands/AppNameOptions.cs +++ b/src/Cli/Commands/AppNameOptions.cs @@ -5,7 +5,6 @@ using Azure.DataApiBuilder.Config; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Config.Telemetry; -using Azure.DataApiBuilder.Core.Configurations; using Cli.Constants; using CommandLine; using Microsoft.Extensions.Logging; @@ -58,14 +57,15 @@ public int Handler(ILogger logger, FileSystemRuntimeConfigLoader loader, IFileSy // We intentionally do NOT run full `validate` here — validation opens a database // connection, whereas encoding only needs the parsed runtime/entity settings. // Requiring a live database would defeat the purpose of this static inspection command. - if (!ConfigGenerator.TryGetConfigForRuntimeEngine(Config, loader, fileSystem, out _)) + if (!ConfigGenerator.TryGetConfigForRuntimeEngine(Config, loader, fileSystem, out string configPath)) { logger.LogError("Could not determine the config file to use."); return CliReturnCode.GENERAL_ERROR; } - RuntimeConfigProvider runtimeConfigProvider = new(loader); - if (!runtimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig) || runtimeConfig is null) + // An inspection command must not use the runtime provider: that path resolves Key Vault + // references and creates file watchers. Keep unresolved credentials as Missing instead. + if (!AppNameConfigLoader.TryLoadConfig(configPath, fileSystem, out RuntimeConfig? runtimeConfig)) { logger.LogError("Failed to parse the config file."); return CliReturnCode.GENERAL_ERROR; diff --git a/src/Config/Azure.DataApiBuilder.Config.csproj b/src/Config/Azure.DataApiBuilder.Config.csproj index 0d1655df76..f7e51a720e 100644 --- a/src/Config/Azure.DataApiBuilder.Config.csproj +++ b/src/Config/Azure.DataApiBuilder.Config.csproj @@ -23,6 +23,7 @@ + diff --git a/src/Config/RuntimeConfigLoader.cs b/src/Config/RuntimeConfigLoader.cs index e2f9ff795f..44ec3c62aa 100644 --- a/src/Config/RuntimeConfigLoader.cs +++ b/src/Config/RuntimeConfigLoader.cs @@ -429,6 +429,10 @@ public static JsonSerializerOptions GetSerializationOptions( /// The connection string with the telemetry-bearing Application Name embedded. public static string GetConnectionStringWithApplicationName(string connectionString, RuntimeConfig config, DataSource dataSource) { + // Hosted initialization and explicit overrides can supply a different connection string from + // the one parsed into the config. Per-pool authentication telemetry must describe that string. + dataSource = dataSource with { ConnectionString = connectionString }; + return dataSource.DatabaseType switch { DatabaseType.MSSQL or DatabaseType.DWSQL => GetMsSqlConnectionStringWithApplicationName(connectionString, config, dataSource), @@ -499,18 +503,43 @@ internal static string GetMsSqlConnectionStringWithApplicationName(string connec if (string.IsNullOrWhiteSpace(connectionStringBuilder.ApplicationName) || connectionStringBuilder.ApplicationName.Equals(defaultApplicationName, StringComparison.OrdinalIgnoreCase)) { - connectionStringBuilder.ApplicationName = applicationName; + connectionStringBuilder.ApplicationName = ComposeSqlApplicationName(string.Empty, applicationName); } else { // If the connection string contains the `Application Name` property with a value, update the value by adding the DataApiBuilder Application Name. - connectionStringBuilder.ApplicationName += $",{applicationName}"; + connectionStringBuilder.ApplicationName = ComposeSqlApplicationName(connectionStringBuilder.ApplicationName, applicationName); } // Return the updated connection string. return connectionStringBuilder.ConnectionString; } + /// + /// SqlClient rejects names longer than 128 UTF-16 code units before opening a connection. Keep + /// the existing name (including any isolation prefix) intact and fit only the DAB-owned suffix + /// into the remaining budget. The decoder already accepts a truncated payload. + /// + private static string ComposeSqlApplicationName(string existingName, string telemetry) + { + const int maxLength = 128; + string separator = existingName.Length == 0 ? string.Empty : ","; + int available = maxLength - existingName.Length - separator.Length; + if (available <= 0) + { + return existingName; + } + + int length = Math.Min(available, telemetry.Length); + if (length < telemetry.Length && length > 0 + && char.IsHighSurrogate(telemetry[length - 1]) && char.IsLowSurrogate(telemetry[length])) + { + length--; + } + + return length == 0 ? existingName : existingName + separator + telemetry[..length]; + } + /// /// It adds or replaces a property in the connection string with `Application Name` property. /// If the connection string already contains the property, it appends the property `Application Name` to the connection string, diff --git a/src/Config/Telemetry/ApplicationNameTelemetry.cs b/src/Config/Telemetry/ApplicationNameTelemetry.cs index d0b3d3bb48..6e683cc492 100644 --- a/src/Config/Telemetry/ApplicationNameTelemetry.cs +++ b/src/Config/Telemetry/ApplicationNameTelemetry.cs @@ -1,9 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Data.Common; using System.Text; +using System.Text.RegularExpressions; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Product; +using Microsoft.Data.SqlClient; +using MySqlConnector; +using Npgsql; namespace Azure.DataApiBuilder.Config.Telemetry; @@ -13,9 +18,9 @@ namespace Azure.DataApiBuilder.Config.Telemetry; /// /// Format: /// -/// <marker><version>+<context>||<runtime>|<entity>+ +/// <marker><version>+<context>|<general>|<runtime>|<entity>+ /// -/// Example: dab_oss_1.2.3+XXSX||11111M10...|10111101M...+ +/// Example: dab_oss_1.2.3+XXSX|L1AC0M|11111M10...|10111101M...+ /// /// The block is self-delimiting: it always starts with a dab_ marker (dab_oss_ for open /// source or dab_hosted_ when hosted) and ends @@ -50,6 +55,19 @@ public static class ApplicationNameTelemetry /// public const string OPT_OUT_ENV_VAR = "DAB_TELEMETRY_APPNAME_OPT_OUT"; + /// + /// Overrides best-effort hosting detection. Accepts L/Local, A/Azure, W/AWS, G/GCP, O/Other, + /// or M/Missing (case-insensitive). Invalid nonblank values encode as Missing. + /// + public const string HOSTING_ENVIRONMENT_ENV_VAR = "DAB_HOSTING_ENVIRONMENT"; + + /// + /// Overrides Azure service detection. Accepts C/ContainerApps, K/AKS, S/AppService, I/ACI, + /// O/Other, N/NotAzure, or M/Missing (case-insensitive; spaced names are also accepted). + /// A specific Azure service implies Azure unless HOSTING_ENVIRONMENT_ENV_VAR overrides it. + /// + public const string AZURE_HOSTING_SERVICE_ENV_VAR = "DAB_AZURE_HOSTING_SERVICE"; + /// Placeholder used for values that are unknown/not-applicable at the current scope. private const char NOT_APPLICABLE = 'X'; @@ -62,31 +80,47 @@ public static class ApplicationNameTelemetry private const char SECTION_SEPARATOR = '|'; private const char PAYLOAD_DELIMITER = '+'; + private static readonly Regex _unresolvedReference = new( + $"{DeserializationVariableReplacementSettings.OUTER_ENV_PATTERN}|{DeserializationVariableReplacementSettings.OUTER_AKV_PATTERN}", + RegexOptions.CultureInvariant, + TimeSpan.FromSeconds(1)); + /// Inputs available to a setting encoder. - private readonly record struct EncodeInputs(RuntimeConfig Config, DataSource? LiveDataSource); + private readonly record struct EncodeInputs( + RuntimeConfig Config, + DataSource? LiveDataSource, + ApplicationNameTelemetryEnvironment Environment); /// A single telemetry setting: its name, how to encode it, and how to describe a value. private sealed record Setting(string Name, Func Encode, Func Describe); /// - /// Produces the pure telemetry string (<marker><version>+<context>||<runtime>|<entity>+), + /// Produces the pure telemetry string (<marker><version>+<context>|<general>|<runtime>|<entity>+), /// where the marker is dab_oss_ for open source or dab_hosted_ when DAB_APP_NAME_ENV /// is set. Independent of the opt-out switch. Used by the CLI and as the telemetry-bearing portion of - /// the connection-string segment. The empty section after context reserves the general-settings - /// position for future use. + /// the connection-string segment. Host information is sampled without network calls each time + /// the token is computed, not per request or per logical connection open. /// /// The runtime config to encode. /// /// The data source whose connection is being opened, or null when there is no live /// connection context (e.g. the dab appname --config CLI command). When null, the - /// Source field is emitted as X and per–data-source flags (such as OBO) fall back to the + /// Source field is emitted as X and per-data-source flags (OBO and managed identity) fall back to the /// config's default data source. /// - public static string EncodeTelemetryString(RuntimeConfig config, DataSource? liveDataSource = null) + public static string EncodeTelemetryString(RuntimeConfig config, DataSource? liveDataSource = null) => + EncodeTelemetryString(config, liveDataSource, ApplicationNameTelemetryEnvironment.Capture()); + + /// Encodes a token from an immutable host snapshot, allowing deterministic offline tests. + internal static string EncodeTelemetryString( + RuntimeConfig config, + DataSource? liveDataSource, + ApplicationNameTelemetryEnvironment environment) { - EncodeInputs inputs = new(config, liveDataSource); + EncodeInputs inputs = new(config, liveDataSource, environment); string context = EncodeSection(_contextSettings, inputs); + string general = EncodeSection(_generalSettings, inputs); string runtime = EncodeSection(_runtimeSettings, inputs); string entity = EncodeSection(_entitySettings, inputs); @@ -94,9 +128,7 @@ public static string EncodeTelemetryString(RuntimeConfig config, DataSource? liv .Append(ProductInfo.GetTelemetryApplicationNameBase()) .Append(PAYLOAD_DELIMITER) .Append(context).Append(SECTION_SEPARATOR) - // General settings are not defined yet. Reserve their position so adding them later - // does not shift the runtime and entity sections or make the payload ambiguous. - .Append(SECTION_SEPARATOR) + .Append(general).Append(SECTION_SEPARATOR) .Append(runtime).Append(SECTION_SEPARATOR) .Append(entity) .Append(PAYLOAD_DELIMITER) @@ -171,7 +203,7 @@ public static IReadOnlyList Decode(string? applicationName) string[] sections = payload.Split(SECTION_SEPARATOR); DecodeSection(lines, "Context", _contextSettings, sections, index: 0); - // Index 1 is the reserved general-settings section. + DecodeSection(lines, "General", _generalSettings, sections, index: 1); DecodeSection(lines, "Runtime", _runtimeSettings, sections, index: 2); DecodeSection(lines, "Entity", _entitySettings, sections, index: 3); @@ -254,6 +286,85 @@ private static void DecodeSection( /// Encodes a presence flag: 1=present, 0=absent. private static char Present(bool present) => present ? '1' : '0'; + /// Counts loaded data sources, not file references (which may be missing or nested). + private static char EncodeMultipleDataSources(EncodeInputs inputs) => inputs.Config.ListAllDataSources().Take(2).Count() switch + { + 0 => MISSING, + 1 => '0', + _ => '1', + }; + + /// + /// Reports configured database authentication for this pool, not the presence of an identity in + /// the host. DefaultAzureCredential, externally supplied tokens, and workload identity do not + /// prove that managed identity was selected and remain Missing. No credentials are acquired. + /// + private static char EncodeManagedIdentity(EncodeInputs inputs) + { + DataSource? dataSource = inputs.LiveDataSource ?? inputs.Config.DataSource; + if (dataSource is null) + { + return MISSING; + } + + string connectionString = dataSource.ConnectionString; + if (string.IsNullOrWhiteSpace(connectionString)) + { + return MISSING; + } + + try + { + switch (dataSource.DatabaseType) + { + case DatabaseType.MSSQL: + case DatabaseType.DWSQL: + SqlConnectionStringBuilder sql = new(connectionString); + if (_unresolvedReference.IsMatch(sql.UserID) || _unresolvedReference.IsMatch(sql.Password)) + { + return MISSING; + } + + return sql.Authentication switch + { + // The same source token covers metadata/startup and OBO request pools. MI + // metadata plus delegated-user requests is mixed, not proof of 0 or 1. + SqlAuthenticationMethod.ActiveDirectoryManagedIdentity or SqlAuthenticationMethod.ActiveDirectoryMSI => + dataSource.IsUserDelegatedAuthEnabled ? MISSING : '1', + SqlAuthenticationMethod.ActiveDirectoryDefault or SqlAuthenticationMethod.ActiveDirectoryWorkloadIdentity => MISSING, + SqlAuthenticationMethod.NotSpecified => sql.IntegratedSecurity + || !string.IsNullOrEmpty(sql.UserID) || !string.IsNullOrEmpty(sql.Password) ? '0' : MISSING, + SqlAuthenticationMethod.SqlPassword or SqlAuthenticationMethod.ActiveDirectoryPassword + or SqlAuthenticationMethod.ActiveDirectoryIntegrated or SqlAuthenticationMethod.ActiveDirectoryInteractive + or SqlAuthenticationMethod.ActiveDirectoryServicePrincipal or SqlAuthenticationMethod.ActiveDirectoryDeviceCodeFlow => '0', + _ => MISSING, + }; + case DatabaseType.PostgreSQL: + NpgsqlConnectionStringBuilder postgres = new(connectionString); + return EncodeExplicitCredential(postgres.Password); + case DatabaseType.MySQL: + // Use the provider's alias/last-value-wins rules; checking Password and Pwd + // independently can mistake a superseded password for the effective credential. + MySqlConnectionStringBuilder mysql = new(connectionString); + return EncodeExplicitCredential(mysql.Password); + case DatabaseType.CosmosDB_NoSQL: + DbConnectionStringBuilder cosmos = new() { ConnectionString = connectionString }; + return EncodeExplicitCredential(cosmos.TryGetValue("AccountKey", out object? key) ? key as string : null); + default: + return MISSING; + } + } + catch (Exception ex) when (ex is ArgumentException or FormatException or OverflowException or RegexMatchTimeoutException) + { + // Telemetry is not a connection-string validator. Offline inspection can encounter + // placeholders or invalid values; do not fail it or log secret-bearing parser errors. + return MISSING; + } + } + + private static char EncodeExplicitCredential(string? value) => + string.IsNullOrEmpty(value) || _unresolvedReference.IsMatch(value) ? MISSING : '0'; + /// Evaluates an "any entity matches" predicate, returning M when no entities exist. private static char AnyEntity(RuntimeConfig config, Func predicate) { @@ -360,6 +471,39 @@ p.Actions is not null && // Describers (value char -> human-readable meaning) used for decoding. // --------------------------------------------------------------------------------------------- + private static string DescribeOperatingSystem(char value) => value switch + { + 'W' => "Windows", + 'L' => "Linux", + 'M' => "macOS", + 'O' => "Other", + 'U' => "Unknown", + _ => "unrecognized", + }; + + private static string DescribeHostingEnvironment(char value) => value switch + { + 'L' => "Local", + 'A' => "Azure", + 'W' => "AWS", + 'G' => "GCP", + 'O' => "Other", + MISSING => "missing", + _ => "unrecognized", + }; + + private static string DescribeAzureHostingService(char value) => value switch + { + 'C' => "Container Apps", + 'K' => "AKS", + 'S' => "App Service", + 'I' => "ACI", + 'O' => "Other", + 'N' => "Not Azure", + MISSING => "missing", + _ => "unrecognized", + }; + private static string DescribeFlag(char value) => value switch { '1' => "enabled/yes", @@ -442,6 +586,16 @@ p.Actions is not null && new Setting("Role", _ => NOT_APPLICABLE, DescribeRole), }; + private static readonly IReadOnlyList _generalSettings = new[] + { + new Setting("operating-system", i => i.Environment.OperatingSystem, DescribeOperatingSystem), + new Setting("running-in-container", i => i.Environment.RunningInContainer, DescribeFlag), + new Setting("hosting-environment", i => i.Environment.HostingEnvironment, DescribeHostingEnvironment), + new Setting("azure-hosting-service", i => i.Environment.AzureHostingService, DescribeAzureHostingService), + new Setting("multiple-data-sources", EncodeMultipleDataSources, DescribeFlag), + new Setting("managed-identity", EncodeManagedIdentity, DescribeFlag), + }; + private static readonly IReadOnlyList _runtimeSettings = new[] { new Setting("runtime.rest.enabled", i => Flag(i.Config.Runtime?.Rest?.Enabled), DescribeFlag), diff --git a/src/Config/Telemetry/ApplicationNameTelemetryEnvironment.cs b/src/Config/Telemetry/ApplicationNameTelemetryEnvironment.cs new file mode 100644 index 0000000000..425bb0e95b --- /dev/null +++ b/src/Config/Telemetry/ApplicationNameTelemetryEnvironment.cs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Azure.DataApiBuilder.Config.Telemetry; + +/// +/// Categorical host information captured once when a telemetry token is computed. Detection is +/// offline and best effort: no metadata endpoints, file-system probes, or credential resolution. +/// Environment values are never retained in the snapshot or included in the token. +/// +internal readonly record struct ApplicationNameTelemetryEnvironment( + char OperatingSystem, + char RunningInContainer, + char HostingEnvironment, + char AzureHostingService) +{ + /// + /// Reads current host signals without caching them across configuration loads. The optional reader + /// lets tests supply an isolated environment without mutating process-wide variables. + /// + internal static ApplicationNameTelemetryEnvironment Capture(Func? readEnvironmentVariable = null) + { + readEnvironmentVariable ??= Environment.GetEnvironmentVariable; + + char operatingSystem = System.OperatingSystem.IsWindows() ? 'W' + : System.OperatingSystem.IsLinux() ? 'L' + : System.OperatingSystem.IsMacOS() ? 'M' + : 'O'; + + bool? container = ParseBoolean(readEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER")); + bool? containers = ParseBoolean(readEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINERS")); + // Either supported variable can supply the answer. Contradictory valid values, or no valid + // value at all, are unknown; missing flags do not prove that this is outside a container. + char runningInContainer = container.HasValue && containers.HasValue && container != containers + ? 'M' + : (container ?? containers) switch { true => '1', false => '0', null => 'M' }; + + (char hostingEnvironment, char azureHostingService) = DetectHosting(readEnvironmentVariable); + return new(operatingSystem, runningInContainer, hostingEnvironment, azureHostingService); + } + + private static bool? ParseBoolean(string? value) => value?.Trim().ToUpperInvariant() switch + { + "1" or "TRUE" => true, + "0" or "FALSE" => false, + _ => null, + }; + + private static (char HostingEnvironment, char AzureHostingService) DetectHosting(Func read) + { + bool HasValue(string name) => !string.IsNullOrWhiteSpace(read(name)); + + char? hostingOverride = ParseHostingOverride(read(ApplicationNameTelemetry.HOSTING_ENVIRONMENT_ENV_VAR)); + char? serviceOverride = ParseAzureServiceOverride(read(ApplicationNameTelemetry.AZURE_HOSTING_SERVICE_ENV_VAR)); + + bool containerApps = HasValue("CONTAINER_APP_NAME") || HasValue("CONTAINER_APP_REVISION") + || HasValue("CONTAINER_APP_JOB_NAME") || HasValue("CONTAINER_APP_JOB_EXECUTION_NAME"); + bool appService = HasValue("WEBSITE_SITE_NAME") || HasValue("WEBSITE_INSTANCE_ID"); + // An explicit Not Azure service suppresses stale automatic Azure signals as well. + bool azure = serviceOverride != 'N' && (containerApps || appService); + bool aws = HasValue("AWS_EXECUTION_ENV") || HasValue("AWS_LAMBDA_FUNCTION_NAME") + || HasValue("ECS_CONTAINER_METADATA_URI") || HasValue("ECS_CONTAINER_METADATA_URI_V4"); + bool gcp = HasValue("CLOUD_RUN_JOB") || HasValue("CLOUD_RUN_WORKER_POOL") || HasValue("GAE_ENV"); + + // SDK credentials/project/region settings also exist on developer machines. Do not use them + // to infer a cloud, and do not treat generic Kubernetes as proof of AKS (or any other cloud). + int cloudCount = (azure ? 1 : 0) + (aws ? 1 : 0) + (gcp ? 1 : 0); + char detectedHost = cloudCount > 1 ? 'M' : azure ? 'A' : aws ? 'W' : gcp ? 'G' + // K_SERVICE is part of the portable Knative contract, not a GCP-specific signal. + : HasValue("KUBERNETES_SERVICE_HOST") || HasValue("K_SERVICE") ? 'O' : 'L'; + char detectedAzureService = containerApps && appService ? 'M' + : containerApps ? 'C' : appService ? 'S' : 'M'; + + // An explicit Azure service also identifies the cloud, unless an explicit hosting override + // says otherwise. AKS and ACI require this override: neither has a universal runtime marker. + char host = hostingOverride ?? (serviceOverride is 'C' or 'K' or 'S' or 'I' or 'O' ? 'A' : detectedHost); + char service = (hostingOverride, serviceOverride) switch + { + // Only an EXPLICIT cloud override outranks an explicit service override. Automatic + // cloud inference must not erase a deliberate Missing or Not Azure service value. + (not null, _) => host switch + { + 'A' => serviceOverride == 'N' ? 'M' : serviceOverride ?? detectedAzureService, + 'M' => 'M', + _ => 'N', + }, + (null, not null) => serviceOverride.Value, + _ => host == 'A' ? detectedAzureService : host == 'M' ? 'M' : 'N', + }; + + return (host, service); + } + + // Blank overrides are absent. Nonblank invalid overrides deliberately produce Missing rather + // than silently falling back to automatic detection. Only the bounded code is ever emitted. + private static char? ParseHostingOverride(string? value) => value?.Trim().ToUpperInvariant() switch + { + null or "" => null, + "L" or "LOCAL" => 'L', + "A" or "AZURE" => 'A', + "W" or "AWS" => 'W', + "G" or "GCP" => 'G', + "O" or "OTHER" => 'O', + _ => 'M', + }; + + private static char? ParseAzureServiceOverride(string? value) => value?.Trim().ToUpperInvariant() switch + { + null or "" => null, + "C" or "CONTAINERAPPS" or "CONTAINER APPS" => 'C', + "K" or "AKS" => 'K', + "S" or "APPSERVICE" or "APP SERVICE" => 'S', + "I" or "ACI" or "CONTAINERINSTANCES" or "CONTAINER INSTANCES" => 'I', + "O" or "OTHER" => 'O', + "N" or "NOTAZURE" or "NOT AZURE" => 'N', + _ => 'M', + }; +} diff --git a/src/Service.Tests/Configuration/RuntimeConfigLoaderTests.cs b/src/Service.Tests/Configuration/RuntimeConfigLoaderTests.cs index c48a401e29..87514bdb26 100644 --- a/src/Service.Tests/Configuration/RuntimeConfigLoaderTests.cs +++ b/src/Service.Tests/Configuration/RuntimeConfigLoaderTests.cs @@ -602,8 +602,11 @@ public async Task MultiDbChildDataSourceConnectionStringEncodesGlobalTelemetry() DataSource parentDataSource = runtimeConfig.GetDataSourceFromDataSourceName(runtimeConfig.GetDataSourceNameFromEntityName("ParentEntity")); DataSource childDataSource = runtimeConfig.GetDataSourceFromDataSourceName(runtimeConfig.GetDataSourceNameFromEntityName("ChildEntity")); - (_, string parentRuntime, string parentEntity) = GetTelemetrySections(parentDataSource.ConnectionString); - (_, string childRuntime, string childEntity) = GetTelemetrySections(childDataSource.ConnectionString); + (_, string parentGeneral, string parentRuntime, string parentEntity) = GetTelemetrySections(parentDataSource.ConnectionString); + (_, string childGeneral, string childRuntime, string childEntity) = GetTelemetrySections(childDataSource.ConnectionString); + + Assert.AreEqual('1', parentGeneral[4], "The default pool must report the merged data-source count."); + Assert.AreEqual('1', childGeneral[4], "The child pool must report the merged data-source count."); // Sanity: the root has a real runtime, so its encoded runtime section is meaningful, i.e. not // entirely the 'M' (missing) sentinel. This guarantees the equality checks below are meaningful. @@ -711,8 +714,11 @@ public async Task MultiDbPostgresChildDataSourceEncodesGlobalTelemetryWithPostgr DataSource parentDataSource = runtimeConfig.GetDataSourceFromDataSourceName(runtimeConfig.GetDataSourceNameFromEntityName("ParentEntity")); DataSource childDataSource = runtimeConfig.GetDataSourceFromDataSourceName(runtimeConfig.GetDataSourceNameFromEntityName("ChildEntity")); - (string parentContext, string parentRuntime, string parentEntity) = GetTelemetrySections(parentDataSource.ConnectionString); - (string childContext, string childRuntime, string childEntity) = GetTelemetrySections(childDataSource.ConnectionString); + (string parentContext, string parentGeneral, string parentRuntime, string parentEntity) = GetTelemetrySections(parentDataSource.ConnectionString); + (string childContext, string childGeneral, string childRuntime, string childEntity) = GetTelemetrySections(childDataSource.ConnectionString); + + Assert.AreEqual('1', parentGeneral[4], "The default pool must report multiple parsed data sources."); + Assert.AreEqual('1', childGeneral[4], "The PostgreSQL pool must report multiple parsed data sources."); // Context = [Protocol][Object][Source][Role]; only Source is known at pool time. // The PostgreSQL pool encodes Source 'P'; the MSSQL pool encodes Source 'S'. @@ -940,11 +946,11 @@ public void FlushLogBuffer_IsNullSafe_AndEmitsBufferedTelemetryLog() } /// - /// Extracts the populated telemetry sections (context, runtime, entity) from the DAB usage-telemetry + /// Extracts the populated telemetry sections (context, general, runtime, entity) from the DAB usage-telemetry /// payload embedded in a connection string's "Application Name" property. - /// Payload shape: <marker><version>+<context>||<runtime>|<entity>+ + /// Payload shape: <marker><version>+<context>|<general>|<runtime>|<entity>+ /// - private static (string Context, string Runtime, string Entity) GetTelemetrySections(string connectionString) + private static (string Context, string General, string Runtime, string Entity) GetTelemetrySections(string connectionString) { // Use the engine-agnostic base builder so this works for both SQL Server and PostgreSQL connection strings. DbConnectionStringBuilder builder = new() { ConnectionString = connectionString }; @@ -964,9 +970,9 @@ private static (string Context, string Runtime, string Entity) GetTelemetrySecti string[] sections = sectionsRegion.Split('|'); Assert.AreEqual(4, sections.Length, $"Telemetry payload in '{applicationName}' should have 4 positional sections, but was '{sectionsRegion}'."); - Assert.AreEqual(string.Empty, sections[1], "The reserved general-settings section should be empty."); + Assert.AreEqual(6, sections[1].Length, "The general section should have six flags."); - return (sections[0], sections[2], sections[3]); + return (sections[0], sections[1], sections[2], sections[3]); } /// Minimal in-memory that records formatted messages for assertions. diff --git a/src/Service.Tests/UnitTests/ApplicationNameGeneralTelemetryTests.cs b/src/Service.Tests/UnitTests/ApplicationNameGeneralTelemetryTests.cs new file mode 100644 index 0000000000..121b37982f --- /dev/null +++ b/src/Service.Tests/UnitTests/ApplicationNameGeneralTelemetryTests.cs @@ -0,0 +1,820 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Globalization; +using System.IO; +using System.IO.Abstractions.TestingHelpers; +using System.Linq; +using System.Security.Claims; +using System.Text; +using System.Threading.Tasks; +using Azure.DataApiBuilder.Config; +using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Telemetry; +using Azure.DataApiBuilder.Core.Configurations; +using Azure.DataApiBuilder.Core.Resolvers; +using Azure.DataApiBuilder.Product; +using Microsoft.AspNetCore.Http; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using MySqlConnector; + +namespace Azure.DataApiBuilder.Service.Tests.UnitTests; + +/// Offline tests of the six General flags and their positional/connection-string contracts. +[TestClass] +[DoNotParallelize] +public class ApplicationNameGeneralTelemetryTests +{ + private const string SQL_MI = "Server=unit-test.invalid;Authentication=Active Directory Managed Identity;"; + private const string SQL_PASSWORD = "Server=unit-test.invalid;User ID=test-user;Password=test-only;"; + private static readonly ApplicationNameTelemetryEnvironment _environment = new('L', '1', 'A', 'C'); + private string? _originalOptOut; + private string? _originalAppName; + + /// Embedding tests must not inherit an ambient opt-out or custom marker. + [TestInitialize] + public void SaveEnvironment() + { + _originalOptOut = Environment.GetEnvironmentVariable(ApplicationNameTelemetry.OPT_OUT_ENV_VAR); + _originalAppName = Environment.GetEnvironmentVariable(ProductInfo.DAB_APP_NAME_ENV); + Environment.SetEnvironmentVariable(ApplicationNameTelemetry.OPT_OUT_ENV_VAR, null); + Environment.SetEnvironmentVariable(ProductInfo.DAB_APP_NAME_ENV, null); + } + + /// Preserves environment settings supplied by the test runner or developer. + [TestCleanup] + public void RestoreEnvironment() + { + Environment.SetEnvironmentVariable(ApplicationNameTelemetry.OPT_OUT_ENV_VAR, _originalOptOut); + Environment.SetEnvironmentVariable(ProductInfo.DAB_APP_NAME_ENV, _originalAppName); + } + + /// All six fields are in the specified order without shifting the other sections. + [TestMethod] + public void EncodeGeneral_HasExpectedPositions() + { + DataSource source = new(DatabaseType.MSSQL, SQL_MI); + string telemetry = ApplicationNameTelemetry.EncodeTelemetryString(Config(source), source, _environment); + string[] sections = Sections(telemetry); + + Assert.AreEqual("XXSX", sections[0]); + Assert.AreEqual("L1AC01", sections[1]); + Assert.AreEqual(20, sections[2].Length); + Assert.AreEqual(14, sections[3].Length); + Assert.IsTrue(telemetry.Length <= 128, "The plain token must fit the SQL Server application-name budget."); + } + + /// OS detection uses the runtime platform, not the mutable OS environment variable. + [TestMethod] + public void CaptureOperatingSystem_UsesRuntimePlatform() + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new() { ["OS"] = "untrusted-host-value" }); + char expected = OperatingSystem.IsWindows() ? 'W' : OperatingSystem.IsLinux() ? 'L' + : OperatingSystem.IsMacOS() ? 'M' : 'O'; + + Assert.AreEqual(expected, snapshot.OperatingSystem); + } + + /// Both .NET container flags support booleans and 0/1 without assuming absent means false. + [DataTestMethod] + [DataRow(null, null, 'M')] + [DataRow("", " ", 'M')] + [DataRow("true", null, '1')] + [DataRow(null, "TRUE", '1')] + [DataRow("1", null, '1')] + [DataRow(null, "1", '1')] + [DataRow("false", null, '0')] + [DataRow(null, "False", '0')] + [DataRow("0", null, '0')] + [DataRow(null, "0", '0')] + [DataRow(" true ", "1", '1')] + [DataRow("false", "0", '0')] + [DataRow("yes", "invalid", 'M')] + [DataRow("invalid", "true", '1')] + [DataRow("false", "true", 'M')] + [DataRow("1", "0", 'M')] + public void CaptureContainer_EncodesKnownOrMissing(string singular, string plural, char expected) + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new() + { + ["DOTNET_RUNNING_IN_CONTAINER"] = singular, + ["DOTNET_RUNNING_IN_CONTAINERS"] = plural, + }); + + Assert.AreEqual(expected, snapshot.RunningInContainer); + } + + /// Only runtime hosting signals identify a cloud; ordinary SDK configuration does not. + [DataTestMethod] + [DataRow("CONTAINER_APP_NAME", "example", 'A', 'C')] + [DataRow("CONTAINER_APP_REVISION", "example", 'A', 'C')] + [DataRow("CONTAINER_APP_JOB_NAME", "example", 'A', 'C')] + [DataRow("CONTAINER_APP_JOB_EXECUTION_NAME", "example", 'A', 'C')] + [DataRow("WEBSITE_SITE_NAME", "example", 'A', 'S')] + [DataRow("WEBSITE_INSTANCE_ID", "example", 'A', 'S')] + [DataRow("AWS_EXECUTION_ENV", "example", 'W', 'N')] + [DataRow("AWS_LAMBDA_FUNCTION_NAME", "example", 'W', 'N')] + [DataRow("ECS_CONTAINER_METADATA_URI", "example", 'W', 'N')] + [DataRow("ECS_CONTAINER_METADATA_URI_V4", "example", 'W', 'N')] + [DataRow("K_SERVICE", "example", 'O', 'N')] + [DataRow("CLOUD_RUN_JOB", "example", 'G', 'N')] + [DataRow("CLOUD_RUN_WORKER_POOL", "example", 'G', 'N')] + [DataRow("GAE_ENV", "standard", 'G', 'N')] + [DataRow("KUBERNETES_SERVICE_HOST", "example", 'O', 'N')] + [DataRow("WEBSITE_SITE_NAME", " ", 'L', 'N')] + [DataRow("AZURE_CLIENT_ID", "example", 'L', 'N')] + [DataRow("AZURE_TENANT_ID", "example", 'L', 'N')] + [DataRow("AZURE_FEDERATED_TOKEN_FILE", "example", 'L', 'N')] + [DataRow("IDENTITY_ENDPOINT", "example", 'L', 'N')] + [DataRow("AWS_REGION", "example", 'L', 'N')] + [DataRow("GOOGLE_CLOUD_PROJECT", "example", 'L', 'N')] + [DataRow("DAB_APP_NAME_ENV", "dab_hosted", 'L', 'N')] + public void CaptureHosting_UsesRuntimeSignals(string name, string value, char host, char service) + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new() { [name] = value }); + + Assert.AreEqual(host, snapshot.HostingEnvironment); + Assert.AreEqual(service, snapshot.AzureHostingService); + } + + /// Absent cloud signals use the agreed Local/Not Azure best-effort fallback. + [TestMethod] + public void CaptureHosting_AbsentSignalsAreLocal() + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new()); + + Assert.AreEqual('M', snapshot.RunningInContainer); + Assert.AreEqual('L', snapshot.HostingEnvironment); + Assert.AreEqual('N', snapshot.AzureHostingService); + } + + /// Conflicting automatic signals are not arbitrarily assigned to one cloud or Azure service. + [TestMethod] + public void CaptureHosting_ConflictingSignalsAreMissing() + { + ApplicationNameTelemetryEnvironment clouds = Capture(new() + { + ["WEBSITE_SITE_NAME"] = "example", + ["AWS_EXECUTION_ENV"] = "example", + }); + Assert.AreEqual('M', clouds.HostingEnvironment); + Assert.AreEqual('M', clouds.AzureHostingService); + + ApplicationNameTelemetryEnvironment azureServices = Capture(new() + { + ["WEBSITE_SITE_NAME"] = "example", + ["CONTAINER_APP_NAME"] = "example", + }); + Assert.AreEqual('A', azureServices.HostingEnvironment); + Assert.AreEqual('M', azureServices.AzureHostingService); + } + + /// Portable Knative variables are not evidence that a Kubernetes cluster is on GCP. + [TestMethod] + public void Review_KnativeIsNotProofOfGcp() + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new() + { + ["K_SERVICE"] = "example", + ["KUBERNETES_SERVICE_HOST"] = "example", + }); + + Assert.AreEqual('O', snapshot.HostingEnvironment); + Assert.AreEqual('N', snapshot.AzureHostingService); + } + + /// Automatic cloud inference must not replace an explicit missing/not-Azure service. + [DataTestMethod] + [DataRow("M", false, 'W', 'M')] + [DataRow("N", true, 'M', 'N')] + public void Review_ServiceOverrideWinsOverAutomaticHosting(string service, bool conflictingClouds, char expectedHost, char expectedService) + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new() + { + [ApplicationNameTelemetry.AZURE_HOSTING_SERVICE_ENV_VAR] = service, + ["AWS_EXECUTION_ENV"] = "example", + ["CLOUD_RUN_JOB"] = conflictingClouds ? "example" : null, + }); + + Assert.AreEqual(expectedHost, snapshot.HostingEnvironment); + Assert.AreEqual(expectedService, snapshot.AzureHostingService); + } + + /// Not Azure suppresses Azure detection but still permits a different cloud signal. + [DataTestMethod] + [DataRow(false, 'L')] + [DataRow(true, 'W')] + public void CaptureAzureService_NotAzureOverrideWinsOverDetection(bool aws, char expectedHost) + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new() + { + [ApplicationNameTelemetry.AZURE_HOSTING_SERVICE_ENV_VAR] = "NotAzure", + ["WEBSITE_SITE_NAME"] = "stale-azure-hint", + ["AWS_EXECUTION_ENV"] = aws ? "example" : null, + }); + + Assert.AreEqual(expectedHost, snapshot.HostingEnvironment); + Assert.AreEqual('N', snapshot.AzureHostingService); + } + + /// Explicit hosting overrides win over automatic Azure signals and accept names or codes. + [DataTestMethod] + [DataRow("L", 'L', 'N')] + [DataRow(" local ", 'L', 'N')] + [DataRow("A", 'A', 'S')] + [DataRow("Azure", 'A', 'S')] + [DataRow("W", 'W', 'N')] + [DataRow("aws", 'W', 'N')] + [DataRow("G", 'G', 'N')] + [DataRow("GCP", 'G', 'N')] + [DataRow("O", 'O', 'N')] + [DataRow("Other", 'O', 'N')] + [DataRow("M", 'M', 'M')] + [DataRow("Missing", 'M', 'M')] + [DataRow("invalid", 'M', 'M')] + [DataRow(" ", 'A', 'S')] + public void CaptureHosting_OverrideTakesPrecedence(string value, char host, char service) + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new() + { + [ApplicationNameTelemetry.HOSTING_ENVIRONMENT_ENV_VAR] = value, + ["WEBSITE_SITE_NAME"] = "example", + }); + + Assert.AreEqual(host, snapshot.HostingEnvironment); + Assert.AreEqual(service, snapshot.AzureHostingService); + } + + /// A service override supports services without universal runtime environment markers. + [DataTestMethod] + [DataRow("C", 'C')] + [DataRow("ContainerApps", 'C')] + [DataRow(" container apps ", 'C')] + [DataRow("K", 'K')] + [DataRow("aks", 'K')] + [DataRow("S", 'S')] + [DataRow("AppService", 'S')] + [DataRow("App Service", 'S')] + [DataRow("I", 'I')] + [DataRow("ACI", 'I')] + [DataRow("ContainerInstances", 'I')] + [DataRow("Container Instances", 'I')] + [DataRow("O", 'O')] + [DataRow("Other", 'O')] + public void CaptureAzureService_OverrideAlsoIdentifiesAzure(string value, char service) + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new() + { + [ApplicationNameTelemetry.AZURE_HOSTING_SERVICE_ENV_VAR] = value, + }); + + Assert.AreEqual('A', snapshot.HostingEnvironment); + Assert.AreEqual(service, snapshot.AzureHostingService); + } + + /// Overrides remain internally consistent, with hosting taking precedence over service. + [DataTestMethod] + [DataRow("AWS", "AKS", 'W', 'N')] + [DataRow("Local", "ContainerApps", 'L', 'N')] + [DataRow("Missing", "AKS", 'M', 'M')] + [DataRow("Azure", "N", 'A', 'M')] + [DataRow("Azure", "NotAzure", 'A', 'M')] + [DataRow("Azure", "Not Azure", 'A', 'M')] + [DataRow("Azure", "M", 'A', 'M')] + [DataRow("Azure", "Missing", 'A', 'M')] + [DataRow("Azure", "invalid", 'A', 'M')] + [DataRow("Azure", null, 'A', 'M')] + [DataRow(null, "NotAzure", 'L', 'N')] + [DataRow(null, "Not Azure", 'L', 'N')] + public void CaptureHosting_OverrideCombinations(string hostOverride, string serviceOverride, char host, char service) + { + ApplicationNameTelemetryEnvironment snapshot = Capture(new() + { + [ApplicationNameTelemetry.HOSTING_ENVIRONMENT_ENV_VAR] = hostOverride, + [ApplicationNameTelemetry.AZURE_HOSTING_SERVICE_ENV_VAR] = serviceOverride, + }); + + Assert.AreEqual(host, snapshot.HostingEnvironment); + Assert.AreEqual(service, snapshot.AzureHostingService); + } + + /// Public encoding rereads explicit overrides rather than keeping a process-wide snapshot. + [TestMethod] + [DoNotParallelize] + public void EncodeGeneral_RereadsEnvironment() + { + string variable = ApplicationNameTelemetry.HOSTING_ENVIRONMENT_ENV_VAR; + string original = Environment.GetEnvironmentVariable(variable); + try + { + RuntimeConfig config = Config(new(DatabaseType.MSSQL, SQL_PASSWORD)); + Environment.SetEnvironmentVariable(variable, "Local"); + Assert.AreEqual('L', Sections(ApplicationNameTelemetry.EncodeTelemetryString(config))[1][2]); + Environment.SetEnvironmentVariable(variable, "AWS"); + Assert.AreEqual('W', Sections(ApplicationNameTelemetry.EncodeTelemetryString(config))[1][2]); + } + finally + { + Environment.SetEnvironmentVariable(variable, original); + } + } + + /// Data-source count includes loaded children and excludes nonexistent file references. + [DataTestMethod] + [DataRow(false, 0, 'M')] + [DataRow(true, 0, '0')] + [DataRow(false, 1, '0')] + [DataRow(true, 1, '1')] + [DataRow(false, 2, '1')] + public void EncodeMultipleDataSources_CountsParsedSources(bool defaultSource, int childCount, char expected) + { + string directory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(directory); + try + { + DataSource source = new(DatabaseType.MSSQL, SQL_MI); + List files = new() { Path.Combine(directory, "missing.json") }; + for (int index = 0; index < childCount; index++) + { + string path = Path.Combine(directory, $"child-{index}.json"); + File.WriteAllText(path, Config(source).ToJson()); + files.Add(path); + } + + RuntimeConfig config = new(Schema: "test", DataSource: defaultSource ? source : null, + Entities: new(new Dictionary()), DataSourceFiles: new(files)); + Assert.AreEqual(childCount + (defaultSource ? 1 : 0), config.ListAllDataSources().Count()); + Assert.AreEqual(expected, General(config)[4]); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + /// MI is a configured authentication fact; unspecified/default credentials are unknown. + [DataTestMethod] + [DataRow(DatabaseType.MSSQL, SQL_MI, '1')] + [DataRow(DatabaseType.DWSQL, SQL_MI, '1')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Active Directory MSI;", '1')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Active Directory Managed Identity;User ID=client-id;", '1')] + [DataRow(DatabaseType.MSSQL, SQL_PASSWORD, '0')] + [DataRow(DatabaseType.DWSQL, SQL_PASSWORD, '0')] + [DataRow(DatabaseType.MSSQL, "Server=test;Integrated Security=true;", '0')] + [DataRow(DatabaseType.MSSQL, "Server=test;User ID=test-user;", '0')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Sql Password;", '0')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Active Directory Password;", '0')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Active Directory Service Principal;", '0')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Active Directory Integrated;", '0')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Active Directory Interactive;", '0')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Active Directory Device Code Flow;", '0')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Active Directory Default;", 'M')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=Active Directory Workload Identity;", 'M')] + [DataRow(DatabaseType.MSSQL, "Server=test;", 'M')] + [DataRow(DatabaseType.MSSQL, "", 'M')] + [DataRow(DatabaseType.MSSQL, " ", 'M')] + [DataRow(DatabaseType.MSSQL, "not a connection string", 'M')] + [DataRow(DatabaseType.MSSQL, "Server=test;Authentication=unsupported;", 'M')] + [DataRow(DatabaseType.MSSQL, "Server=test;Connect Timeout=invalid;", 'M')] + [DataRow(DatabaseType.MSSQL, "Server=test;Connect Timeout=99999999999999999999;", 'M')] + [DataRow(DatabaseType.MSSQL, "Server=test;Integrated Security=invalid;", 'M')] + [DataRow(DatabaseType.MSSQL, "@env('UNRESOLVED_CONNECTION')", 'M')] + [DataRow(DatabaseType.MSSQL, "Server=test;User ID=user;Password=@akv('unresolved');", 'M')] + [DataRow(DatabaseType.MSSQL, "Server=test;User ID=@env('unresolved');", 'M')] + [DataRow(DatabaseType.PostgreSQL, "Host=test;Username=user;Password=test-only;", '0')] + [DataRow(DatabaseType.PostgreSQL, "Host=test;Username=user;", 'M')] + [DataRow(DatabaseType.PostgreSQL, "not a connection string", 'M')] + [DataRow(DatabaseType.PostgreSQL, "Host=test;Timeout=invalid;", 'M')] + [DataRow(DatabaseType.PostgreSQL, "Host=test;Password=@akv('unresolved');", 'M')] + [DataRow(DatabaseType.MySQL, "Server=test;User ID=user;Pwd=test-only;", '0')] + [DataRow(DatabaseType.MySQL, "Server=test;User ID=user;Password=test-only;", '0')] + [DataRow(DatabaseType.MySQL, "Server=test;User ID=user;", 'M')] + [DataRow(DatabaseType.MySQL, "not a connection string", 'M')] + [DataRow(DatabaseType.MySQL, "Server=test;Pwd=@env('unresolved');", 'M')] + [DataRow(DatabaseType.CosmosDB_NoSQL, "AccountEndpoint=https://unit-test.invalid;AccountKey=test-only;", '0')] + [DataRow(DatabaseType.CosmosDB_NoSQL, "AccountEndpoint=https://unit-test.invalid;", 'M')] + [DataRow(DatabaseType.CosmosDB_NoSQL, "not a connection string", 'M')] + [DataRow(DatabaseType.CosmosDB_NoSQL, "AccountEndpoint=https://unit-test.invalid;AccountKey=@akv('unresolved');", 'M')] + [DataRow(DatabaseType.CosmosDB_PostgreSQL, "Server=test;", 'M')] + public void EncodeManagedIdentity_UsesConfigurationOnly(DatabaseType type, string connectionString, char expected) + { + DataSource source = new(type, connectionString); + Assert.AreEqual(expected, General(Config(source), source)[5]); + } + + /// Per-pool MI follows the live source; only CLI/no-live-source encoding uses the default. + [TestMethod] + public void EncodeManagedIdentity_UsesLiveSourceOrDefault() + { + DataSource source = new(DatabaseType.MSSQL, SQL_MI); + DataSource live = new(DatabaseType.PostgreSQL, "Host=test;Password=test-only;"); + RuntimeConfig config = Config(source); + + Assert.AreEqual('1', General(config)[5]); + Assert.AreEqual('0', General(config, live)[5]); + Assert.AreEqual('M', General(Config(null))[5]); + Assert.AreEqual('1', General(Config(null), source)[5]); + } + + /// OBO request pools and metadata MI pools share one source token, so MI is inconclusive. + [TestMethod] + public void EncodeManagedIdentity_OboWithMiIsMixed() + { + DataSource source = new(DatabaseType.MSSQL, SQL_MI) + { + UserDelegatedAuth = new(Enabled: true), + }; + Assert.AreEqual('M', General(Config(source), source)[5]); + } + + /// One token covers both OBO requests and metadata; do not claim a mixed/unknown mode is non-MI. + [DataTestMethod] + [DataRow(SQL_MI, 'M')] + [DataRow("Server=unit-test.invalid;", 'M')] + [DataRow(SQL_PASSWORD, '0')] + public void Review_OboMetadataAuthenticationIsNotMisclassified(string connectionString, char expected) + { + DataSource source = new(DatabaseType.MSSQL, connectionString) { UserDelegatedAuth = new(Enabled: true) }; + + Assert.AreEqual(expected, General(Config(source), source)[5]); + } + + /// Alias handling must agree with the actual provider's effective password. + [DataTestMethod] + [DataRow("Server=test;User ID=user;Password=old;Pwd=;")] + [DataRow("Server=test;User ID=user;Pwd=old;Password=;")] + [DataRow("Server=test;User ID=user;Password=old;Pwd=;Password=new;")] + [DataRow("Server=test;User ID=user;Pwd=old;Password=;Pwd=new;")] + public void Review_MySqlPasswordAliasesFollowProviderSemantics(string connectionString) + { + MySqlConnectionStringBuilder provider = new(connectionString); + DataSource source = new(DatabaseType.MySQL, connectionString); + char expected = string.IsNullOrEmpty(provider.Password) ? 'M' : '0'; + + Assert.AreEqual(expected, General(Config(source), source)[5]); + } + + /// Ordinary application-name text must not be mistaken for an unresolved secret reference. + [TestMethod] + public void Review_LiteralReferenceTextDoesNotSuppressExplicitAuthentication() + { + DataSource source = new(DatabaseType.MSSQL, SQL_MI + "Application Name=tag@env(label);"); + + Assert.AreEqual('1', General(Config(source), source)[5]); + } + + /// Telemetry must not make an otherwise valid application name fail client validation. + [DataTestMethod] + [DataRow(DatabaseType.MSSQL, false, 64)] + [DataRow(DatabaseType.MSSQL, false, 65)] + [DataRow(DatabaseType.MSSQL, false, 66)] + [DataRow(DatabaseType.MSSQL, false, 70)] + [DataRow(DatabaseType.MSSQL, true, 67)] + [DataRow(DatabaseType.DWSQL, false, 70)] + [DataRow(DatabaseType.MSSQL, false, 128)] + [DataRow(DatabaseType.MSSQL, false, 127)] + public void Review_SqlApplicationNameFitsProviderLimit(DatabaseType type, bool hosted, int customLength) + { + Environment.SetEnvironmentVariable(ProductInfo.DAB_APP_NAME_ENV, hosted ? "dab_hosted" : null); + string customName = new('a', customLength); + SqlConnectionStringBuilder original = new(SQL_PASSWORD) { ApplicationName = customName }; + using SqlConnection validOriginal = new(original.ConnectionString); + DataSource source = new(type, original.ConnectionString); + string updated = RuntimeConfigLoader.GetConnectionStringWithApplicationName(source.ConnectionString, Config(source), source); + + using SqlConnection actual = new(updated); + SqlConnectionStringBuilder builder = new(actual.ConnectionString); + Assert.IsTrue(builder.ApplicationName.Length <= 128); + Assert.IsTrue(builder.ApplicationName.StartsWith(customName, StringComparison.Ordinal)); + Assert.AreEqual(updated, RuntimeConfigLoader.GetConnectionStringWithApplicationName(updated, Config(source), source)); + } + + /// The actual executor keeps metadata valid and per-user hashes complete after token growth. + [TestMethod] + public void Review_OboMetadataAndRequestNamesRespectClientLimit() + { + string customName = new('a', 70); + DataSource source = new(DatabaseType.MSSQL, "Server=unit-test.invalid;Application Name=" + customName) + { + UserDelegatedAuth = new(Enabled: true), + }; + RuntimeConfig config = Config(source); + DataSource updated = source with + { + ConnectionString = RuntimeConfigLoader.GetConnectionStringWithApplicationName(source.ConnectionString, config, source), + }; + config.UpdateDataSourceNameToDataSource(config.DefaultDataSourceName, updated); + config = config with { DataSource = updated }; + using FileSystemRuntimeConfigLoader loader = new(new MockFileSystem()) { RuntimeConfig = config }; + using RuntimeConfigProvider provider = new(loader); + HttpContextAccessor accessor = new(); + Mock parser = new(provider); + MsSqlQueryExecutor executor = new(provider, parser.Object, NullLogger.Instance, accessor); + + using SqlConnection metadata = executor.CreateConnection(config.DefaultDataSourceName); + Assert.IsTrue(new SqlConnectionStringBuilder(metadata.ConnectionString).ApplicationName.StartsWith(customName, StringComparison.Ordinal)); + Assert.AreEqual('M', GeneralFromConnectionString(metadata.ConnectionString)[5]); + + accessor.HttpContext = new DefaultHttpContext + { + User = new ClaimsPrincipal(new ClaimsIdentity(new[] + { + new Claim("iss", "https://issuer.invalid"), + new Claim("oid", "test-user-one"), + }, "test")), + }; + using SqlConnection request = executor.CreateConnection(config.DefaultDataSourceName); + string name = new SqlConnectionStringBuilder(request.ConnectionString).ApplicationName; + Assert.IsTrue(name.Length <= 128); + Assert.AreEqual(22, name.IndexOf('|'), "Per-user isolation hash must not be truncated."); + Assert.IsTrue(name[23..].StartsWith(customName, StringComparison.Ordinal)); + Assert.IsTrue(ApplicationNameTelemetry.Decode(name).Any(line => line.Contains("managed-identity: M", StringComparison.Ordinal))); + + accessor.HttpContext.User = new ClaimsPrincipal(new ClaimsIdentity(new[] + { + new Claim("iss", "https://issuer.invalid"), new Claim("oid", "test-user-two"), + }, "test")); + using SqlConnection otherUser = executor.CreateConnection(config.DefaultDataSourceName); + Assert.AreNotEqual(name[..22], new SqlConnectionStringBuilder(otherUser.ConnectionString).ApplicationName[..22]); + } + + /// + /// PostgreSQL normalizes non-printable UTF-8 bytes to ASCII hex escapes before publishing a + /// 63-byte statistics value. Decode only the fields that survive, not every Runtime field. + /// + [DataTestMethod] + [DataRow(false, "", 6, 20, 14)] + [DataRow(true, "", 6, 20, 13)] + [DataRow(false, "用户", 6, 12, 0)] + [DataRow(true, "用户", 6, 9, 0)] + [DataRow(true, "abcdefghijklmnopqrst", 6, 13, 0)] + [DataRow(true, "\t", 6, 20, 8)] + [DataRow(true, "用户甲", 4, 0, 0)] + [DataRow(true, "用户用户", 0, 0, 0)] + [DataRow(true, "用户用户用户用户", 0, 0, 0)] + public void Review_PostgresNormalizationDecodesOnlySurvivingFields( + bool hosted, string customName, int generalFields, int runtimeFields, int entityFields) + { + Environment.SetEnvironmentVariable(ProductInfo.DAB_APP_NAME_ENV, hosted ? "dab_hosted" : null); + DataSource source = new(DatabaseType.PostgreSQL, "Host=unit-test.invalid;Password=test-only;"); + string encoded = ApplicationNameTelemetry.EncodeTelemetryString(Config(source), source, _environment); + // A fixed-width version keeps the boundary expectations independent of future release numbers. + string token = (hosted ? "dab_hosted_1.2.3" : "dab_oss_1.2.3") + encoded[encoded.IndexOf('+')..]; + string name = (customName.Length == 0 ? string.Empty : customName + ",") + token; + string serverName = SimulatePostgres16ApplicationName(name); + IReadOnlyList decoded = ApplicationNameTelemetry.Decode(serverName); + IReadOnlyList fullDecoded = ApplicationNameTelemetry.Decode(token); + + AssertSurvivingFields("General", generalFields); + AssertSurvivingFields("Runtime", runtimeFields); + AssertSurvivingFields("Entity", entityFields); + Assert.IsTrue(Encoding.UTF8.GetByteCount(serverName) <= 63); + + if (hosted && customName == "用户") + { + // PostgreSQL 16.15 server-observed regression: six UTF-8 prefix bytes expand to 24. + Assert.AreEqual(@"\xe7\x94\xa8\xe6\x88\xb7,dab_hosted_1.2.3+XXPX|L1AC00|MMMM00MMM", serverName); + } + + void AssertSurvivingFields(string section, int expectedCount) + { + string prefix = section + " >"; + string[] actual = decoded.Where(line => line.StartsWith(prefix, StringComparison.Ordinal)).ToArray(); + string[] expected = fullDecoded.Where(line => line.StartsWith(prefix, StringComparison.Ordinal)).Take(expectedCount).ToArray(); + Assert.AreEqual(expectedCount, actual.Length, $"Surviving {section} fields in '{serverName}'."); + CollectionAssert.AreEqual(expected, actual, $"{section} must decode the exact surviving fields without shifting positions."); + } + } + + /// Pin the test model's escaping to known server values instead of testing only its own calculations. + [DataTestMethod] + [DataRow(" !~\\", " !~\\")] + [DataRow("用户", @"\xe7\x94\xa8\xe6\x88\xb7")] + [DataRow("\t\n\r\u001f\u007f", @"\x09\x0a\x0d\x1f\x7f")] + [DataRow("\U0001F680", @"\xf0\x9f\x9a\x80")] + public void PostgresNormalizationModel_EscapesUtf8Bytes(string value, string expected) + { + Assert.AreEqual(expected, SimulatePostgres16ApplicationName(value)); + } + + /// Initial GUC clipping must not keep part of a multibyte character that straddles byte 63. + [DataTestMethod] + [DataRow(60, "用", @"\xe")] + [DataRow(61, "用", "")] + [DataRow(62, "用", "")] + [DataRow(59, "\U0001F680", @"\xf0")] + [DataRow(60, "\U0001F680", "")] + public void PostgresNormalizationModel_ClipsBeforeEscaping(int asciiLength, string suffix, string survivingEscape) + { + string prefix = new('a', asciiLength); + Assert.AreEqual(prefix + survivingEscape, SimulatePostgres16ApplicationName(prefix + suffix)); + } + + /// The effective connection string, not a config placeholder, supplies per-pool auth. + [DataTestMethod] + [DataRow(DatabaseType.MSSQL, SQL_MI, '1')] + [DataRow(DatabaseType.DWSQL, SQL_MI, '1')] + [DataRow(DatabaseType.MSSQL, SQL_PASSWORD, '0')] + [DataRow(DatabaseType.PostgreSQL, "Host=test;Password=test-only;", '0')] + public void ConnectionStringOverride_ControlsManagedIdentity(DatabaseType type, string connectionString, char expected) + { + DataSource source = new(type, type == DatabaseType.PostgreSQL ? "Host=placeholder;" : "Server=placeholder;"); + RuntimeConfig config = Config(source); + string updated = RuntimeConfigLoader.GetConnectionStringWithApplicationName(connectionString, config, source); + + Assert.AreEqual(expected, GeneralFromConnectionString(updated)[5]); + Assert.AreEqual(source.ConnectionString, config.DataSource.ConnectionString, "Encoding must not mutate the input config."); + } + + /// File-load overrides must reach the same authentication detection used for late config. + [TestMethod] + public void ParseConfig_ConnectionStringOverrideControlsManagedIdentity() + { + RuntimeConfig original = Config(new(DatabaseType.MSSQL, SQL_PASSWORD)); + bool parsed = RuntimeConfigLoader.TryParseConfig(original.ToJson(), out RuntimeConfig config, out _, + new(doReplaceEnvVar: true), connectionString: SQL_MI); + + Assert.IsTrue(parsed); + Assert.AreEqual('1', GeneralFromConnectionString(config.DataSource.ConnectionString)[5]); + } + + /// The hosted path must describe the separately supplied connection string. + [TestMethod] + public async Task HostedConnectionStringOverride_ControlsManagedIdentity() + { + FileSystemRuntimeConfigLoader loader = new(new MockFileSystem()); + using RuntimeConfigProvider provider = new(loader); + bool initialized = await provider.Initialize(Config(new(DatabaseType.MSSQL, SQL_PASSWORD)).ToJson(), + graphQLSchema: null, connectionString: SQL_MI, accessToken: null, + replacementSettings: new(doReplaceEnvVar: false)); + + Assert.IsTrue(initialized); + Assert.AreEqual('1', GeneralFromConnectionString(provider.GetConfig().DataSource.ConnectionString)[5]); + } + + /// The runtime opt-out omits the new General section along with the rest of the payload. + [TestMethod] + public void BuildApplicationNameSegment_OptOutOmitsGeneral() + { + Environment.SetEnvironmentVariable(ApplicationNameTelemetry.OPT_OUT_ENV_VAR, "1"); + DataSource source = new(DatabaseType.MSSQL, SQL_MI); + + Assert.AreEqual(ProductInfo.DAB_USER_AGENT, ApplicationNameTelemetry.BuildApplicationNameSegment(Config(source), source)); + } + + /// Every defined General alphabet, including macOS versus missing, has a decoder legend. + [DataTestMethod] + [DataRow(0, 'W', "operating-system", "Windows")] + [DataRow(0, 'L', "operating-system", "Linux")] + [DataRow(0, 'M', "operating-system", "macOS")] + [DataRow(0, 'O', "operating-system", "Other")] + [DataRow(0, 'U', "operating-system", "Unknown")] + [DataRow(0, 'Z', "operating-system", "unrecognized")] + [DataRow(1, '0', "running-in-container", "disabled/no")] + [DataRow(1, '1', "running-in-container", "enabled/yes")] + [DataRow(1, 'M', "running-in-container", "missing")] + [DataRow(2, 'L', "hosting-environment", "Local")] + [DataRow(2, 'A', "hosting-environment", "Azure")] + [DataRow(2, 'W', "hosting-environment", "AWS")] + [DataRow(2, 'G', "hosting-environment", "GCP")] + [DataRow(2, 'O', "hosting-environment", "Other")] + [DataRow(2, 'M', "hosting-environment", "missing")] + [DataRow(2, 'Z', "hosting-environment", "unrecognized")] + [DataRow(3, 'C', "azure-hosting-service", "Container Apps")] + [DataRow(3, 'K', "azure-hosting-service", "AKS")] + [DataRow(3, 'S', "azure-hosting-service", "App Service")] + [DataRow(3, 'I', "azure-hosting-service", "ACI")] + [DataRow(3, 'O', "azure-hosting-service", "Other")] + [DataRow(3, 'N', "azure-hosting-service", "Not Azure")] + [DataRow(3, 'M', "azure-hosting-service", "missing")] + [DataRow(3, 'Z', "azure-hosting-service", "unrecognized")] + [DataRow(4, '0', "multiple-data-sources", "disabled/no")] + [DataRow(4, '1', "multiple-data-sources", "enabled/yes")] + [DataRow(4, 'M', "multiple-data-sources", "missing")] + [DataRow(5, '0', "managed-identity", "disabled/no")] + [DataRow(5, '1', "managed-identity", "enabled/yes")] + [DataRow(5, 'M', "managed-identity", "missing")] + public void DecodeGeneral_DescribesAlphabet(int position, char value, string name, string description) + { + char[] general = "W0LN00".ToCharArray(); + general[position] = value; + IReadOnlyList lines = ApplicationNameTelemetry.Decode($"dab_oss_1.2.3+XXSX|{new string(general)}|M|M+"); + + CollectionAssert.Contains(lines.ToArray(), $"General > {name}: {value} ({description})"); + } + + /// Old tokens with an empty General section retain the same runtime/entity positions. + [TestMethod] + public void DecodeGeneral_EmptyLegacySectionIsCompatible() + { + IReadOnlyList lines = ApplicationNameTelemetry.Decode("dab_oss_1.2.3+XXSX||1|0+"); + + Assert.IsFalse(lines.Any(line => line.StartsWith("General >", StringComparison.Ordinal))); + CollectionAssert.Contains(lines.ToArray(), "Runtime > runtime.rest.enabled: 1 (enabled/yes)"); + CollectionAssert.Contains(lines.ToArray(), "Entity > entities.any.table: 0 (disabled/no)"); + } + + /// All truncation points within General decode only the surviving positions. + [DataTestMethod] + [DataRow(0)] + [DataRow(1)] + [DataRow(2)] + [DataRow(3)] + [DataRow(4)] + [DataRow(5)] + [DataRow(6)] + public void DecodeGeneral_TruncatedSectionIsTolerated(int survivingFlags) + { + IReadOnlyList lines = ApplicationNameTelemetry.Decode("dab_hosted_1.2.3+XXSX|" + "L1AC01"[..survivingFlags]); + + Assert.AreEqual(survivingFlags, lines.Count(line => line.StartsWith("General >", StringComparison.Ordinal))); + Assert.IsFalse(lines.Any(line => line.StartsWith("Runtime >", StringComparison.Ordinal))); + Assert.IsFalse(lines.Any(line => line.StartsWith("Entity >", StringComparison.Ordinal))); + } + + /// Neither host values nor connection-string values appear in the token or decoded output. + [TestMethod] + public void EncodeGeneral_DoesNotExposeEnvironmentOrCredentials() + { + const string sensitiveValue = "private-value-never-emit"; + ApplicationNameTelemetryEnvironment environment = Capture(new() + { + ["WEBSITE_SITE_NAME"] = sensitiveValue, + [ApplicationNameTelemetry.AZURE_HOSTING_SERVICE_ENV_VAR] = sensitiveValue, + }); + DataSource source = new(DatabaseType.MSSQL, $"Server={sensitiveValue};User ID={sensitiveValue};Password={sensitiveValue};"); + string token = ApplicationNameTelemetry.EncodeTelemetryString(Config(source), source, environment); + + Assert.IsFalse(token.Contains(sensitiveValue, StringComparison.Ordinal)); + Assert.IsFalse(string.Join('\n', ApplicationNameTelemetry.Decode(token)).Contains(sensitiveValue, StringComparison.Ordinal)); + } + + /// + /// Test-only model for UTF-8 PostgreSQL 16 with the standard NAMEDATALEN=64: + /// GUC_IS_NAME clips on a character boundary, check_application_name/pg_clean_ascii hex-escapes + /// bytes outside 0x20..0x7e, then pgstat_report_appname clips the expanded ASCII string to 63 bytes. + /// The last clip may split a four-character escape. Do not add this normalization to DAB itself. + /// See PostgreSQL REL_16_15: src/backend/utils/misc/guc.c, src/backend/commands/variable.c, + /// src/common/string.c, and src/backend/utils/activity/backend_status.c. + /// + private static string SimulatePostgres16ApplicationName(string value) + { + const int maxBytes = 63; + StringBuilder clippedName = new(); + int bytes = 0; + foreach (Rune rune in value.EnumerateRunes()) + { + if (bytes + rune.Utf8SequenceLength > maxBytes) + { + break; + } + + bytes += rune.Utf8SequenceLength; + clippedName.Append(rune.ToString()); + } + + StringBuilder escapedName = new(); + foreach (byte item in Encoding.UTF8.GetBytes(clippedName.ToString())) + { + if (item is >= 32 and <= 126) + { + escapedName.Append((char)item); + } + else + { + escapedName.Append("\\x").Append(item.ToString("x2", CultureInfo.InvariantCulture)); + } + } + + return escapedName.ToString(0, Math.Min(escapedName.Length, maxBytes)); + } + + private static ApplicationNameTelemetryEnvironment Capture(Dictionary values) => + ApplicationNameTelemetryEnvironment.Capture(name => values.TryGetValue(name, out string value) ? value : null); + + private static RuntimeConfig Config(DataSource source) => + new(Schema: "test", DataSource: source, Entities: new(new Dictionary())); + + private static string General(RuntimeConfig config, DataSource liveDataSource = null) => + Sections(ApplicationNameTelemetry.EncodeTelemetryString(config, liveDataSource, _environment))[1]; + + private static string[] Sections(string token) + { + string[] sections = token[(token.IndexOf('+') + 1)..^1].Split('|'); + Assert.AreEqual(4, sections.Length); + Assert.AreEqual(6, sections[1].Length); + return sections; + } + + private static string GeneralFromConnectionString(string connectionString) + { + DbConnectionStringBuilder builder = new() { ConnectionString = connectionString }; + return Sections((string)builder["Application Name"])[1]; + } +} diff --git a/src/Service.Tests/UnitTests/ApplicationNameTelemetryTests.cs b/src/Service.Tests/UnitTests/ApplicationNameTelemetryTests.cs index 9a7a94207a..aa4dac91ad 100644 --- a/src/Service.Tests/UnitTests/ApplicationNameTelemetryTests.cs +++ b/src/Service.Tests/UnitTests/ApplicationNameTelemetryTests.cs @@ -41,7 +41,7 @@ public void ResetEnvironment() /// /// The encoded string must start with the product user agent, be wrapped in '+', and contain - /// four positional sections: context, an empty reserved general section, runtime, and entity. + /// four positional sections: context, general, runtime, and entity. /// [TestMethod] public void EncodeTelemetryString_HasExpectedShape() @@ -361,11 +361,11 @@ public void Decode_HostedMarker_IsRecognizedWithoutHostedEnvironment() [TestMethod] public void Decode_FutureGeneralSettings_DoNotShiftKnownSections() { - string telemetry = ApplicationNameTelemetry.EncodeTelemetryString(BuildConfig(), Source(DatabaseType.MSSQL)); - string telemetryWithGeneralSettings = telemetry.Replace("||", "|10|", StringComparison.Ordinal); + const string telemetryWithGeneralSettings = "dab_oss_1.2.3+XXSX|L1AC00Z|M|M+"; IReadOnlyList lines = ApplicationNameTelemetry.Decode(telemetryWithGeneralSettings); + Assert.IsTrue(lines.Any(l => l.Contains("General > [position 7]: Z", StringComparison.Ordinal)), string.Join(Environment.NewLine, lines)); Assert.IsTrue(lines.Any(l => l.Contains("Runtime > runtime.rest.enabled: M", StringComparison.Ordinal)), string.Join(Environment.NewLine, lines)); Assert.IsTrue(lines.Any(l => l.Contains("Entity > entities.any.table: M", StringComparison.Ordinal)), string.Join(Environment.NewLine, lines)); } @@ -571,7 +571,7 @@ private static (string context, string runtime, string entity) Sections(string t string payload = telemetry[(firstPlus + 1)..].TrimEnd('+'); string[] parts = payload.Split('|'); Assert.AreEqual(4, parts.Length, $"Telemetry payload should have four positional sections but was '{payload}'."); - Assert.AreEqual(string.Empty, parts[1], "The reserved general-settings section should be empty."); + Assert.AreEqual(6, parts[1].Length, "general width"); return (parts[0], parts[2], parts[3]); } }