Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 108 additions & 10 deletions docs/design/application-name-telemetry.md

Large diffs are not rendered by default.

132 changes: 132 additions & 0 deletions src/Cli.Tests/EndToEndTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,12 @@ public void TestAppNameEncodeAndDecode()
string telemetry = _fileSystem.File.ReadAllText("appname-out.txt");
Assert.IsTrue(telemetry.StartsWith("dab_oss_"), telemetry);
Assert.IsTrue(telemetry.EndsWith("+"), telemetry);
string[] sections = telemetry[(telemetry.IndexOf('+') + 1)..^1].Split('|');
Assert.AreEqual(4, sections.Length, "General settings must not shift runtime/entity sections.");
Assert.AreEqual("XXXX", sections[0], "The CLI still has no live connection context.");
Assert.AreEqual(6, sections[1].Length, "The CLI should emit all six general flags.");
Assert.AreEqual('0', sections[1][4], "One parsed data source is not multiple data sources.");
Assert.AreEqual('0', sections[1][5], "The default data source uses explicit SQL credentials, not MI.");

// Act: decode the produced string back into a human-readable description.
int decodeCode = Program.Execute(
Expand All @@ -180,6 +186,12 @@ public void TestAppNameEncodeAndDecode()
Assert.AreEqual(0, decodeCode, "appname --decode should succeed");
string decoded = _fileSystem.File.ReadAllText("appname-decoded.txt");
Assert.IsTrue(decoded.Contains("Version: dab_oss_"), decoded);
Assert.IsTrue(decoded.Contains("General > operating-system:"), decoded);
Assert.IsTrue(decoded.Contains("General > running-in-container:"), decoded);
Assert.IsTrue(decoded.Contains("General > hosting-environment:"), decoded);
Assert.IsTrue(decoded.Contains("General > azure-hosting-service:"), decoded);
Assert.IsTrue(decoded.Contains("General > multiple-data-sources: 0"), decoded);
Assert.IsTrue(decoded.Contains("General > managed-identity: 0"), decoded);
Assert.IsTrue(decoded.Contains("runtime.rest.enabled"), decoded);
Assert.IsTrue(decoded.Contains("entities.any.table"), decoded);
}
Expand Down Expand Up @@ -209,6 +221,126 @@ public void TestAppNameEncodeSupportsAbsolutePaths()
Assert.IsTrue(_fileSystem.File.ReadAllText(outputPath).StartsWith("dab_oss_", StringComparison.Ordinal));
}

/// <summary>Inspection must not require a resolvable connection string or create a Key Vault client.</summary>
[DataTestMethod]
[DataRow("@env('DAB_GENERAL_REVIEW_UNSET_CONNECTION')")]
[DataRow("not a connection string")]
[DataRow("@akv('database-connection')")]
public void TestAppNameReview_InspectsUnresolvedCredentialsOffline(string connectionString)
{
// The invalid URI fails before any network access if the runtime AKV resolver is invoked.
string configJson = $$"""
{
"data-source": { "database-type": "mssql", "connection-string": "{{connectionString}}" },
"azure-key-vault": { "endpoint": "not-a-valid-vault-uri" },
"entities": {}
}
""";
_fileSystem!.File.WriteAllText("appname-review.json", configJson);

int code = Program.Execute(
new[] { "appname", "--config", "appname-review.json", "--output", "appname-review.txt" },
_cliLogger!, _fileSystem, _runtimeConfigLoader!);

Assert.AreEqual(CliReturnCode.SUCCESS, code);
string telemetry = _fileSystem.File.ReadAllText("appname-review.txt");
string[] sections = telemetry[(telemetry.IndexOf('+') + 1)..^1].Split('|');
Assert.AreEqual('M', sections[1][5]);
}

/// <summary>Inspection applies the offline policy recursively and preserves global/runtime semantics.</summary>
[TestMethod]
public void TestAppNameReview_NestedConfigurationsAreOffline()
{
const string root = """
{
"data-source-files": ["appname-child.json", "appname-missing.json"],
"runtime": { "rest": { "enabled": false } },
"entities": {}
}
""";
const string child = """
{
"data-source": { "database-type": "mssql", "connection-string": "@akv('connection')" },
"azure-key-vault": { "endpoint": "not-a-vault-uri" },
"data-source-files": ["appname-grandchild.json"],
"entities": { "Table": { "source": { "object": "table", "type": "table" }, "permissions": [] } }
}
""";
const string grandchild = """
{
"data-source": { "database-type": "postgresql", "connection-string": "@akv('connection')" },
"azure-key-vault": { "endpoint": "not-a-vault-uri" },
"entities": { "View": { "source": { "object": "view", "type": "view" }, "permissions": [] } }
}
""";
_fileSystem!.File.WriteAllText("appname-review.json", root);
_fileSystem.File.WriteAllText("appname-child.json", child);
_fileSystem.File.WriteAllText("appname-grandchild.json", grandchild);

int code = Program.Execute(
new[] { "appname", "--config", "appname-review.json", "--output", "appname-review.txt" },
_cliLogger!, _fileSystem, _runtimeConfigLoader!);

Assert.AreEqual(CliReturnCode.SUCCESS, code);
string telemetry = _fileSystem.File.ReadAllText("appname-review.txt");
string[] sections = telemetry[(telemetry.IndexOf('+') + 1)..^1].Split('|');
Assert.AreEqual('1', sections[1][4], "Two loaded sources, not three file references.");
Assert.AreEqual('M', sections[1][5], "The root has no default connection context.");
Assert.AreEqual('0', sections[2][0], "Use the root runtime, not the child's defaults.");
Assert.AreEqual("11", sections[3][..2], "Both descendants' entities must be counted.");
Assert.IsNull(_runtimeConfigLoader!.RuntimeConfig, "Inspection must not configure or start the runtime loader.");
Assert.AreEqual(root, _fileSystem.File.ReadAllText("appname-review.json"));
Assert.AreEqual(child, _fileSystem.File.ReadAllText("appname-child.json"));
}

/// <summary>Bad/cyclic child configs fail predictably without runaway recursive loading.</summary>
[DataTestMethod]
[DataRow("{ not json }")]
[DataRow("{\"data-source-files\":[\"appname-review.json\"],\"entities\":{}}")]
public void TestAppNameReview_InvalidOrCyclicChildFails(string child)
{
_fileSystem!.File.WriteAllText("appname-review.json", "{\"data-source-files\":[\"appname-child.json\"],\"entities\":{}}");
_fileSystem.File.WriteAllText("appname-child.json", child);

int code = Program.Execute(
new[] { "appname", "--config", "appname-review.json", "--output", "appname-review.txt" },
_cliLogger!, _fileSystem, _runtimeConfigLoader!);

Assert.AreEqual(CliReturnCode.GENERAL_ERROR, code);
Assert.IsFalse(_fileSystem.File.Exists("appname-review.txt"));
}

/// <summary>Local environment substitution remains supported without resolving external secrets.</summary>
[TestMethod]
public void TestAppNameReview_UsesLocalEnvironmentAuthentication()
{
const string variable = "DAB_GENERAL_REVIEW_CONNECTION";
string? original = Environment.GetEnvironmentVariable(variable);
try
{
Environment.SetEnvironmentVariable(variable, "Server=unit-test.invalid;Authentication=Active Directory Managed Identity;");
_fileSystem!.File.WriteAllText("appname-review.json", """
{
"data-source": { "database-type": "mssql", "connection-string": "@env('DAB_GENERAL_REVIEW_CONNECTION')" },
"entities": {}
}
""");

int code = Program.Execute(
new[] { "appname", "--config", "appname-review.json", "--output", "appname-review.txt" },
_cliLogger!, _fileSystem, _runtimeConfigLoader!);

Assert.AreEqual(CliReturnCode.SUCCESS, code);
string telemetry = _fileSystem.File.ReadAllText("appname-review.txt");
Assert.AreEqual('1', telemetry[(telemetry.IndexOf('+') + 1)..^1].Split('|')[1][5]);
}
finally
{
Environment.SetEnvironmentVariable(variable, original);
}
}

/// <summary>
/// The `appname` encode path returns GENERAL_ERROR (and writes no output file) when the config
/// file cannot be found.
Expand Down
110 changes: 110 additions & 0 deletions src/Cli/AppNameConfigLoader.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Diagnostics.CodeAnalysis;
using System.IO.Abstractions;
using System.Text.Json;
using System.Text.Json.Nodes;
using Azure.DataApiBuilder.Config;
using Azure.DataApiBuilder.Config.Converters;
using Azure.DataApiBuilder.Config.ObjectModel;
using Azure.DataApiBuilder.Service.Exceptions;

namespace Cli;

/// <summary>
/// Reads a configuration for telemetry inspection without starting the runtime, creating watchers,
/// injecting connection strings, or resolving Key Vault secrets. Uses the normal property converters
/// but explicitly owns child-file traversal, because the runtime model's JSON constructor loads
/// children with the runtime's secret-resolution policy.
/// </summary>
internal static class AppNameConfigLoader
{
internal static bool TryLoadConfig(string path, IFileSystem fileSystem, [NotNullWhen(true)] out RuntimeConfig? config)
{
DeserializationVariableReplacementSettings replacements = new(
doReplaceEnvVar: true,
doReplaceAkvVar: false,
envFailureMode: EnvironmentVariableReplacementFailureMode.Ignore)
{
SkipApplicationNameInjection = true,
};

try
{
JsonSerializerOptions options = RuntimeConfigLoader.GetSerializationOptions(replacements);
HashSet<string> ancestors = new(OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal);
config = ReadConfig(path, fileSystem, options, ancestors);
return true;
}
catch (Exception ex) when (ex is JsonException or DataApiBuilderException or IOException
or UnauthorizedAccessException or ArgumentException or InvalidOperationException)
{
// Exceptions from deserialization may include credential-bearing values. The command
// reports a generic parse failure instead of exposing those details to the logger.
config = null;
return false;
}
}

private static RuntimeConfig ReadConfig(string path, IFileSystem fileSystem, JsonSerializerOptions options, HashSet<string> ancestors)
{
const int maxConfigDepth = 64;
string fullPath = fileSystem.Path.GetFullPath(path);
if (ancestors.Count >= maxConfigDepth || !ancestors.Add(fullPath))
{
throw new JsonException("Cyclic or excessively nested data-source-files.");
}

try
{
JsonObject document = JsonNode.Parse(
fileSystem.File.ReadAllText(fullPath),
documentOptions: new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip }) as JsonObject
?? throw new JsonException("Configuration must be a JSON object.");

DataSourceFiles? files = document["data-source-files"]?.Deserialize<DataSourceFiles>(options);
document.Remove("data-source-files");
RuntimeConfig root = document.Deserialize<RuntimeConfig>(options)
?? throw new JsonException("Configuration is empty.");

// Removing only the child-file property preserves the standard converters/defaults for
// every other property while preventing constructor-driven runtime secret resolution.
List<(string FileName, RuntimeConfig Config)> children = new();
foreach (string childPath in files?.SourceFiles ?? Enumerable.Empty<string>())
{
// Match runtime semantics: file paths are relative to the process working directory,
// and absent optional child files are not counted as configured data sources.
if (fileSystem.File.Exists(childPath))
{
RuntimeConfig child = ReadConfig(childPath, fileSystem, options, ancestors);
child.IsChildConfig = true;
children.Add((childPath, child));
}
}

IEnumerable<RuntimeConfig> configs = new[] { root }.Concat(children.Select(child => child.Config));
Dictionary<string, DataSource> sources = configs.SelectMany(config => config.GetDataSourceNamesToDataSourcesIterator())
.ToDictionary(entry => entry.Key, entry => entry.Value);
Dictionary<string, Entity> entities = configs.SelectMany(config => config.Entities)
.ToDictionary(entry => entry.Key, entry => entry.Value);
Dictionary<string, Autoentity> autoentities = configs.SelectMany(config => config.Autoentities)
.ToDictionary(entry => entry.Key, entry => entry.Value);
Dictionary<string, string> entitySources = configs.SelectMany(config => config.Entities
.Where(_ => config.DataSource is not null || config.ChildConfigs.Count > 0)
.Select(entity => new KeyValuePair<string, string>(entity.Key, config.GetDataSourceNameFromEntityName(entity.Key))))
.ToDictionary(entry => entry.Key, entry => entry.Value);

// Use the existing explicit-data-source-map constructor: it performs no file loading.
// This snapshot is for inspection only, never published to the runtime/provider.
RuntimeConfig result = new(root.Schema, root.DataSource!, root.Runtime!, new(entities),
root.DefaultDataSourceName, sources, entitySources, files, root.AzureKeyVault, new(autoentities));
result.ChildConfigs.AddRange(children);
return result;
}
finally
{
ancestors.Remove(fullPath);
}
}
}
8 changes: 4 additions & 4 deletions src/Cli/Commands/AppNameOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using Azure.DataApiBuilder.Config;
using Azure.DataApiBuilder.Config.ObjectModel;
using Azure.DataApiBuilder.Config.Telemetry;
using Azure.DataApiBuilder.Core.Configurations;
using Cli.Constants;
using CommandLine;
using Microsoft.Extensions.Logging;
Expand Down Expand Up @@ -58,14 +57,15 @@ public int Handler(ILogger logger, FileSystemRuntimeConfigLoader loader, IFileSy
// We intentionally do NOT run full `validate` here — validation opens a database
// connection, whereas encoding only needs the parsed runtime/entity settings.
// Requiring a live database would defeat the purpose of this static inspection command.
if (!ConfigGenerator.TryGetConfigForRuntimeEngine(Config, loader, fileSystem, out _))
if (!ConfigGenerator.TryGetConfigForRuntimeEngine(Config, loader, fileSystem, out string configPath))
{
logger.LogError("Could not determine the config file to use.");
return CliReturnCode.GENERAL_ERROR;
}

RuntimeConfigProvider runtimeConfigProvider = new(loader);
if (!runtimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig) || runtimeConfig is null)
// An inspection command must not use the runtime provider: that path resolves Key Vault
// references and creates file watchers. Keep unresolved credentials as Missing instead.
if (!AppNameConfigLoader.TryLoadConfig(configPath, fileSystem, out RuntimeConfig? runtimeConfig))
{
logger.LogError("Failed to parse the config file.");
return CliReturnCode.GENERAL_ERROR;
Expand Down
1 change: 1 addition & 0 deletions src/Config/Azure.DataApiBuilder.Config.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
<PackageReference Include="System.IO.Abstractions" />
<PackageReference Include="System.Drawing.Common" />
<PackageReference Include="Microsoft.Data.SqlClient" />
<PackageReference Include="MySqlConnector" />
<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" />
<PackageReference Include="Humanizer" />
Expand Down
33 changes: 31 additions & 2 deletions src/Config/RuntimeConfigLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -429,6 +429,10 @@ public static JsonSerializerOptions GetSerializationOptions(
/// <returns>The connection string with the telemetry-bearing <c>Application Name</c> embedded.</returns>
public static string GetConnectionStringWithApplicationName(string connectionString, RuntimeConfig config, DataSource dataSource)
{
// Hosted initialization and explicit overrides can supply a different connection string from
// the one parsed into the config. Per-pool authentication telemetry must describe that string.
dataSource = dataSource with { ConnectionString = connectionString };

return dataSource.DatabaseType switch
{
DatabaseType.MSSQL or DatabaseType.DWSQL => GetMsSqlConnectionStringWithApplicationName(connectionString, config, dataSource),
Expand Down Expand Up @@ -499,18 +503,43 @@ internal static string GetMsSqlConnectionStringWithApplicationName(string connec
if (string.IsNullOrWhiteSpace(connectionStringBuilder.ApplicationName)
|| connectionStringBuilder.ApplicationName.Equals(defaultApplicationName, StringComparison.OrdinalIgnoreCase))
{
connectionStringBuilder.ApplicationName = applicationName;
connectionStringBuilder.ApplicationName = ComposeSqlApplicationName(string.Empty, applicationName);
}
else
{
// If the connection string contains the `Application Name` property with a value, update the value by adding the DataApiBuilder Application Name.
connectionStringBuilder.ApplicationName += $",{applicationName}";
connectionStringBuilder.ApplicationName = ComposeSqlApplicationName(connectionStringBuilder.ApplicationName, applicationName);
}

// Return the updated connection string.
return connectionStringBuilder.ConnectionString;
}

/// <summary>
/// SqlClient rejects names longer than 128 UTF-16 code units before opening a connection. Keep
/// the existing name (including any isolation prefix) intact and fit only the DAB-owned suffix
/// into the remaining budget. The decoder already accepts a truncated payload.
/// </summary>
private static string ComposeSqlApplicationName(string existingName, string telemetry)
{
const int maxLength = 128;
string separator = existingName.Length == 0 ? string.Empty : ",";
int available = maxLength - existingName.Length - separator.Length;
if (available <= 0)
{
return existingName;
}

int length = Math.Min(available, telemetry.Length);
if (length < telemetry.Length && length > 0
&& char.IsHighSurrogate(telemetry[length - 1]) && char.IsLowSurrogate(telemetry[length]))
{
length--;
}

return length == 0 ? existingName : existingName + separator + telemetry[..length];
}

/// <summary>
/// It adds or replaces a property in the connection string with `Application Name` property.
/// If the connection string already contains the property, it appends the property `Application Name` to the connection string,
Expand Down
Loading
Loading