Skip to content

feat(monitoring): preserve connection visibility over gRPC #497

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yordis wants to merge 1 commit into from
Open

feat(monitoring): preserve connection visibility over gRPC #497

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,600 changes: 52 additions & 1,548 deletions proto.lock
View file
Open in desktop

Large diffs are not rendered by default.

209 changes: 121 additions & 88 deletions src/EventStore.ClusterNode/Components/Pages/Observability.razor
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<div>
<p class="text-sm font-black uppercase tracking-[0.28em] text-es-green">Observability</p>
<h1 class="mt-4 text-5xl font-black tracking-tight text-es-ink sm:text-6xl">A map of the signals the node already emits.</h1>
<p class="mt-5 text-lg leading-8 text-es-muted">Inspect live queue pressure, processing throughput, health, metrics, replication, TCP, and grouped runtime statistics.</p>
<p class="mt-5 text-lg leading-8 text-es-muted">Inspect live queue pressure, processing throughput, health, metrics, replication, and grouped runtime statistics.</p>
</div>
<div class="grid gap-3 rounded-[2rem] border border-white/80 bg-white/85 p-6 shadow-[0_20px_70px_rgba(23,32,51,0.09)] sm:grid-cols-2">
<MetricCard Label="Queues" Value="@(Page?.QueueCountLabel ?? "-")" QueueMetric="queue-count" />
Expand All @@ -20,6 +20,112 @@
</div>
</section>

<section class="mt-8">
<div class="mb-4 flex flex-col gap-2 sm:flex-row sm:items-end sm:justify-between">
<div>
<p class="text-xs font-black uppercase tracking-[0.22em] text-es-green">Network boundary</p>
<h2 class="mt-2 text-3xl font-black tracking-tight text-es-ink">Shared HTTP and gRPC connections</h2>
<p class="mt-2 text-sm text-es-muted">Active connections accepted by the node endpoint, including per-second traffic rates and pending bytes.</p>
</div>
<div class="flex flex-wrap gap-2">
<span class="rounded-full border border-es-green/20 bg-es-green/10 px-4 py-2 text-sm font-bold text-es-forest" data-network-status>@NetworkStatusLabel</span>
<span class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-muted" data-network-page-status>@NetworkPageStatusLabel</span>
<button type="button" class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-ink transition hover:border-es-green/30 hover:text-es-green disabled:cursor-not-allowed disabled:opacity-40" data-network-page="previous" disabled>Previous</button>
<button type="button" class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-ink transition hover:border-es-green/30 hover:text-es-green disabled:cursor-not-allowed disabled:opacity-40" data-network-page="next" disabled>Next</button>
</div>
</div>

<div class="overflow-hidden rounded-[2rem] border border-white/80 bg-white/85 shadow-[0_20px_70px_rgba(23,32,51,0.09)]">
<div class="overflow-x-auto">
<table class="min-w-full divide-y divide-es-ink/10 text-left text-sm">
<thead class="bg-es-ink/5 text-xs font-black uppercase tracking-[0.18em] text-es-muted">
<tr>
<th class="px-5 py-4">Connection</th>
<th class="px-5 py-4">Client</th>
<th class="px-5 py-4">Type</th>
<th class="px-5 py-4">Remote endpoint</th>
<th class="px-5 py-4 text-right">Sent rate</th>
<th class="px-5 py-4 text-right">Sent current</th>
<th class="px-5 py-4 text-right">Sent pending</th>
<th class="px-5 py-4 text-right">Received rate</th>
<th class="px-5 py-4 text-right">Received current</th>
<th class="px-5 py-4 text-right">Received pending</th>
</tr>
</thead>
<tbody class="divide-y divide-es-ink/10" data-network-table-body>
@if (NetworkRows.Count == 0)
{
<tr><td class="px-5 py-4 text-es-muted" colspan="10">No active shared-endpoint connections.</td></tr>
}
else
{
foreach (var connection in NetworkRows.Take(5))
{
<tr>
<td class="px-5 py-4 font-mono text-xs text-es-muted">@connection.ConnectionId</td>
<td class="px-5 py-4 font-bold text-es-ink">@Display(connection.ClientName)</td>
<td class="px-5 py-4 text-es-muted">@connection.Application · @Display(connection.Protocol) · @(connection.IsTls ? "TLS" : "Cleartext")</td>
<td class="px-5 py-4 font-mono text-xs text-es-muted" title="@ConnectionDetails(connection)">@connection.RemoteEndPoint</td>
<td class="px-5 py-4 text-right font-mono text-es-muted">Waiting</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.TotalBytesSent)</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.PendingSendBytes)</td>
<td class="px-5 py-4 text-right font-mono text-es-muted">Waiting</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.TotalBytesReceived)</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.PendingReceivedBytes)</td>
</tr>
}
}
</tbody>
</table>
</div>
</div>
</section>

<section class="mt-8">
<div class="mb-4">
<p class="text-xs font-black uppercase tracking-[0.22em] text-es-green">Cluster transport</p>
<h2 class="mt-2 text-3xl font-black tracking-tight text-es-ink">gRPC replication connections</h2>
<p class="mt-2 text-sm text-es-muted">Live database replication sessions on the shared HTTP/2 endpoint.</p>
</div>

<div class="overflow-hidden rounded-[2rem] border border-white/80 bg-white/85 shadow-[0_20px_70px_rgba(23,32,51,0.09)]">
<div class="overflow-x-auto">
<table class="min-w-full divide-y divide-es-ink/10 text-left text-sm">
<thead class="bg-es-ink/5 text-xs font-black uppercase tracking-[0.18em] text-es-muted">
<tr>
<th class="px-5 py-4">Endpoint</th>
<th class="px-5 py-4">Connection</th>
<th class="px-5 py-4 text-right">Sent</th>
<th class="px-5 py-4 text-right">Received</th>
<th class="px-5 py-4 text-right">Pending</th>
<th class="px-5 py-4 text-right">Send queue</th>
</tr>
</thead>
<tbody class="divide-y divide-es-ink/10" data-replication-table-body>
@if (Page is not null && Page.ReplicationConnections.Count > 0)
{
@foreach (var connection in Page.ReplicationConnections)
{
<tr>
<td class="px-5 py-4 font-bold text-es-ink">@connection.Endpoint</td>
<td class="px-5 py-4 font-mono text-xs text-es-muted" title="Subscription @connection.SubscriptionId">@connection.ConnectionId</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.TotalBytesSent)</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.TotalBytesReceived)</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.PendingSendBytes + connection.PendingReceivedBytes)</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.SendQueueSize.ToString("N0", CultureInfo.InvariantCulture)</td>
</tr>
}
}
else
{
<tr><td class="px-5 py-4 text-es-muted" colspan="6">No active gRPC replication connections.</td></tr>
}
</tbody>
</table>
</div>
</div>
</section>

<section class="mt-8">
<details id="dashboard-snapshot" class="rounded-[2rem] border border-white/80 bg-white/85 p-6 shadow-[0_20px_70px_rgba(23,32,51,0.09)]">
<summary class="cursor-pointer text-sm font-black uppercase tracking-[0.22em] text-es-green">Dashboard snapshot</summary>
Expand Down Expand Up @@ -87,58 +193,6 @@
}
</section>

<section class="mt-8">
<div class="mb-4 flex flex-col gap-2 sm:flex-row sm:items-end sm:justify-between">
<div>
<p class="text-xs font-black uppercase tracking-[0.22em] text-es-green">TCP dashboard</p>
<h2 class="mt-2 text-3xl font-black tracking-tight text-es-ink">Realtime connections</h2>
<p class="mt-2 text-sm text-es-muted">Shows active connections with per-second sent and received byte rates.</p>
</div>
<div class="flex flex-wrap gap-2">
<span class="rounded-full border border-es-green/20 bg-es-green/10 px-4 py-2 text-sm font-bold text-es-forest" data-tcp-status>@TcpStatusLabel</span>
<span class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-muted" data-tcp-page-status>@TcpPageStatusLabel</span>
<button type="button" class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-ink transition hover:border-es-green/30 hover:text-es-green disabled:cursor-not-allowed disabled:opacity-40" data-tcp-page="previous" disabled>Previous</button>
<button type="button" class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-ink transition hover:border-es-green/30 hover:text-es-green disabled:cursor-not-allowed disabled:opacity-40" data-tcp-page="next" disabled>Next</button>
</div>
</div>

<div class="overflow-hidden rounded-[2rem] border border-white/80 bg-white/85 shadow-[0_20px_70px_rgba(23,32,51,0.09)]">
<div class="overflow-x-auto">
<table class="min-w-full divide-y divide-es-ink/10 text-left text-sm">
<thead class="bg-es-ink/5 text-xs font-black uppercase tracking-[0.18em] text-es-muted">
<tr>
<th class="px-5 py-4">Connection</th>
<th class="px-5 py-4">Client</th>
<th class="px-5 py-4">Type</th>
<th class="px-5 py-4">IP Address</th>
<th class="px-5 py-4 text-right">Sent rate</th>
<th class="px-5 py-4 text-right">Sent current</th>
<th class="px-5 py-4 text-right">Sent pending</th>
<th class="px-5 py-4 text-right">Received rate</th>
<th class="px-5 py-4 text-right">Received current</th>
<th class="px-5 py-4 text-right">Received pending</th>
</tr>
</thead>
<tbody class="divide-y divide-es-ink/10" data-tcp-table-body>
@if (TcpRows.Count == 0)
{
<tr>
<td class="px-5 py-4 text-es-muted" colspan="10">@TcpEmptyMessage</td>
</tr>
}
else
{
foreach (var connection in TcpRows.Take(5))
{
@RenderTcpRow(connection)
}
}
</tbody>
</table>
</div>
</div>
</section>

</div>

@code {
Expand All @@ -148,29 +202,21 @@
private string Expanded { get; set; } = "";

private string DashboardPayloadJson => Page?.ClientPayloadJson ?? "{}";
private IReadOnlyList<TcpConnectionRow> TcpRows => Page?.TcpConnections ?? Array.Empty<TcpConnectionRow>();
private string TcpErrorMessage => !string.IsNullOrWhiteSpace(Page?.TcpMessage)
? Page.TcpMessage
: Page?.Message ?? "";
private string TcpStatusLabel {
get {
if (Page is null)
return "Connecting TCP stats...";

if (!string.IsNullOrWhiteSpace(TcpErrorMessage))
return "TCP unavailable";

return TcpRows.Count == 0
? "TCP live"
: string.Create(CultureInfo.InvariantCulture, $"TCP live · {TcpRows.Count} connection{(TcpRows.Count == 1 ? "" : "s")}");
}
}
private string TcpPageStatusLabel => TcpRows.Count == 0
private IReadOnlyList<NodeConnectionSnapshot> NetworkRows =>
Page?.NodeConnections ?? Array.Empty<NodeConnectionSnapshot>();
private string NetworkStatusLabel => NetworkRows.Count == 0
? "Network live"
: string.Create(CultureInfo.InvariantCulture, $"Network live · {NetworkRows.Count} connection{(NetworkRows.Count == 1 ? "" : "s")}");
Comment thread
cursor[bot] marked this conversation as resolved.
private string NetworkPageStatusLabel => NetworkRows.Count == 0
? "No pages"
: string.Create(CultureInfo.InvariantCulture, $"Page 1 of {Math.Max(1, (int)Math.Ceiling(TcpRows.Count / 5.0))}");
private string TcpEmptyMessage => string.IsNullOrWhiteSpace(TcpErrorMessage)
? "No TCP connections are currently reported."
: TcpErrorMessage;
: string.Create(CultureInfo.InvariantCulture, $"Page 1 of {Math.Max(1, (int)Math.Ceiling(NetworkRows.Count / 5.0))}");

private static string FormatBytes(long value) =>
value.ToString("N0", CultureInfo.InvariantCulture) + " B";
private static string Display(string value) => string.IsNullOrWhiteSpace(value) ? "<none>" : value;
private static string ConnectionDetails(NodeConnectionSnapshot connection) =>
string.Create(CultureInfo.InvariantCulture,
$"Local {connection.LocalEndPoint}, connected {connection.ConnectedAt:u}");

protected override async Task OnParametersSetAsync() {
Page = null;
Expand Down Expand Up @@ -216,19 +262,6 @@
<td class="px-5 py-4 font-mono text-xs text-es-muted">@row.CurrentLastMessageLabel</td>
</tr>;

private RenderFragment RenderTcpRow(TcpConnectionRow connection) => @<tr class="bg-white/70 text-es-ink">
<td class="max-w-[14rem] truncate px-5 py-4 font-mono text-xs text-es-muted">@connection.IdLabel</td>
<td class="px-5 py-4 font-bold text-es-ink">@connection.ClientLabel</td>
<td class="px-5 py-4 text-es-muted">@connection.TypeLabel</td>
<td class="px-5 py-4 font-mono text-xs text-es-muted">@connection.RemoteEndPointLabel</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.SentRateLabel</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.TotalBytesSentLabel</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.PendingSendBytesLabel</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.ReceivedRateLabel</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.TotalBytesReceivedLabel</td>
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.PendingReceivedBytesLabel</td>
</tr>;

private bool IsExpanded(string groupName) =>
ParseExpandedGroups().Contains(groupName);

Expand Down
29 changes: 27 additions & 2 deletions src/EventStore.ClusterNode/Components/Services/NodeConnectionTracker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using EventStore.Core.Services.Transport.Grpc;
using Microsoft.AspNetCore.Connections;

namespace EventStore.ClusterNode.Components.Services;

public sealed class NodeConnectionTracker
public sealed class NodeConnectionTracker : IConnectionStatsProvider
{
private readonly ConcurrentDictionary<string, NodeConnectionState> _connections = new();

Expand All @@ -20,6 +21,8 @@ public IReadOnlyList<NodeConnectionSnapshot> Snapshot() =>
.ThenBy(x => x.ConnectionId, StringComparer.Ordinal)
.ToArray();

IReadOnlyList<ConnectionStatsSnapshot> IConnectionStatsProvider.Snapshot() => Snapshot();

public async Task Track(ConnectionContext context, ConnectionDelegate next, bool isTls)
{
var state = new NodeConnectionState(
Expand Down Expand Up @@ -49,7 +52,9 @@ public void ObserveRequest(
string userAgent)
{
if (_connections.TryGetValue(connectionId, out var connection))
{
connection.ObserveRequest(protocol, isGrpc, connectionName, userAgent);
}
}
}

Expand All @@ -65,7 +70,19 @@ public sealed record NodeConnectionSnapshot(
long TotalBytesSent,
long TotalBytesReceived,
long PendingSendBytes,
long PendingReceivedBytes);
long PendingReceivedBytes) : ConnectionStatsSnapshot(
ConnectionId,
RemoteEndPoint,
LocalEndPoint,
ClientName,
Application,
Protocol,
IsTls,
ConnectedAt,
TotalBytesSent,
TotalBytesReceived,
PendingSendBytes,
PendingReceivedBytes);

internal sealed class NodeConnectionState
{
Expand Down Expand Up @@ -172,7 +189,10 @@ public NodeConnectionSnapshot Snapshot()
private static string Merge(string current, string observed)
{
if (string.IsNullOrWhiteSpace(observed) || current == observed)
{
return current;
}

return string.IsNullOrWhiteSpace(current) ? observed : "Mixed";
}
}
Expand Down Expand Up @@ -228,7 +248,9 @@ public override async ValueTask<ReadResult> ReadAsync(CancellationToken cancella
public override bool TryRead(out ReadResult result)
{
if (!_inner.TryRead(out result))
{
return false;
}

Observe(result);
return true;
Expand Down Expand Up @@ -268,7 +290,10 @@ public override async ValueTask<FlushResult> FlushAsync(CancellationToken cancel
{
var result = await _inner.FlushAsync(cancellationToken);
if (!result.IsCanceled)
{
_state.Sent();
}

return result;
}

Expand Down
Loading
Loading