diff --git a/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs b/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs index b8628a36..cc239e44 100644 --- a/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs +++ b/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs @@ -21,6 +21,16 @@ private sealed class StubNode : IAsyncDisposable public string Url { get; } public int Hits; + /// Whether this node serves account metadata. Nodes that strip it + /// answer with a well-formed account whose posting_json_metadata is empty. + public bool ServesMetadata = true; + + // A healthy node's get_accounts result. Account-metadata presence matters: + // GetAccounts prefers a node that serves it, so the default stub carries it. + private string AccountResultBody => + "{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":[{\"name\":\"served-by\",\"port\":\"" + Url + + "\",\"posting_json_metadata\":" + (ServesMetadata ? "\"{\\\"profile\\\":{}}\"" : "\"\"") + "}]}"; + public StubNode(Func handler) { _handler = handler; @@ -45,7 +55,7 @@ private async Task Loop() if (status == 200) { body = Encoding.UTF8.GetBytes( - "{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":[{\"name\":\"served-by\",\"port\":\"" + Url + "\"}]}"); + AccountResultBody); } else if (status == -1) { @@ -60,7 +70,7 @@ private async Task Loop() // 1s unproven prior, below any test timeout). await Task.Delay(1500); body = Encoding.UTF8.GetBytes( - "{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":[{\"name\":\"served-by\",\"port\":\"" + Url + "\"}]}"); + AccountResultBody); status = 200; } else if (status == -3) @@ -70,6 +80,22 @@ private async Task Loop() body = Encoding.UTF8.GetBytes("{\"jsonrpc\":\"2.0\",\"id\":1}"); status = 200; } + else if (status == -4) + { + // RPC-level error: the node answered, the error is the + // application's (no failover, no health penalty). + body = Encoding.UTF8.GetBytes( + "{\"jsonrpc\":\"2.0\",\"id\":1,\"error\":{\"message\":\"boom\"}}"); + status = 200; + } + else if (status == -5) + { + // Well-formed array whose entries are scalars: passes a bare + // "is an array" check but carries no readable account. + body = Encoding.UTF8.GetBytes( + "{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":[\"invalid\"]}"); + status = 200; + } else { body = Encoding.UTF8.GetBytes("rate limited"); @@ -211,6 +237,99 @@ public async Task MalformedResultNode_IsDemotedOnSubsequentCalls() Assert.True(good.Hits >= 2); } + // A node can strip account metadata: balances and reputation are correct but + // posting_json_metadata comes back empty. That is a well-formed array, so shape + // validation passes and the latency EWMA happily keeps such a node first — + // silently blanking portfolio engine/chain token visibility, which is derived + // entirely from that field. GetAccounts routes around it. + [Fact] + public async Task MetadataStrippingNode_IsSkippedForAccountFetches() + { + await using var stripped = new StubNode(() => 200) { ServesMetadata = false }; + await using var full = new StubNode(() => 200); + + var client = new HiveRpcClient(new[] { stripped.Url, full.Url }, timeoutMs: 1500); + + var accounts = await client.GetAccounts(new[] { "good-karma" }); + + Assert.NotNull(accounts); + var meta = accounts![0]!["posting_json_metadata"]!.GetValue(); + Assert.False(string.IsNullOrEmpty(meta), "should have used the node serving metadata"); + Assert.Equal(1, stripped.Hits); // consulted once, no same-node retry + Assert.True(full.Hits >= 1); + } + + // The preference is soft: an account that genuinely has no metadata looks + // identical to a stripped response, so once no node can do better the answer + // is returned rather than failing the request. + [Fact] + public async Task NoNodeServesMetadata_StillReturnsTheAccount() + { + await using var a = new StubNode(() => 200) { ServesMetadata = false }; + await using var b = new StubNode(() => 200) { ServesMetadata = false }; + + var client = new HiveRpcClient(new[] { a.Url, b.Url }, timeoutMs: 1500); + + var accounts = await client.GetAccounts(new[] { "good-karma" }); + + Assert.NotNull(accounts); + Assert.Equal("served-by", accounts![0]!["name"]!.GetValue()); + // The *first* well-formed answer is the floor, not whichever probe ran last. + Assert.Equal(a.Url, accounts[0]!["port"]!.GetValue()); + } + + // The probe is optional, so its failure must not fail the request: an RPC-level + // error from the node we only consulted to improve on an answer we already hold + // would otherwise turn a call that previously succeeded into a hard failure. + [Fact] + public async Task PreferenceProbeHittingRpcError_StillReturnsTheFirstAnswer() + { + await using var stripped = new StubNode(() => 200) { ServesMetadata = false }; + await using var erroring = new StubNode(() => -4); + + var client = new HiveRpcClient(new[] { stripped.Url, erroring.Url }, timeoutMs: 1500); + + var accounts = await client.GetAccounts(new[] { "good-karma" }); + + Assert.NotNull(accounts); + Assert.Equal(stripped.Url, accounts![0]!["port"]!.GetValue()); + } + + // A node answering with scalar entries passes a bare "is an array" check but + // reads as empty downstream — the same silent blanking a metadata-stripping + // node causes, so it must fail over rather than be accepted. + [Fact] + public async Task ScalarAccountArray_FailsOverAsUnusable() + { + await using var scalar = new StubNode(() => -5); + await using var good = new StubNode(() => 200); + + var client = new HiveRpcClient(new[] { scalar.Url, good.Url }, timeoutMs: 1500); + + var accounts = await client.GetAccounts(new[] { "good-karma" }); + + Assert.NotNull(accounts); + Assert.Equal(good.Url, accounts![0]!["port"]!.GetValue()); + Assert.Equal(1, scalar.Hits); // unusable result advances immediately + } + + // Probing is bounded: an account with no metadata is a common case that no node + // can satisfy, so the pool must not be swept on every such request. + [Fact] + public async Task MetadataPreference_ProbesAtMostTwoNodes() + { + await using var a = new StubNode(() => 200) { ServesMetadata = false }; + await using var b = new StubNode(() => 200) { ServesMetadata = false }; + await using var c = new StubNode(() => 200) { ServesMetadata = false }; + await using var d = new StubNode(() => 200) { ServesMetadata = false }; + + var client = new HiveRpcClient(new[] { a.Url, b.Url, c.Url, d.Url }, timeoutMs: 1500); + + Assert.NotNull(await client.GetAccounts(new[] { "good-karma" })); + + Assert.Equal(2, a.Hits + b.Hits + c.Hits + d.Hits); + } + [Fact] public async Task AllNodesMalformed_ThrowsNamingTheNode() { diff --git a/dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs b/dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs index 4d556c44..151280fd 100644 --- a/dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs +++ b/dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs @@ -42,6 +42,10 @@ public RpcException(string message) : base(message) { } private static long NowMs => Environment.TickCount64; + /// How many nodes may be consulted to satisfy a soft result + /// preference before the first well-formed answer is accepted as-is. + private const int MaxPreferenceProbes = 2; + // ---- calls ------------------------------------------------------------- /// Optional shape check for the RPC result. A node @@ -50,8 +54,21 @@ public RpcException(string message) : base(message) { } /// yielding no array); without validation that response counts as a SUCCESS, /// so the health tracker keeps the poisoned node ranked first for the whole /// window. A failed check is treated as node failure and fails over. + /// Optional *soft* check: the result is well-formed + /// but this node cannot serve the caller's needs. Unlike validateResult this is + /// not a health signal — the node is fine for other calls — so it is neither + /// retried nor marked unhealthy; we simply move on and keep its answer. If no + /// node satisfies the preference, the first such answer is returned rather than + /// throwing, so the caller is never worse off than without the preference. + /// + /// Exists because some Hive nodes serve accounts with account metadata stripped: + /// balances and reputation are correct, posting_json_metadata is empty. That is a + /// valid 200 with a usable array, so shape validation passes and the latency EWMA + /// keeps such a node ranked first — silently blanking every metadata-derived + /// feature (portfolio engine/chain token visibility) with no error and no log. public async Task Call(string api, string method, JsonNode @params, - Func? validateResult = null) + Func? validateResult = null, + Func? preferResult = null) { var request = new JsonObject { @@ -65,6 +82,9 @@ public RpcException(string message) : base(message) { } var body = JsJson.Stringify(request); Exception? lastError = null; + JsonNode? unpreferred = null; + var haveUnpreferred = false; + var unpreferredCount = 0; foreach (var nodeIndex in _health.OrderedNodeIndices()) { @@ -84,7 +104,23 @@ public RpcException(string message) : base(message) { } $"RPC node {node} returned unusable {method} result", advanceImmediately: true); } + // The node is healthy either way — record the success before + // deciding whether its answer is the one we wanted. _health.RecordSuccess(nodeIndex, NowMs - started); + if (preferResult != null && !preferResult(result)) + { + // Keep the first such answer as the floor and try the next + // node; same-node retry would return the same thing. + if (!haveUnpreferred) { unpreferred = result; haveUnpreferred = true; } + // Bounded on purpose. Roughly an eighth of active accounts + // genuinely carry no metadata, and for those NO node can + // satisfy the preference — probing the whole pool every time + // would multiply RPC load on a common case to route around a + // rare one. One alternative is enough to get past a single + // metadata-stripping node, which is all this guards against. + if (++unpreferredCount >= MaxPreferenceProbes) return unpreferred; + break; + } return result; } catch (RpcException) @@ -92,6 +128,11 @@ public RpcException(string message) : base(message) { } // The node answered; the error is the application's. No // failover (dhive semantics), and no failure mark. _health.RecordSuccess(nodeIndex, NowMs - started); + // ...but if we only came to this node to improve on an answer we + // already hold, its error belongs to the optional probe, not to + // the caller's request. Rethrowing here would fail a call that + // would have succeeded without the preference. + if (haveUnpreferred) return unpreferred; throw; } catch (NodeUnavailableException e) @@ -118,6 +159,11 @@ public RpcException(string message) : base(message) { } } } + // No node satisfied the preference, but one answered well-formed: that is + // the normal outcome when the preference is genuinely unsatisfiable (an + // account really has no metadata), so return it instead of failing. + if (haveUnpreferred) return unpreferred; + // Every node exhausted — surface the last transport error (dhive throws // after cycling the whole list). throw lastError ?? new InvalidOperationException("no RPC nodes configured"); @@ -220,10 +266,56 @@ public NodeUnavailableException(string message, bool advanceImmediately, nameArr.Add(n is null ? null : JsonValue.Create(n)); } var result = await Call("condenser_api", "get_accounts", new JsonArray(nameArr), - r => r is JsonArray); + IsAccountArray, + HasAnyAccountMetadata); return result as JsonArray; } + /// + /// A usable get_accounts result: an array whose entries are account objects or + /// JSON null (an unknown account). A node answering with scalar entries passes a + /// bare "is an array" check but yields nothing readable downstream — metadata + /// reads off it come back empty, which silently blanks portfolio token + /// visibility exactly like a metadata-stripping node. Treat it as node failure. + /// + internal static bool IsAccountArray(JsonNode? result) + { + if (result is not JsonArray accounts) return false; + + foreach (var account in accounts) + { + if (account is not null and not JsonObject) return false; + } + + return true; + } + + /// + /// True when at least one returned account carries a non-empty + /// posting_json_metadata. Nodes that strip account metadata answer with a + /// well-formed array whose entries have it blank; portfolio token visibility + /// is derived from that field, so such an answer silently reads as "this user + /// enabled nothing". Soft preference, not a health signal: an account that + /// genuinely has no metadata produces the same shape, and after every node + /// declines the caller still gets the response. + /// + internal static bool HasAnyAccountMetadata(JsonNode? result) + { + if (result is not JsonArray accounts || accounts.Count == 0) return true; + + var sawAccount = false; + foreach (var account in accounts) + { + if (account is not JsonObject) continue; + sawAccount = true; + var meta = JsVal.AsString(JsVal.Prop(account, "posting_json_metadata")); + if (!string.IsNullOrEmpty(meta)) return true; + } + + // An all-null array (unknown account) has nothing to prefer either way. + return !sawAccount; + } + public Task GetDynamicGlobalProperties() => Call("condenser_api", "get_dynamic_global_properties", new JsonArray(), r => r is JsonObject); @@ -237,15 +329,19 @@ public NodeUnavailableException(string message, bool advanceImmediately, /// public static class HiveClients { + // techcoderx.com and hiveapi.actifit.io are deliberately absent: both serve + // accounts with posting_json_metadata stripped (balances correct, metadata + // empty). They are fast, so the latency EWMA ranked them first and the + // portfolio engine/chain layers came back empty for everyone. GetAccounts + // also routes around such a node at runtime, but keeping them out of the pool + // means correctness here does not depend on that fallback firing. public static readonly HiveRpcClient Default = new(new[] { "https://api.hive.blog", - "https://techcoderx.com", "https://api.deathwing.me", "https://rpc.mahdiyari.info", "https://hive-api.arcange.eu", "https://api.openhive.network", - "https://hiveapi.actifit.io", "https://hive-api.3speak.tv", "https://api.syncad.com", "https://api.c0ff33a.uk",