From 9255ab3e5a0c4461bbc67e1a52347f314a676a6d Mon Sep 17 00:00:00 2001 From: test Date: Fri, 17 Jul 2026 14:04:18 +0000 Subject: [PATCH 01/45] =?UTF-8?q?docs(design):=20add=20045=20=E2=80=94=20r?= =?UTF-8?q?etire=20aimdb-ws-protocol,=20converge=20every=20transport=20on?= =?UTF-8?q?=20AimX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mapping-table design doc 036 A2 gated the protocol unification on (038 §3.9, 034 §3.10 rc10): AimX ↔ ws-protocol frame mapping, the gap decisions (wildcard subscribe via optional Event.topic, Snapshot routing via a new sub field, the explicit subscribed ack, the 6→3 error-code collapse, shared record.query/record.list result shapes), two scope corrections verified against the code (the aimdb-client demux fold was already done in PR #124; query passthrough already existed — only the result shape was missing), and the go/no-go. Co-Authored-By: Claude Fable 5 --- ...045-retire-ws-protocol-converge-on-aimx.md | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 docs/design/045-retire-ws-protocol-converge-on-aimx.md diff --git a/docs/design/045-retire-ws-protocol-converge-on-aimx.md b/docs/design/045-retire-ws-protocol-converge-on-aimx.md new file mode 100644 index 00000000..62e735c2 --- /dev/null +++ b/docs/design/045-retire-ws-protocol-converge-on-aimx.md @@ -0,0 +1,226 @@ +# 045 — Retire `aimdb-ws-protocol`: every transport speaks AimX + +**Status:** 🚧 Proposed (mapping-table gate from 036 A2; implementation bound +to the next protocol-breaking release) + +**Scope:** delete the `aimdb-ws-protocol` crate and the WS-JSON envelope it +defines, porting the WebSocket connector (`aimdb-websocket-connector`), the +browser bridge (`aimdb-wasm-adapter::ws_bridge`), and the UI's raw discovery +path onto the AimX-v2 envelope (`aimdb-core::session::aimx`). Extends AimX +with the two features that originally justified the WS fork — wildcard / +multi-record live subscribe and a query passthrough result shape — as +additive core changes. + +**Origin:** design 038 §3.9 (estimated ~1,200–1,800 lines saved), carrying +036 A2 and 034 §3.10 root cause 10. + +**Consumers:** `aimdb-websocket-connector` (server + `ws-client`), +`aimdb-wasm-adapter` (`WsBridge`, `WasmDb.discover`), `_external/aimdb-ui` +(`useAimDb.tsx` raw discovery), `aimdb-client`/CLI/MCP (unchanged wire, +extended result shapes), `aimdb-pro` weather hub (WS builder knobs, query +handler). + +--- + +## 1. Context + +The workspace ships two wire protocols for one feature. AimX +(`aimdb-core/src/session/aimx/`, NDJSON tagged frames) is ridden by the +UDS/serial/TCP connectors and `aimdb-client`; `aimdb-ws-protocol` +(`ServerMessage`/`ClientMessage`, `type`-tagged JSON) is ridden by the +WebSocket stack and the browser `WsBridge`. Every protocol feature — +snapshots, acks, writes, queries, subscribe — exists twice, with two codecs +and two error mappings. + +Crucially, both protocols already ride the same session engine: the WS codec +implements the same `EnvelopeCodec` trait as `AimxCodec` and is driven by +the same `run_session`/`serve`/`run_client`. The genuinely WS-specific code +is the wire types, the WS codec, the query/list halves of the WS dispatch, +and the hand-rolled browser bridge. Everything below the codec (transport, +HTTP upgrade auth, fan-out bus, pumps) is engine-generic and stays. + +### Corrections to the originally scoped work (verified against the code) + +1. **"Fold `aimdb-client`'s hand-rolled demux onto `run_client`" is done.** + `AimxClient` was deleted in PR #124; `aimdb-client::AimxConnection` rides + `run_client` and holds zero correlation logic. The only residue is dead + re-exports in `aimdb-client/src/protocol.rs` (`RequestExt`/`ResponseExt`/ + `serialize_message`/`parse_message`/`EventMessage`/`cli_hello` — zero + consumers). That is a cleanup item here, not a work package. +2. **Query passthrough already exists in AimX.** `record.query` + + `QueryHandlerFn` (`aimdb-core/src/remote/query.rs`, registered by + `aimdb-persistence::with_persistence`) already delegates a pattern query + whose `name` accepts `*`. What's missing is the *result shape* (per-record + `ts`, `total`), not the passthrough. + +The one genuinely missing AimX feature is **wildcard / multi-record live +subscribe**: `Inbound::Subscribe` carries a single exact `topic` resolved by +exact key. + +## 2. Mapping table + +### 2.1 Frames + +AimX wire tags from `session/aimx/codec.rs`; ws-protocol messages from +`aimdb-ws-protocol/src/lib.rs`. "→" marks the surviving form. + +| Semantics | ws-protocol | AimX-v2 (surviving) | Notes | +|---|---|---|---| +| RPC request | `Query{id,pattern,from,to,limit}`, `ListTopics{id}` | `{"t":"req","id":N,"method":M,"params":P}` | String ids → engine-owned `u64` ids. `query`→`record.query` `{name,limit,start,end}`; `list_topics`→`record.list`. | +| RPC reply | `QueryResult{id,records,total}`, `TopicList{id,topics}` | `{"t":"reply","id":N,"ok":V}` | Result shapes: §3.4. | +| RPC error | `Error{code,topic,message}` | `{"t":"reply","id":N,"err":CODE}` | 6→3 code collapse: §3.5. | +| Subscribe | `Subscribe{topics:[..]}` (multi-topic, wildcards) | `{"t":"sub","id":N,"topic":T}` | One frame per pattern; a wildcard pattern is **one** subscription fanning in many records (§3.1). | +| Subscribe ack | `Subscribed{topics}` | `{"t":"subscribed","sub":S}` (**new**) | §3.2. | +| Unsubscribe | `Unsubscribe{topics}` | `{"t":"unsub","sub":S}` | By sub id, not topic. | +| Live data | `Data{topic,payload,ts}` | `{"t":"event","sub":S,"seq":N,"topic":T,"data":V}` | `topic` is **new**, present when the server tags it (always on WS; on wildcard subs elsewhere). Server-side `ts` is dropped — timestamps belong to the record layer / query results. `seq` is new for WS clients (drop detection). | +| Late-join snapshot | `Snapshot{topic,payload}` | `{"t":"snap","sub":S,"topic":T,"data":V}` | `sub` is **new** (§3.3). | +| Client write | `Write{topic,payload}` | `{"t":"write","topic":T,"payload":V}` | Identical semantics (fire-and-forget, producer/arbiter path). | +| Keepalive | `Ping`/`Pong` | `{"t":"ping"}` / `{"t":"pong"}` | Identical. | +| Discovery | `ListTopics` over a raw socket | `record.list` req over a raw socket | UI's `discoverTopics` and `WasmDb.discover` reissue as AimX. | + +### 2.2 Deleted with the ws wire + +- `topic_matches` — moves into `aimdb-core::session::topic_match` (same + MQTT-style semantics, same tests). `now_ms()` does **not** move; nothing + event-side needs a server clock. +- `WsCodec` and its per-connection id↔topic maps — the browser now speaks + engine ids natively. +- The transport's multi-topic `split_multi_topic` — the bridge issues one + `sub` frame per pattern. +- `ClientManager`'s pre-serialized `Data`-frame broadcast — the bus carries + `(topic, payload)`; the per-connection codec envelopes each event (payload + bytes stay `Arc`-shared; only the small envelope is per-subscriber). +- `WebSocketConnectorBuilder::with_raw_payload` — its purpose was bypassing + the ws `Data` envelope; under AimX the envelope *is* the protocol. No + in-tree or aimdb-pro users. +- `ClientConfig::topic_routed_subs` — existed solely because the ws wire + pushed `Data{topic}` without ids. All subscriptions are id-routed now. + +## 3. Gap analysis and decisions + +### 3.1 Wildcard subscribe (DECIDED: optional `topic` on `Event`) + +One wildcard subscription fans in many records. The server matches the +pattern against the registry **once at subscribe time** — the record set is +frozen at builder time (`configure` exists only on `AimDbBuilder`), so +subscribe-time enumeration is complete. MQTT-style dynamic membership +(records registered later) is deferred until core grows runtime +registration, which would be its own design. + +- `AimxSession::subscribe`: a topic containing `#`/`*` matches all current + records via `topic_matches`, merges their update streams + (`stream::select_all`) under the one subscription id, and emits one + `Snapshot` per matched record on open. Records without remote access are + skipped (logged), matching "subscribe to what can stream". A wildcard that + matches nothing yields an empty stream that ends immediately — with a + frozen record set, "no matches now" is "no matches ever". Exact-topic + subscribe keeps its existing fast path and (unchanged) emits no snapshot + on the UDS/serial/TCP paths. +- Core stays pull-per-record (no fan-out bus — no_std targets must not pay a + per-record pump); the WS server's `ClientManager` bus keeps per-event + pattern matching for many-client fan-out. +- **Engine plumbing:** subscription streams (server `Session::subscribe`, + client `ClientHandle::subscribe`) change item type from `Payload` to + `SubUpdate { topic: Option>, data: Payload }` so each event can + carry which record fired. This is the one place the implementation extends + the message set beyond "add `topic` to `Outbound::Event`": without an item + type carrying the topic, the engine pump has nothing to thread into the + frame. + +### 3.2 Subscribe ack (DECIDED: teach `AimxCodec` the frame) + +ws-protocol emits `Subscribed{topics}` and the browser relies on it; AimX +runs `acks_subscribe:false` and `AimxCodec` rejects `Outbound::Subscribed`. +Decision: `AimxCodec` encodes/decodes `{"t":"subscribed","sub":S}`; the WS +server keeps `acks_subscribe:true`. UDS/serial/TCP stay +`acks_subscribe:false` (unchanged wire). + +### 3.3 Snapshot routing (DECIDED: `sub` on `Snapshot`) + +A deviation the plan's phase list didn't spell out, forced by the client +demux: `run_client` routes frames by subscription id, and a wildcard +subscription's snapshots arrive tagged with concrete record topics that do +not string-match the pattern key. Rather than teaching the client demux +pattern matching (state + O(subs) per snapshot), `Outbound::Snapshot` gains +the `sub` of the subscription that triggered it — the server knows it at +emission (snapshots are only emitted inside the subscribe handshake). The +engine hook `Session::snapshot(topic) -> Option` becomes +`Session::snapshots(topic) -> Vec<(String, Payload)>` ("one snapshot per +matched record"); the default stays "none". + +### 3.4 Query / list result shapes (DECIDED) + +- `record.query` params stay `{name, limit, start, end}` (`name` accepts + MQTT wildcards and `*`; `start`/`end` are the time range — units are the + handler's contract, ms for the persistence backend). The **result** + becomes `{"records":[{"topic":T,"payload":V,"ts":MS}, …], "total":N}` — + the old `QueryResult` shape minus the ws envelope. `QueryRecord` moves to + `aimdb-core::remote` as the canonical row type; + `aimdb-persistence::with_persistence` registration is updated to produce + it (was `{values:[{record,value,stored_at}],count}`). +- `record.list`: `RecordMetadata` gains optional `schema_type` and `entity` + fields. Core populates `entity` from the record key's final `.` segment + (the server owns the naming convention; clients must not parse topics). + `schema_type` is populated by dispatches that own a schema registry — the + WS dispatch answers `record.list` from its `StreamableRegistry`-derived + topic list as `[{name, schema_type, entity}]` (the old `TopicInfo` rows), + which is what the browser discovery path consumes. Core alone cannot + resolve contract schema names and leaves it `None`. + +### 3.5 Error vocabulary (DECIDED: keep the 3-code core) + +ws-protocol has 6 `ErrorCode`s + per-error `topic`/`message`; AimX collapses +to `not_found` / `denied` / `internal` as a bare `err` string. Decision: +accept the 6→3 collapse rather than widening `RpcError`. +`UNAUTHORIZED` stays out-of-band (`AuthError` at the HTTP upgrade, exactly +as today); `FORBIDDEN`→`denied`; `UNKNOWN_TOPIC`→`not_found`; +`SERIALIZATION_ERROR`/`WRITE_ERROR`/`SERVER_ERROR`→`internal`. The wire +tolerates an optional human-readable `msg` field alongside `err` (decoders +ignore unknown fields), reserved as headroom; plumbing messages through +`RpcError` is deliberately out of scope. + +### 3.6 Auto-subscribe under engine demux (accepted limitation) + +`with_auto_subscribe` seeds server-side subscriptions whose events carry +server-chosen sub ids (synthesized from `u64::MAX` downward so they cannot +collide with client-chosen ids). A raw WS consumer sees them fine; a +`run_client`-based consumer drops events for sub ids it never issued — +engine clients (including the new `WsBridge`) must subscribe explicitly. +The aimdb-pro hub's UI already does (`subscribeTopics` = exact names from +discovery), so nothing user-visible changes. + +## 4. What lands where + +- **Phase 1 (core, additive):** `topic_match.rs`, `SubUpdate` item type, + `Event.topic` + `Snapshot.sub`, `subscribed` frame, wildcard subscribe + + per-match snapshots in `AimxDispatch`, `QueryRecord` + persistence result + shape, `RecordMetadata.schema_type`/`entity`, `AimxCodec` roundtrip suite. +- **Phase 2 (WS connector):** `AimxCodec` on the WS transport (one codec + blob per WS text frame — no extra framing needed), dispatch converged on + `record.query`/`record.list`, bus carries `(topic, payload)`, tests + rewritten to drive AimX frames. +- **Phase 3 (browser):** `WsBridge` rewritten on `run_client` + + `ClientHandle` over a `web_sys::WebSocket`-backed `Connection`/`Dialer` + (single-threaded wasm `Send` wrappers, same pattern as `SendFuture`); + reply/subscription correlation then exists exactly once + (`aimdb-core/src/session/client.rs`). JS API (`write`, `query`, + `listTopics`, `onStatusChange`, offline queue, reconnect) is preserved. + `useAimDb.tsx` discovery reissued as a raw `record.list` req. +- **Phase 4 (delete):** `aimdb-ws-protocol` crate, `WsCodec`, + `protocol.rs` shims, `aimdb-client` dead helpers; design 038 §2.5/§3.9 + annotated; CHANGELOGs. + +Breaking for browser clients ⇒ Phases 1–4 land in the next +protocol-breaking release as one branch, one commit per work item (no +stacked PRs — house invariant). This document may land ahead of the code. + +## 5. Go / no-go + +Convergence pays: the fork costs a 353-line protocol crate, a 507-line +codec, two error mappings, a hand-rolled 835-line browser demux duplicating +`run_client`, and a WS-only query/list vocabulary — against a one-page +additive AimX extension (wildcard subscribe + two result-shape changes) +whose features every other transport inherits (CLI `watch 'temp.#'`, MCP +wildcard reads, serial/TCP browsers-of-the-future). The browser break is +contained to the bridge's wire (JS API preserved) and lands in a release +already flagged protocol-breaking. **Go.** From 33e2c4c2892fd09907d655772ebc182ca2a1a9e5 Mon Sep 17 00:00:00 2001 From: test Date: Fri, 17 Jul 2026 14:04:41 +0000 Subject: [PATCH 02/45] feat(core): wildcard subscribe + topic-tagged subscription streams for AimX (design 045) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two features that justified the ws-protocol fork, added to AimX so every transport inherits them: - session::topic_match — the MQTT-style matcher moved in from aimdb-ws-protocol (same semantics, same tests) plus is_wildcard. - Subscription streams yield SubUpdate {topic: Option>, data} on both engines; Outbound::Event gains an optional topic and Outbound::Snapshot the routing sub, so a wildcard subscription's events and late-join snapshots name the record that fired. Session::snapshot becomes snapshots() — one per covered record. - AimxDispatch matches wildcard patterns against the registry once at subscribe time (builder-frozen record set), merges matched streams under one subscription id, and snapshots each match on open. - AimxCodec learns the {"t":"subscribed"} ack (servers running acks_subscribe:true — the WS connector) and gets a dedicated roundtrip test suite; exact-topic frames are unchanged on the wire. - ClientConfig::topic_routed_subs removed — it existed solely for the retired ws wire; all subscriptions are id-routed. - Shared result vocabulary: remote::QueryRecord {topic, payload, ts}, with_persistence's QueryHandlerFn returning {records, total} (sorted by ts), and RecordMetadata gaining optional schema_type/entity. - aimdb-client: AimxConnection::subscribe_with_topics yields (Option, Value) pairs; wildcard e2e test over the production UDS server; query-shape test against the persistence registration. Co-Authored-By: Claude Fable 5 --- aimdb-client/CHANGELOG.md | 11 + aimdb-client/src/engine.rs | 22 +- aimdb-client/tests/aimx_session.rs | 80 ++++++ aimdb-client/tests/pump_client.rs | 1 - aimdb-core/CHANGELOG.md | 28 +++ aimdb-core/src/lib.rs | 6 +- aimdb-core/src/remote/metadata.rs | 20 ++ aimdb-core/src/remote/mod.rs | 2 +- aimdb-core/src/remote/query.rs | 17 +- aimdb-core/src/session/aimx/codec.rs | 234 +++++++++++++++++- aimdb-core/src/session/aimx/dispatch.rs | 68 ++++- aimdb-core/src/session/client.rs | 51 ++-- aimdb-core/src/session/mod.rs | 62 ++++- aimdb-core/src/session/server.rs | 28 ++- aimdb-core/src/session/topic_match.rs | 122 +++++++++ aimdb-core/tests/session_engine.rs | 55 ++-- aimdb-persistence/CHANGELOG.md | 7 + aimdb-persistence/src/builder_ext.rs | 21 +- .../tests/query_handler_shape.rs | 95 +++++++ 19 files changed, 839 insertions(+), 91 deletions(-) create mode 100644 aimdb-core/src/session/topic_match.rs create mode 100644 aimdb-persistence/tests/query_handler_shape.rs diff --git a/aimdb-client/CHANGELOG.md b/aimdb-client/CHANGELOG.md index 68231014..2677f3bf 100644 --- a/aimdb-client/CHANGELOG.md +++ b/aimdb-client/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed (breaking) — Design 045 + +- **Wildcard subscriptions:** new `AimxConnection::subscribe_with_topics` + yields `(Option, Value)` pairs so one pattern subscription (`#`, + `*`) can fan in many records with each update naming the record that fired. + `ClientHandle`-level streams now carry `SubUpdate` (see `aimdb-core`). +- **Dead protocol helpers removed** (retired with the hand-rolled client in + PR #124, deleted now): `RequestExt`, `ResponseExt`, `serialize_message`, + `parse_message`, `EventMessage`, `cli_hello`. `RecordMetadata`, + `WelcomeMessage`, `Request`, `Response`, `Event` re-exports remain. + ### Added - **Transport-agnostic endpoint resolver — pick the transport at runtime via a `scheme://` URL (Issue #123, follow-up to #39 / #122).** New `endpoint` module: `parse_endpoint` (pure, feature-independent grammar) and `dial(url) -> Box` map an endpoint string to a transport `Dialer`, the way records already pick one for links. Schemes: `unix://PATH` / `uds://PATH`, a bare path (the `unix://` shorthand), and `serial://DEVICE?baud=N`. An unknown scheme — or one whose transport isn't compiled in — is rejected with a clear error. New `AimxConnection::connect_over(dialer)` / `connect_over_with_timeout` dial over an explicit `Dialer`, bypassing resolution. (Rides a new `impl Dialer for Box` in `aimdb-core`.) diff --git a/aimdb-client/src/engine.rs b/aimdb-client/src/engine.rs index f622b025..95a9522b 100644 --- a/aimdb-client/src/engine.rs +++ b/aimdb-client/src/engine.rs @@ -194,8 +194,26 @@ impl AimxConnection { /// `subscription_id` to track. Dropping the stream stops local delivery. pub fn subscribe(&self, name: &str) -> ClientResult> { let raw = self.handle.subscribe(name).map_err(rpc_err)?; - // Decode each Payload into a JSON value; drop any that fail to parse. - let decoded = raw.filter_map(|p| async move { serde_json::from_slice(&p).ok() }); + // Decode each update's payload into a JSON value; drop any that fail to + // parse. For the per-record topic (wildcard subscriptions), see + // [`subscribe_with_topics`](Self::subscribe_with_topics). + let decoded = raw.filter_map(|u| async move { serde_json::from_slice(&u.data).ok() }); + Ok(Box::pin(decoded)) + } + + /// Subscribe to a topic pattern (wildcards supported: `#`, `*`), yielding + /// `(topic, value)` pairs. One wildcard subscription fans in every matching + /// record; each update names the record that fired. The topic is `None` + /// only when the server left the event untagged (exact-topic subscribe). + pub fn subscribe_with_topics( + &self, + pattern: &str, + ) -> ClientResult, serde_json::Value)>> { + let raw = self.handle.subscribe(pattern).map_err(rpc_err)?; + let decoded = raw.filter_map(|u| async move { + let value = serde_json::from_slice(&u.data).ok()?; + Some((u.topic.as_deref().map(String::from), value)) + }); Ok(Box::pin(decoded)) } diff --git a/aimdb-client/tests/aimx_session.rs b/aimdb-client/tests/aimx_session.rs index 514bacf6..80269960 100644 --- a/aimdb-client/tests/aimx_session.rs +++ b/aimdb-client/tests/aimx_session.rs @@ -142,6 +142,86 @@ async fn aimx_roundtrip_over_uds_production_server() { drop(conn); // stops the client engine } +/// One wildcard subscription fans in every matching record: events arrive +/// tagged with the concrete record topic, and matched records with a current +/// value are delivered up front as snapshots (design 045 §3.1/§3.3). +#[tokio::test] +async fn wildcard_subscribe_fans_in_matching_records() { + let dir = tempfile::tempdir().unwrap(); + let sock = dir.path().join("aimdb.sock"); + + let config = AimxConfig::uds_default().socket_path(sock.to_str().unwrap()); + let mut builder = AimDbBuilder::new() + .runtime(Arc::new(TokioAdapter)) + .with_connector(UdsServer::from_config(config)); + for key in ["temp/vienna", "temp/berlin", "humidity/london"] { + builder.configure::(key, |reg| { + reg.buffer(BufferCfg::SingleLatest).with_remote_access(); + }); + } + let (db, runner) = builder.build().await.expect("build db"); + let db = Arc::new(db); + tokio::spawn(runner.run()); + + // Seed one matched record before subscribing — it must arrive as a + // late-join snapshot on the wildcard stream. + db.set_record_from_json("temp/vienna", json!({ "level": 1 })) + .expect("seed vienna"); + + let conn = AimxConnection::connect(sock.to_str().unwrap()) + .await + .expect("connect"); + let mut stream = conn + .subscribe_with_topics("temp/#") + .expect("wildcard subscribe"); + + // The snapshot for the seeded record arrives first, tagged with its topic. + let (topic, value) = tokio::time::timeout(Duration::from_secs(2), stream.next()) + .await + .expect("snapshot within timeout") + .expect("snapshot"); + assert_eq!(topic.as_deref(), Some("temp/vienna")); + assert_eq!(value, json!({ "level": 1 })); + + // Live updates from both matched records ride the one subscription, each + // tagged; the non-matching record must never appear. + let db2 = db.clone(); + tokio::spawn(async move { + for n in 1..=50u64 { + let _ = db2.set_record_from_json("temp/berlin", json!({ "level": n })); + let _ = db2.set_record_from_json("humidity/london", json!({ "level": n })); + tokio::time::sleep(Duration::from_millis(10)).await; + } + }); + // Drain events until the other matched record fires (the seeded record's + // current value may replay first — SingleLatest readers start warm). Every + // event must be topic-tagged and inside the pattern. + let mut saw_berlin = false; + for _ in 0..20 { + match tokio::time::timeout(Duration::from_secs(2), stream.next()).await { + Ok(Some((topic, value))) => { + let topic = topic.expect("wildcard events are topic-tagged"); + assert!( + topic.starts_with("temp/"), + "event from outside the pattern: {topic}" + ); + assert!(value.get("level").is_some()); + if topic == "temp/berlin" { + saw_berlin = true; + break; + } + } + _ => break, + } + } + assert!( + saw_berlin, + "the wildcard stream must carry the other matched record's updates" + ); + + drop(conn); +} + /// `record.get` on a ring (`SpmcRing`) record has no canonical latest, so it /// falls back to draining the connection's cursor for the most-recent value. #[tokio::test] diff --git a/aimdb-client/tests/pump_client.rs b/aimdb-client/tests/pump_client.rs index 6df0b8ba..0360dde3 100644 --- a/aimdb-client/tests/pump_client.rs +++ b/aimdb-client/tests/pump_client.rs @@ -86,7 +86,6 @@ async fn pump_client_mirrors_record_both_directions() { max_reconnect_attempts: 0, keepalive_interval: None, max_offline_queue: usize::MAX, - topic_routed_subs: false, sends_hello: false, })); cb.configure::("cfg", |reg| { diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index a60b0fd0..477c8941 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -7,6 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed (breaking) — Design 045: one protocol (AimX) for every transport + +- **Wildcard / multi-record subscribe.** `Inbound::Subscribe` topics may carry + MQTT-style wildcards (`#`, `*`): `AimxDispatch` matches the pattern against + the registry once at subscribe time (the record set is builder-frozen), + merges the matched records' update streams under the one subscription id, + and emits one late-join `Snapshot` per matched record. The matcher moved in + from the retired `aimdb-ws-protocol` crate as + `session::topic_match::{topic_matches, is_wildcard}` (re-exported at the + crate root). +- **Subscription streams carry the firing record.** `Session::subscribe` and + `ClientHandle::subscribe` now yield `SubUpdate { topic: Option>, + data: Payload }` instead of bare `Payload`; `Outbound::Event` gains an + optional `topic` and `Outbound::Snapshot` gains the routing `sub` (frames + without them are unchanged on the wire). `Session::snapshot` became + `Session::snapshots(topic) -> Vec<(String, Payload)>` (one per covered + record). +- **`AimxCodec` learned the `subscribed` ack frame** (`{"t":"subscribed", + "sub":S}`) for servers running `acks_subscribe:true` (the WebSocket + connector); UDS/serial/TCP keep the implicit ack. A dedicated `AimxCodec` + roundtrip suite now locks the frame set. +- **`ClientConfig::topic_routed_subs` removed** — it existed solely for the + retired ws wire; all subscriptions are id-routed. +- **Shared query/list vocabulary.** New `remote::QueryRecord { topic, payload, + ts }` is the canonical `record.query` result row (result shape + `{records, total}`); `RecordMetadata` gains optional `schema_type` / + `entity` fields (`entity` derived from the record key's final `.` segment). + ### Added - **Issue #177 — bounded into-slice serialization for outbound links.** New diff --git a/aimdb-core/src/lib.rs b/aimdb-core/src/lib.rs index 7ec25886..84cd218e 100644 --- a/aimdb-core/src/lib.rs +++ b/aimdb-core/src/lib.rs @@ -79,9 +79,9 @@ pub use codec::{JsonCodec, RemoteSerialize, SerdeJsonCodec}; // compatible). See docs/design/remote-access-via-connectors.md. #[cfg(feature = "connector-session")] pub use session::{ - pump_sink, pump_source, AuthError, BoxFut, BoxStream, CodecError, Connection, Dialer, Dispatch, - EnvelopeCodec, Inbound, Listener, Outbound, Payload, PeerInfo, RpcError, SessionCtx, - SessionLimits, Source, TransportError, TransportResult, + is_wildcard, pump_sink, pump_source, topic_matches, AuthError, BoxFut, BoxStream, CodecError, + Connection, Dialer, Dispatch, EnvelopeCodec, Inbound, Listener, Outbound, Payload, PeerInfo, + RpcError, SessionCtx, SessionLimits, Source, SubUpdate, TransportError, TransportResult, }; // Signal gauge handle (always available; inert without `observability`) diff --git a/aimdb-core/src/remote/metadata.rs b/aimdb-core/src/remote/metadata.rs index fb2c8426..7e408faa 100644 --- a/aimdb-core/src/remote/metadata.rs +++ b/aimdb-core/src/remote/metadata.rs @@ -59,6 +59,19 @@ pub struct RecordMetadata { /// Number of outbound connector links registered pub outbound_connector_count: usize, + /// Data-contract schema name (e.g. `"temperature"`), when the serving + /// dispatch owns a schema registry that can resolve it (the WebSocket + /// connector's `StreamableRegistry`). Core alone cannot map a `TypeId` to a + /// contract name and leaves it `None`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema_type: Option, + + /// Entity / node identifier (e.g. `"vienna"` for `"temp.vienna"`), derived + /// from the record key's final `.` segment. The server is the authority on + /// naming conventions — clients use this field instead of parsing keys. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub entity: Option, + // ===== Buffer metrics (feature-gated) ===== /// Total items pushed to the buffer (metrics feature only) #[cfg(feature = "observability")] @@ -125,6 +138,11 @@ impl RecordMetadata { writable: bool, outbound_connector_count: usize, ) -> Self { + let entity = record_key + .as_str() + .rsplit('.') + .next() + .map(|s| s.to_string()); Self { record_id: record_id.raw(), record_key: record_key.as_str().to_string(), @@ -137,6 +155,8 @@ impl RecordMetadata { consumer_count, writable, outbound_connector_count, + schema_type: None, + entity, #[cfg(feature = "observability")] produced_count: None, #[cfg(feature = "observability")] diff --git a/aimdb-core/src/remote/mod.rs b/aimdb-core/src/remote/mod.rs index 187ef103..e3c7ac15 100644 --- a/aimdb-core/src/remote/mod.rs +++ b/aimdb-core/src/remote/mod.rs @@ -56,7 +56,7 @@ pub use metadata::RecordMetadata; pub use protocol::{ ErrorObject, Event, HelloMessage, Request, Response, WelcomeMessage, PROTOCOL_VERSION, }; -pub use query::{QueryHandlerFn, QueryHandlerParams}; +pub use query::{QueryHandlerFn, QueryHandlerParams, QueryRecord}; // Internal exports for implementation #[cfg(feature = "connector-session")] diff --git a/aimdb-core/src/remote/query.rs b/aimdb-core/src/remote/query.rs index 8c5e08f1..f2dea0d7 100644 --- a/aimdb-core/src/remote/query.rs +++ b/aimdb-core/src/remote/query.rs @@ -8,10 +8,13 @@ use alloc::boxed::Box; use alloc::string::String; +use serde::{Deserialize, Serialize}; + /// Type-erased query handler registered by `aimdb-persistence` via Extensions. /// /// A boxed async function that accepts query parameters (record pattern, limit, -/// start/end timestamps) and returns a JSON value with the results. +/// start/end timestamps) and returns a JSON value with the results, shaped +/// `{"records": [QueryRecord…], "total": N}` (design 045 §3.4). pub type QueryHandlerFn = Box< dyn Fn( QueryHandlerParams, @@ -21,6 +24,18 @@ pub type QueryHandlerFn = Box< + Sync, >; +/// One row of a `record.query` result — the canonical shape every transport +/// shares (the retired ws-protocol `QueryRecord`, now core vocabulary). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QueryRecord { + /// Record key / topic the value was stored under (e.g. `"temp.vienna"`). + pub topic: String, + /// Deserialized record value. + pub payload: serde_json::Value, + /// Storage timestamp (milliseconds since Unix epoch). + pub ts: u64, +} + /// Parameters for the type-erased query handler. #[derive(Debug, Clone)] pub struct QueryHandlerParams { diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index 7b2b7845..9e4fde36 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -6,9 +6,15 @@ //! backward-compatible with the legacy AimX wire: //! //! - `record.subscribe` is an [`Inbound::Subscribe`] keyed by the request `id`; -//! there is no `subscription_id` ack — events carry the `id` back as -//! [`Outbound::Event::sub`]. -//! - events carry only `{sub, seq, data}` (no server-side `timestamp`/`dropped`). +//! events carry the `id` back as [`Outbound::Event::sub`]. An explicit +//! `{"t":"subscribed"}` ack exists for servers running +//! `acks_subscribe:true` (the WebSocket connector); UDS/serial/TCP leave it +//! off and the ack stays implicit. +//! - events carry `{sub, seq, data}` plus an optional `topic` naming the +//! concrete record on wildcard/bus subscriptions (no server-side +//! `timestamp`/`dropped`). +//! - snapshots carry the opening subscription's `sub` so the client demux +//! routes them like events (design 045 §3.3). //! - the Hello/Welcome handshake is a normal `call("hello", …)`, so //! `authenticate` stays peer-only. //! @@ -157,25 +163,36 @@ impl EnvelopeCodec for AimxCodec { } } } - Outbound::Event { sub, seq, data } => { + Outbound::Event { + sub, + seq, + topic, + data, + } => { let raw = as_raw(&data)?; let mut frame = Frame::tagged("event"); frame.sub = Some(sub); frame.seq = Some(seq); + frame.topic = topic; frame.data = Some(raw); write_frame(out, &frame) } - Outbound::Snapshot { topic, data } => { + Outbound::Snapshot { sub, topic, data } => { let raw = as_raw(&data)?; let mut frame = Frame::tagged("snap"); + frame.sub = Some(sub); frame.topic = Some(topic); frame.data = Some(raw); write_frame(out, &frame) } Outbound::Pong => write_frame(out, &Frame::tagged("pong")), - // AimX has no explicit subscribe ack; `run_session` only emits this - // when `acks_subscribe` is set, which the AimX server leaves off. - Outbound::Subscribed { .. } => Err(CodecError::Malformed), + // Explicit subscribe ack — only emitted by servers running + // `acks_subscribe:true` (the WebSocket connector). + Outbound::Subscribed { sub } => { + let mut frame = Frame::tagged("subscribed"); + frame.sub = Some(sub); + write_frame(out, &frame) + } } } @@ -231,14 +248,215 @@ impl EnvelopeCodec for AimxCodec { "event" => Ok(Outbound::Event { sub: f.sub.ok_or(CodecError::Malformed)?, seq: f.seq.ok_or(CodecError::Malformed)?, + topic: f.topic, data: payload_of(f.data), }), "snap" => Ok(Outbound::Snapshot { + sub: f.sub.ok_or(CodecError::Malformed)?, topic: f.topic.ok_or(CodecError::Malformed)?, data: payload_of(f.data), }), + "subscribed" => Ok(Outbound::Subscribed { + sub: f.sub.ok_or(CodecError::Malformed)?, + }), "pong" => Ok(Outbound::Pong), _ => Err(CodecError::Malformed), } } } + +#[cfg(test)] +mod tests { + use super::*; + use alloc::string::ToString; + use alloc::vec::Vec; + + fn payload(s: &str) -> Payload { + Arc::from(s.as_bytes()) + } + + /// Encode client-direction, decode server-direction (the request path). + fn roundtrip_inbound(msg: Inbound) -> Inbound { + let codec = AimxCodec; + let mut out = Vec::new(); + codec.encode_inbound(msg, &mut out).expect("encode_inbound"); + codec.decode(&out).expect("decode") + } + + /// Encode server-direction, return the frame bytes (the reply/event path). + fn encode_outbound(msg: Outbound<'_>) -> Vec { + let codec = AimxCodec; + let mut out = Vec::new(); + codec.encode(msg, &mut out).expect("encode"); + out + } + + #[test] + fn request_roundtrip() { + match roundtrip_inbound(Inbound::Request { + id: 7, + method: "record.get".to_string(), + params: payload(r#"{"name":"temp"}"#), + }) { + Inbound::Request { id, method, params } => { + assert_eq!(id, 7); + assert_eq!(method, "record.get"); + assert_eq!(¶ms[..], br#"{"name":"temp"}"#); + } + _ => panic!("expected Request"), + } + } + + #[test] + fn subscribe_and_unsubscribe_roundtrip() { + match roundtrip_inbound(Inbound::Subscribe { + id: 3, + topic: "sensors/#".to_string(), + }) { + Inbound::Subscribe { id, topic } => { + assert_eq!(id, 3); + assert_eq!(topic, "sensors/#"); + } + _ => panic!("expected Subscribe"), + } + match roundtrip_inbound(Inbound::Unsubscribe { + sub: "3".to_string(), + }) { + Inbound::Unsubscribe { sub } => assert_eq!(sub, "3"), + _ => panic!("expected Unsubscribe"), + } + } + + #[test] + fn write_roundtrip_splices_payload_verbatim() { + match roundtrip_inbound(Inbound::Write { + topic: "cfg".to_string(), + payload: payload(r#"{"value":{"level":9}}"#), + }) { + Inbound::Write { topic, payload } => { + assert_eq!(topic, "cfg"); + assert_eq!(&payload[..], br#"{"value":{"level":9}}"#); + } + _ => panic!("expected Write"), + } + } + + #[test] + fn ping_pong_roundtrip() { + assert!(matches!(roundtrip_inbound(Inbound::Ping), Inbound::Ping)); + let frame = encode_outbound(Outbound::Pong); + assert!(matches!( + AimxCodec.decode_outbound(&frame).unwrap(), + Outbound::Pong + )); + } + + #[test] + fn reply_ok_roundtrip() { + let frame = encode_outbound(Outbound::Reply { + id: 11, + result: Ok(payload(r#"{"status":"success"}"#)), + }); + match AimxCodec.decode_outbound(&frame).unwrap() { + Outbound::Reply { id, result } => { + assert_eq!(id, 11); + assert_eq!(&result.unwrap()[..], br#"{"status":"success"}"#); + } + _ => panic!("expected Reply"), + } + } + + #[test] + fn reply_err_codes_roundtrip() { + for err in [RpcError::NotFound, RpcError::Denied, RpcError::Internal] { + let frame = encode_outbound(Outbound::Reply { + id: 1, + result: Err(err.clone()), + }); + match AimxCodec.decode_outbound(&frame).unwrap() { + Outbound::Reply { result, .. } => assert_eq!(result.unwrap_err(), err), + _ => panic!("expected Reply"), + } + } + } + + #[test] + fn event_roundtrip_without_topic() { + let frame = encode_outbound(Outbound::Event { + sub: "5", + seq: 2, + topic: None, + data: payload("42"), + }); + // The optional field skip-serializes — the exact-topic wire is unchanged. + assert!(!frame.windows(7).any(|w| w == b"\"topic\"")); + match AimxCodec.decode_outbound(&frame).unwrap() { + Outbound::Event { + sub, + seq, + topic, + data, + } => { + assert_eq!((sub, seq, topic), ("5", 2, None)); + assert_eq!(&data[..], b"42"); + } + _ => panic!("expected Event"), + } + } + + #[test] + fn event_roundtrip_with_topic() { + let frame = encode_outbound(Outbound::Event { + sub: "5", + seq: 9, + topic: Some("temp.vienna"), + data: payload(r#"{"c":21.5}"#), + }); + match AimxCodec.decode_outbound(&frame).unwrap() { + Outbound::Event { sub, topic, .. } => { + assert_eq!(sub, "5"); + assert_eq!(topic, Some("temp.vienna")); + } + _ => panic!("expected Event"), + } + } + + #[test] + fn snapshot_roundtrip_carries_sub_and_topic() { + let frame = encode_outbound(Outbound::Snapshot { + sub: "8", + topic: "temp.berlin", + data: payload(r#"{"c":18.0}"#), + }); + match AimxCodec.decode_outbound(&frame).unwrap() { + Outbound::Snapshot { sub, topic, data } => { + assert_eq!((sub, topic), ("8", "temp.berlin")); + assert_eq!(&data[..], br#"{"c":18.0}"#); + } + _ => panic!("expected Snapshot"), + } + } + + #[test] + fn subscribed_ack_roundtrip() { + let frame = encode_outbound(Outbound::Subscribed { sub: "13" }); + match AimxCodec.decode_outbound(&frame).unwrap() { + Outbound::Subscribed { sub } => assert_eq!(sub, "13"), + _ => panic!("expected Subscribed"), + } + } + + #[test] + fn malformed_frames_are_rejected() { + let codec = AimxCodec; + assert!(codec.decode(b"{not json").is_err()); + assert!(codec.decode(br#"{"t":"nope"}"#).is_err()); + // A `sub` frame missing its topic is malformed. + assert!(codec.decode(br#"{"t":"sub","id":1}"#).is_err()); + assert!(codec.decode_outbound(br#"{"t":"reply","id":1}"#).is_err()); + // A snap without its routing sub is malformed on the new wire. + assert!(codec + .decode_outbound(br#"{"t":"snap","topic":"x","data":1}"#) + .is_err()); + } +} diff --git a/aimdb-core/src/session/aimx/dispatch.rs b/aimdb-core/src/session/aimx/dispatch.rs index 710eb27c..d3287a23 100644 --- a/aimdb-core/src/session/aimx/dispatch.rs +++ b/aimdb-core/src/session/aimx/dispatch.rs @@ -31,7 +31,8 @@ use serde_json::{json, Value}; use crate::buffer::JsonBufferReader; use crate::remote::{AimxConfig, RecordMetadata, SecurityPolicy, WelcomeMessage, PROTOCOL_VERSION}; use crate::session::{ - AuthError, BoxFut, BoxStream, Dispatch, Payload, PeerInfo, RpcError, Session, SessionCtx, + is_wildcard, topic_matches, AuthError, BoxFut, BoxStream, Dispatch, Payload, PeerInfo, + RpcError, Session, SessionCtx, SubUpdate, }; use crate::{AimDb, DbError}; @@ -100,16 +101,37 @@ impl Session for AimxSession { fn subscribe<'a>( &'a mut self, topic: &'a str, - ) -> BoxFut<'a, Result, RpcError>> { + ) -> BoxFut<'a, Result, RpcError>> { // The engine owns the subscription lifecycle and the per-connection cap; // AimX has no async authorization, so this is a trivial wrapper. Box::pin(async move { + if is_wildcard(topic) { + return Ok(self.subscribe_wildcard(topic)); + } + // Exact-topic fast path: resolve one key, untagged updates. let stream = crate::remote::stream::stream_record_updates(&self.db, topic) .map_err(map_db_err)?; - Ok(Box::pin(stream.map(|v| to_payload(&v))) as BoxStream<'static, Payload>) + Ok(Box::pin(stream.map(|v| SubUpdate::new(to_payload(&v)))) + as BoxStream<'static, SubUpdate>) }) } + fn snapshots(&mut self, topic: &str) -> Vec<(String, Payload)> { + // Late-join state for wildcard subscriptions: the current value of every + // matched record that has one. Exact-topic AimX subscribes stay + // event-only (unchanged wire for UDS/serial/TCP clients). + if !is_wildcard(topic) { + return Vec::new(); + } + self.matched_keys(topic) + .into_iter() + .filter_map(|key| { + let value = self.db.try_latest_as_json(&key)?; + Some((key, to_payload(&value))) + }) + .collect() + } + fn write<'a>( &'a mut self, topic: &'a str, @@ -131,6 +153,46 @@ impl Session for AimxSession { } impl AimxSession { + /// Record keys matching a wildcard pattern. The record set is frozen at + /// builder time, so matching once at subscribe time is complete (design + /// 045 §3.1 — dynamic membership waits for runtime registration). + fn matched_keys(&self, pattern: &str) -> Vec { + self.db + .list_records() + .into_iter() + .map(|m: RecordMetadata| m.record_key) + .filter(|key| topic_matches(pattern, key)) + .collect() + } + + /// One wildcard subscription fans in every matched record's update stream, + /// each event tagged with the record that fired. Records without remote + /// access are skipped — the subscription covers what can stream. + fn subscribe_wildcard(&self, pattern: &str) -> BoxStream<'static, SubUpdate> { + let mut streams: Vec> = Vec::new(); + for key in self.matched_keys(pattern) { + match crate::remote::stream::stream_record_updates(&self.db, &key) { + Ok(stream) => { + let tag: Arc = Arc::from(key.as_str()); + streams.push(Box::pin( + stream.map(move |v| SubUpdate::tagged(tag.clone(), to_payload(&v))), + )); + } + Err(_e) => { + log_warn!( + "wildcard subscribe '{}': skipping '{}': {:?}", + pattern, + key, + _e + ); + } + } + } + // An empty match set yields a stream that ends immediately — with a + // builder-frozen record set, "no matches now" is "no matches ever". + Box::pin(futures_util::stream::select_all(streams)) + } + /// Match the method and produce its JSON result (or an [`RpcError`]). async fn dispatch_call(&mut self, method: &str, params: Value) -> Result { match method { diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index c002e74c..e5f432f4 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -24,6 +24,7 @@ use hashbrown::HashMap; use super::{ BoxFut, BoxStream, Connection, Dialer, EnvelopeCodec, Inbound, Outbound, Payload, RpcError, + SubUpdate, }; use crate::router::RouterBuilder; use crate::AimDb; @@ -51,10 +52,6 @@ pub struct ClientConfig { /// Cap on caller commands buffered while disconnected (oldest dropped past it). /// Defaults to `usize::MAX` (unbounded). pub max_offline_queue: usize, - /// Key the subscription demux by **topic** instead of the request `id`. - /// `false` (default): events echo the id. `true`: the wire pushes data keyed - /// by topic, so `decode_outbound` returns the topic as `Event.sub`. - pub topic_routed_subs: bool, /// Send a Ping handshake on connect and await the Pong before serving caller /// commands. A real protocol swaps Ping/Pong for its Hello. pub sends_hello: bool, @@ -69,7 +66,6 @@ impl Default for ClientConfig { max_reconnect_attempts: 0, keepalive_interval: None, max_offline_queue: usize::MAX, - topic_routed_subs: false, sends_hello: false, } } @@ -107,7 +103,7 @@ enum ClientCmd { }, Subscribe { topic: String, - events: Sender, + events: Sender, }, Write { topic: String, @@ -143,16 +139,19 @@ impl ClientHandle { /// sends the `Subscribe` request asynchronously). Dropping the stream stops /// local delivery. The stream ends on disconnect and is not re-subscribed on /// reconnect (see [`ClientConfig::reconnect`]) — re-call to resume. + /// + /// Each [`SubUpdate`] carries the concrete record topic when the server tags + /// it (wildcard subscriptions fan in many records under this one stream). pub fn subscribe( &self, topic: impl Into, - ) -> Result, RpcError> { - let (events, rx) = async_channel::unbounded::(); + ) -> Result, RpcError> { + let (events, rx) = async_channel::unbounded::(); self.enqueue(ClientCmd::Subscribe { topic: topic.into(), events, })?; - // The receiver is itself a `Stream`. + // The receiver is itself a `Stream`. Ok(Box::pin(rx)) } @@ -314,7 +313,7 @@ where let mut pending: HashMap>> = HashMap::new(); // sub-id → event sink. The sub-id is `id.to_string()` of the opening // request, matching the server's derivation so `Event.sub` routes back. - let mut subs: HashMap> = HashMap::new(); + let mut subs: HashMap> = HashMap::new(); let mut out = Vec::new(); let keepalive_ms = config.keepalive_interval; // Keepalive is deadline-based: activity only records a timestamp (one dyn @@ -389,18 +388,28 @@ where subs.remove(&id.to_string()); } } - Ok(Outbound::Event { sub, seq: _, data }) => { + Ok(Outbound::Event { + sub, + seq: _, + topic, + data, + }) => { let dead = match subs.get(sub) { - Some(tx) => tx.try_send(data).is_err(), + Some(tx) => tx + .try_send(SubUpdate { + topic: topic.map(Arc::from), + data, + }) + .is_err(), None => false, // late event for a dropped sub — ignore }; if dead { subs.remove(sub); } } - Ok(Outbound::Snapshot { topic, data }) => { - if let Some(tx) = subs.get(topic) { - let _ = tx.try_send(data); + Ok(Outbound::Snapshot { sub, topic, data }) => { + if let Some(tx) = subs.get(sub) { + let _ = tx.try_send(SubUpdate::tagged(Arc::from(topic), data)); } } Ok(Outbound::Pong) => {} @@ -468,13 +477,7 @@ where ClientCmd::Subscribe { topic, events } => { let id = next_id; next_id += 1; - // Demux key: topic (topic-routed) or the request id. - let key = if config.topic_routed_subs { - topic.clone() - } else { - id.to_string() - }; - subs.insert(key, events); + subs.insert(id.to_string(), events); out.clear(); let sent = codec .encode_inbound(Inbound::Subscribe { id, topic }, &mut out) @@ -568,8 +571,8 @@ pub fn pump_client(db: &AimDb, scheme: &str, handle: &ClientHandle) -> Vec s, Err(_e) => return, }; - while let Some(payload) = stream.next().await { - let _ = router.route(id.as_ref(), &payload, &ctx); + while let Some(update) = stream.next().await { + let _ = router.route(id.as_ref(), &update.data, &ctx); } })); } diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index 54f2269e..b9d6bf7a 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -35,6 +35,9 @@ mod server; #[cfg(all(feature = "connector-session", feature = "remote"))] pub mod aimx; +mod topic_match; +pub use topic_match::{is_wildcard, topic_matches}; + #[cfg(feature = "connector-session")] pub use client::{pump_client, run_client, ClientConfig, ClientHandle}; #[cfg(feature = "connector-session")] @@ -61,6 +64,37 @@ pub type BoxStream<'a, T> = Pin + Send + 'a>>; /// inspects them. pub type Payload = Arc<[u8]>; +/// One update delivered on a subscription stream (server [`Session::subscribe`] +/// and client [`ClientHandle::subscribe`] alike). +/// +/// `topic` names the concrete record that fired — `Some` on wildcard +/// subscriptions, which fan in many records under one subscription id (and on +/// any transport that tags every event, like the WS bus); `None` where the +/// subscription is exact-topic and the wire stays minimal. `Arc` so +/// per-event tagging is a refcount bump, not a string allocation. +#[derive(Clone)] +pub struct SubUpdate { + /// Concrete record topic that fired, when the producer side tags it. + pub topic: Option>, + /// The serialized record value. + pub data: Payload, +} + +impl SubUpdate { + /// An untagged update (exact-topic subscription). + pub fn new(data: Payload) -> Self { + Self { topic: None, data } + } + + /// A topic-tagged update (wildcard / bus subscription). + pub fn tagged(topic: Arc, data: Payload) -> Self { + Self { + topic: Some(topic), + data, + } + } +} + /// Result of a transport-layer operation. pub type TransportResult = Result; @@ -249,11 +283,18 @@ pub enum Outbound<'a> { sub: &'a str, /// Monotonic sequence number. seq: u64, + /// Concrete record topic that fired, when the subscription tags it + /// (wildcard subscriptions fan in many records — see [`SubUpdate`]). + topic: Option<&'a str>, /// Unparsed record value. data: Payload, }, /// An initial snapshot emitted when a subscription opens (late-join). Snapshot { + /// Subscription id the snapshot belongs to, so the client demux routes + /// it like an [`Event`](Outbound::Event) (a wildcard subscription's + /// snapshots carry topics that don't string-match the pattern key). + sub: &'a str, /// Topic the snapshot is for. topic: &'a str, /// Unparsed record value. @@ -350,25 +391,26 @@ pub trait Session: Send { params: Payload, ) -> BoxFut<'a, Result>; - /// Open a subscription yielding many payloads. The stream is `'static` (it - /// captures cloned handles) so it outlives the `&mut` borrow and lives in the - /// engine. Async so a connector can await per-operation authorization. + /// Open a subscription yielding many [`SubUpdate`]s. The stream is `'static` + /// (it captures cloned handles) so it outlives the `&mut` borrow and lives in + /// the engine. Async so a connector can await per-operation authorization. /// Defaulted to [`RpcError::NotFound`] for dispatches with no streaming. fn subscribe<'a>( &'a mut self, topic: &'a str, - ) -> BoxFut<'a, Result, RpcError>> { + ) -> BoxFut<'a, Result, RpcError>> { let _ = topic; Box::pin(async { Err(RpcError::NotFound) }) } - /// Late-join snapshot: the current value for `topic`, emitted as an + /// Late-join snapshots: one `(topic, value)` per record covered by `topic` + /// (several for a wildcard subscription), each emitted as an /// [`Outbound::Snapshot`] right after a successful - /// [`subscribe`](Session::subscribe) and before the first event. Defaulted to - /// `None` (no snapshot). - fn snapshot(&mut self, topic: &str) -> Option { + /// [`subscribe`](Session::subscribe) and before the first event. Defaulted + /// to empty (no snapshots). + fn snapshots(&mut self, topic: &str) -> Vec<(String, Payload)> { let _ = topic; - None + Vec::new() } /// Fire-and-forget write: no reply. Routes through the producer/arbiter path, @@ -494,7 +536,7 @@ mod tests { fn subscribe<'a>( &'a mut self, _topic: &'a str, - ) -> BoxFut<'a, Result, RpcError>> { + ) -> BoxFut<'a, Result, RpcError>> { unimplemented!() } fn write<'a>( diff --git a/aimdb-core/src/session/server.rs b/aimdb-core/src/session/server.rs index 43274661..76f7dde8 100644 --- a/aimdb-core/src/session/server.rs +++ b/aimdb-core/src/session/server.rs @@ -28,7 +28,7 @@ use hashbrown::HashMap; use super::{ BoxFut, BoxStream, Connection, Dispatch, EnvelopeCodec, Inbound, Listener, Outbound, Payload, - RpcError, SessionLimits, + RpcError, SessionLimits, SubUpdate, }; /// Per-session engine knobs. @@ -55,6 +55,9 @@ const EVENT_BUFFER: usize = 256; struct SubEvent { sub: String, seq: u64, + /// Concrete record topic, when the subscription tags its updates + /// (see [`SubUpdate::topic`]). + topic: Option>, data: Payload, } @@ -165,6 +168,7 @@ pub async fn run_session( Outbound::Event { sub: &ev.sub, seq: ev.seq, + topic: ev.topic.as_deref(), data: ev.data, }, &mut out, @@ -215,13 +219,16 @@ pub async fn run_session( break; } } - // Optional late-join snapshot, before the first event. - if let Some(data) = session.snapshot(&topic) { + // Optional late-join snapshots (one per covered + // record), before the first event. + let mut send_failed = false; + for (snap_topic, data) in session.snapshots(&topic) { out.clear(); if codec .encode( Outbound::Snapshot { - topic: &topic, + sub: &sub_id, + topic: &snap_topic, data, }, &mut out, @@ -229,9 +236,13 @@ pub async fn run_session( .is_ok() && conn.send(&out).await.is_err() { + send_failed = true; break; } } + if send_failed { + break; + } let (cancel_tx, cancel_rx) = oneshot::channel(); cancels.insert(sub_id.clone(), cancel_tx); subs.push(Box::pin(pump_subscription( @@ -305,7 +316,7 @@ async fn send_reply_err( /// later prune is a no-op. async fn pump_subscription( sub_id: String, - mut stream: BoxStream<'static, Payload>, + mut stream: BoxStream<'static, SubUpdate>, tx: Sender, cancel: oneshot::Receiver<()>, ) -> String { @@ -317,12 +328,12 @@ async fn pump_subscription( let mut seq: u64 = 0; loop { // Independent arms, so a direct `select_biased!` is fine here. - let data = select_biased! { + let update = select_biased! { // Resolves on explicit Unsubscribe (send) or on sender drop. _ = cancel => break, // `BoxStream` is not `FusedStream`, so fuse the per-iteration `next`. next = stream.next().fuse() => match next { - Some(data) => data, + Some(update) => update, None => break, // stream exhausted }, }; @@ -332,7 +343,8 @@ async fn pump_subscription( match tx.try_send(SubEvent { sub: sub_id.clone(), seq, - data, + topic: update.topic, + data: update.data, }) { Ok(()) => {} Err(e) if e.is_full() => {} // drop on overflow diff --git a/aimdb-core/src/session/topic_match.rs b/aimdb-core/src/session/topic_match.rs new file mode 100644 index 00000000..26f29d7d --- /dev/null +++ b/aimdb-core/src/session/topic_match.rs @@ -0,0 +1,122 @@ +//! MQTT-style topic pattern matching (pure `&str` ops, no_std-safe). +//! +//! Shared by every transport that supports wildcard subscriptions: the AimX +//! wildcard subscribe ([`crate::session::aimx`]) matches a pattern against the +//! registry once at subscribe time, and the WebSocket connector's fan-out bus +//! matches per broadcast. Moved here from the retired `aimdb-ws-protocol` +//! crate (design 045). + +/// Returns `true` if `topic` matches `pattern`. +/// +/// Follows MQTT wildcard conventions: +/// +/// | Pattern | Semantics | +/// |----------|-----------------------------------| +/// | `#` | Multi-level wildcard (all topics) | +/// | `a/#` | Everything under `a/` | +/// | `a/*/c` | Single-level wildcard in segment | +/// | `a/b/c` | Exact match | +pub fn topic_matches(pattern: &str, topic: &str) -> bool { + // Fast path: exact match + if pattern == topic { + return true; + } + + // Multi-level wildcard: `#` matches everything + if pattern == "#" { + return true; + } + + // `prefix/#` matches everything under prefix — only when prefix is literal + // (no wildcards in the prefix). When wildcards are present, fall through to + // the segment loop which handles `#` at any position. + if let Some(prefix) = pattern.strip_suffix("/#") { + if !prefix.contains('*') && !prefix.contains('#') { + return topic.starts_with(prefix) + && (topic.len() == prefix.len() + || topic.as_bytes().get(prefix.len()) == Some(&b'/')); + } + } + + // Segment-by-segment matching with `*` single-level wildcard + let mut pattern_parts = pattern.split('/'); + let mut topic_parts = topic.split('/'); + + loop { + match (pattern_parts.next(), topic_parts.next()) { + (Some("#"), _) => return true, + (Some("*"), Some(_)) => {} // single-level wildcard — consume one segment + (Some(p), Some(t)) if p == t => {} // literal match + (None, None) => return true, // both exhausted at the same time + _ => return false, + } + } +} + +/// Returns `true` if `pattern` contains a wildcard segment — i.e. subscribing +/// to it means "match against the registry" rather than "resolve one key". +pub fn is_wildcard(pattern: &str) -> bool { + pattern.contains('#') || pattern.contains('*') +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exact_match() { + assert!(topic_matches("a/b/c", "a/b/c")); + assert!(!topic_matches("a/b/c", "a/b/d")); + } + + #[test] + fn hash_wildcard() { + assert!(topic_matches("#", "anything/goes/here")); + assert!(topic_matches("#", "a")); + } + + #[test] + fn prefix_hash_wildcard() { + assert!(topic_matches("sensors/#", "sensors/temperature/vienna")); + assert!(topic_matches("sensors/#", "sensors/humidity/berlin")); + assert!(!topic_matches("sensors/#", "commands/setpoint")); + // Edge: prefix itself + assert!(topic_matches("sensors/#", "sensors")); + } + + #[test] + fn star_wildcard() { + assert!(topic_matches( + "sensors/temperature/*", + "sensors/temperature/vienna" + )); + assert!(topic_matches( + "sensors/temperature/*", + "sensors/temperature/berlin" + )); + assert!(!topic_matches( + "sensors/temperature/*", + "sensors/humidity/vienna" + )); + assert!(!topic_matches( + "sensors/temperature/*", + "sensors/temperature/a/b" + )); + } + + #[test] + fn mixed_wildcards() { + assert!(topic_matches("a/*/c/#", "a/b/c/d/e/f")); + assert!(!topic_matches("a/*/c/#", "a/b/x/d")); + } + + #[test] + fn wildcard_detection() { + assert!(is_wildcard("#")); + assert!(is_wildcard("sensors/#")); + assert!(is_wildcard("a/*/c")); + // Dot-separated keys are single segments — literal unless `#`/`*` appears. + assert!(!is_wildcard("temp.vienna")); + assert!(is_wildcard("temp.*")); + } +} diff --git a/aimdb-core/tests/session_engine.rs b/aimdb-core/tests/session_engine.rs index 948830ec..d05ad04c 100644 --- a/aimdb-core/tests/session_engine.rs +++ b/aimdb-core/tests/session_engine.rs @@ -20,7 +20,7 @@ use futures::StreamExt; use aimdb_core::session::{ run_client, serve, AuthError, BoxFut, BoxStream, ClientConfig, CodecError, Connection, Dialer, Dispatch, EnvelopeCodec, Inbound, Listener, Outbound, Payload, PeerInfo, RpcError, Session, - SessionConfig, SessionCtx, SessionLimits, TransportError, TransportResult, + SessionConfig, SessionCtx, SessionLimits, SubUpdate, TransportError, TransportResult, }; /// Engine-test clock (aimdb-core can't depend on a runtime adapter — that @@ -180,10 +180,24 @@ impl EnvelopeCodec for LineCodec { Ok(data) => format!("REPLY\n{}\nOK\n{}", id, utf8(&data)?), Err(e) => format!("REPLY\n{}\nERR\n{}", id, rpc_code(&e)), }, - Outbound::Event { sub, seq, data } => { - format!("EVENT\n{}\n{}\n{}", sub, seq, utf8(&data)?) + Outbound::Event { + sub, + seq, + topic, + data, + } => { + // `-` marks an untagged event (no per-record topic). + format!( + "EVENT\n{}\n{}\n{}\n{}", + sub, + seq, + topic.unwrap_or("-"), + utf8(&data)? + ) + } + Outbound::Snapshot { sub, topic, data } => { + format!("SNAP\n{}\n{}\n{}", sub, topic, utf8(&data)?) } - Outbound::Snapshot { topic, data } => format!("SNAP\n{}\n{}", topic, utf8(&data)?), Outbound::Pong => "PONG".to_string(), Outbound::Subscribed { sub } => format!("SUBSCRIBED\n{}", sub), }; @@ -225,16 +239,20 @@ impl EnvelopeCodec for LineCodec { } "EVENT" => { let (sub, r) = rest.split_once('\n').ok_or(CodecError::Malformed)?; - let (seq, data) = r.split_once('\n').unwrap_or((r, "")); + let (seq, r) = r.split_once('\n').ok_or(CodecError::Malformed)?; + let (topic, data) = r.split_once('\n').unwrap_or((r, "")); Ok(Outbound::Event { sub, seq: seq.parse().map_err(|_| CodecError::Malformed)?, + topic: (topic != "-").then_some(topic), data: payload_from(data), }) } "SNAP" => { - let (topic, data) = rest.split_once('\n').unwrap_or((rest, "")); + let (sub, r) = rest.split_once('\n').ok_or(CodecError::Malformed)?; + let (topic, data) = r.split_once('\n').unwrap_or((r, "")); Ok(Outbound::Snapshot { + sub, topic, data: payload_from(data), }) @@ -291,7 +309,7 @@ impl Session for EchoSession { fn subscribe<'a>( &'a mut self, topic: &'a str, - ) -> BoxFut<'a, Result, RpcError>> { + ) -> BoxFut<'a, Result, RpcError>> { let topic = topic.to_string(); Box::pin(async move { // Sentinel: let a known topic fail so the subscribe-ack path is testable. @@ -299,10 +317,10 @@ impl Session for EchoSession { return Err(RpcError::NotFound); } // Three synthetic updates derived from the topic, then end. - let items: Vec = (1..=3) - .map(|i| payload_from(&format!("{topic}#{i}"))) + let items: Vec = (1..=3) + .map(|i| SubUpdate::new(payload_from(&format!("{topic}#{i}")))) .collect(); - Ok(Box::pin(futures::stream::iter(items)) as BoxStream<'static, Payload>) + Ok(Box::pin(futures::stream::iter(items)) as BoxStream<'static, SubUpdate>) }) } @@ -352,7 +370,6 @@ async fn echo_roundtrip_rpc_streaming_and_write() { max_reconnect_attempts: 0, keepalive_interval: None, max_offline_queue: usize::MAX, - topic_routed_subs: false, sends_hello: true, }, Arc::new(TestClock), @@ -368,9 +385,9 @@ async fn echo_roundtrip_rpc_streaming_and_write() { let e1 = stream.next().await.expect("event 1"); let e2 = stream.next().await.expect("event 2"); let e3 = stream.next().await.expect("event 3"); - assert_eq!(&*e1, b"temp#1"); - assert_eq!(&*e2, b"temp#2"); - assert_eq!(&*e3, b"temp#3"); + assert_eq!(&*e1.data, b"temp#1"); + assert_eq!(&*e2.data, b"temp#2"); + assert_eq!(&*e3.data, b"temp#3"); // 3) Fire-and-forget write, then a follow-up RPC. FIFO on the single // connection guarantees the write frame is processed before the reply @@ -418,7 +435,6 @@ async fn failed_subscribe_ends_stream_via_ack() { max_reconnect_attempts: 0, keepalive_interval: None, max_offline_queue: usize::MAX, - topic_routed_subs: false, sends_hello: false, }, Arc::new(TestClock), @@ -477,7 +493,6 @@ async fn ended_subscription_frees_its_cap_slot() { max_reconnect_attempts: 0, keepalive_interval: None, max_offline_queue: usize::MAX, - topic_routed_subs: false, sends_hello: false, }, Arc::new(TestClock), @@ -486,13 +501,13 @@ async fn ended_subscription_frees_its_cap_slot() { // Drain a subscription's three echo updates; the server-side stream then // ends, so its pump finishes and is reaped. - async fn drain_three(stream: &mut BoxStream<'static, Payload>, topic: &str) { + async fn drain_three(stream: &mut BoxStream<'static, SubUpdate>, topic: &str) { for i in 1..=3 { let ev = tokio::time::timeout(Duration::from_secs(2), stream.next()) .await .expect("event should arrive") .expect("an accepted subscription must yield its events"); - assert_eq!(&*ev, format!("{topic}#{i}").as_bytes()); + assert_eq!(&*ev.data, format!("{topic}#{i}").as_bytes()); } } @@ -513,8 +528,8 @@ async fn ended_subscription_frees_its_cap_slot() { .await .expect("third subscribe must not hang"); assert_eq!( - first.as_deref(), - Some(&b"c#1"[..]), + first.map(|u| u.data.to_vec()), + Some(b"c#1".to_vec()), "an ended subscription must free its cap slot; the third subscribe was refused" ); diff --git a/aimdb-persistence/CHANGELOG.md b/aimdb-persistence/CHANGELOG.md index 91c373f9..dd1b19e5 100644 --- a/aimdb-persistence/CHANGELOG.md +++ b/aimdb-persistence/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed (breaking) — Design 045 + +- **`with_persistence`'s registered `QueryHandlerFn` returns the shared + `record.query` shape** `{"records": [{topic, payload, ts}, …], "total": N}` + (rows sorted by `ts` ascending) instead of `{values: [{record, value, + stored_at}], count}`, so every transport shares one query vocabulary. + ### Changed (breaking) - **Issue #131:** `RecordRegistrarPersistExt`/`AimDbBuilderPersistExt`/`AimDbQueryExt` are non-generic over the runtime (they extend `RecordRegistrar<'a, T>` / `AimDbBuilder` / `AimDb`); the retention-cleanup `on_start` task receives `RuntimeContext` and sleeps via `ctx.time().sleep_secs(...)`. diff --git a/aimdb-persistence/src/builder_ext.rs b/aimdb-persistence/src/builder_ext.rs index 907d8ab7..6f394d23 100644 --- a/aimdb-persistence/src/builder_ext.rs +++ b/aimdb-persistence/src/builder_ext.rs @@ -55,6 +55,8 @@ impl AimDbBuilderPersistExt for AimDbBuilder { }); // Register a QueryHandlerFn so AimX record.query can delegate to us. + // Result shape is the shared `{records, total}` vocabulary (design 045 + // §3.4) — one row type (`QueryRecord`) for every transport. let query_backend = backend.clone(); let handler: QueryHandlerFn = Box::new(move |params: QueryHandlerParams| { let backend = query_backend.clone(); @@ -70,21 +72,20 @@ impl AimDbBuilderPersistExt for AimDbBuilder { .await .map_err(|e| e.to_string())?; - let values: Vec = stored + let mut records: Vec = stored .into_iter() - .map(|sv| { - serde_json::json!({ - "record": sv.record_name, - "value": sv.value, - "stored_at": sv.stored_at, - }) + .map(|sv| aimdb_core::remote::QueryRecord { + topic: sv.record_name, + payload: sv.value, + ts: sv.stored_at, }) .collect(); + records.sort_by_key(|r| r.ts); - let count = values.len(); + let total = records.len(); Ok(serde_json::json!({ - "values": values, - "count": count, + "records": records, + "total": total, })) }) }); diff --git a/aimdb-persistence/tests/query_handler_shape.rs b/aimdb-persistence/tests/query_handler_shape.rs new file mode 100644 index 00000000..4b1f7458 --- /dev/null +++ b/aimdb-persistence/tests/query_handler_shape.rs @@ -0,0 +1,95 @@ +//! The `QueryHandlerFn` registered by `with_persistence` produces the shared +//! `record.query` result shape — `{"records": [{topic, payload, ts}, …], +//! "total": N}`, rows sorted by `ts` ascending (design 045 §3.4). + +use std::sync::Arc; +use std::time::Duration; + +use aimdb_core::remote::{QueryHandlerFn, QueryHandlerParams}; +use aimdb_core::AimDbBuilder; +use aimdb_persistence::{ + AimDbBuilderPersistExt, BoxFuture, PersistenceBackend, PersistenceError, QueryParams, + StoredValue, +}; +use aimdb_tokio_adapter::TokioAdapter; +use serde_json::{json, Value}; + +/// Mock backend returning fixed rows, deliberately out of `stored_at` order to +/// prove the handler sorts. +struct MockBackend; + +impl PersistenceBackend for MockBackend { + fn store<'a>( + &'a self, + _record_name: &'a str, + _value: &'a Value, + _timestamp: u64, + ) -> BoxFuture<'a, Result<(), PersistenceError>> { + Box::pin(async { Ok(()) }) + } + + fn query<'a>( + &'a self, + _record_pattern: &'a str, + _params: QueryParams, + ) -> BoxFuture<'a, Result, PersistenceError>> { + Box::pin(async { + Ok(vec![ + StoredValue { + record_name: "temp.berlin".into(), + value: json!({"celsius": 18.0}), + stored_at: 2_000, + }, + StoredValue { + record_name: "temp.vienna".into(), + value: json!({"celsius": 21.5}), + stored_at: 1_000, + }, + ]) + }) + } + + fn cleanup(&self, _older_than: u64) -> BoxFuture<'_, Result> { + Box::pin(async { Ok(0) }) + } +} + +#[tokio::test] +async fn query_handler_returns_records_total_shape() { + let (db, runner) = AimDbBuilder::new() + .runtime(Arc::new(TokioAdapter)) + .with_persistence( + Arc::new(MockBackend) as Arc, + Duration::from_secs(3600), + ) + .build() + .await + .expect("AimDb build failed"); + tokio::spawn(runner.run()); + + let handler = db + .extensions() + .get::() + .expect("with_persistence registers the query handler"); + let result = handler(QueryHandlerParams { + name: "*".into(), + limit: None, + start: None, + end: None, + }) + .await + .expect("query succeeds"); + + assert_eq!(result["total"], json!(2)); + let records = result["records"].as_array().expect("records array"); + assert_eq!(records.len(), 2); + // Sorted ascending by ts, regardless of backend order. + assert_eq!( + records[0], + json!({"topic": "temp.vienna", "payload": {"celsius": 21.5}, "ts": 1_000}) + ); + assert_eq!( + records[1], + json!({"topic": "temp.berlin", "payload": {"celsius": 18.0}, "ts": 2_000}) + ); +} From 89d8209244a38a183debf04a451fe5d1786f2ebf Mon Sep 17 00:00:00 2001 From: test Date: Fri, 17 Jul 2026 14:05:01 +0000 Subject: [PATCH 03/45] feat(websocket-connector): speak AimX on the WebSocket transport (design 045) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wire is now AimX — one tagged frame per WS text message, the same envelope as UDS/serial/TCP; the delta really was just the envelope (everything below the codec already rode the shared session engine). - run_session drives the shared AimxCodec; the 507-line per-connection WsCodec and its id↔topic maps are deleted, as are the multi-topic Subscribe split and the protocol re-export shim. - ClientManager delivers topic-tagged SubUpdates; the per-frame envelope is applied by each connection's codec (payload bytes stay Arc-shared). The pre-serialized Data frame, its server-side ts, and with_raw_payload are gone. - WsDispatch converges on the AimX vocabulary: record.query via a plugged-in QueryHandler (now returning core QueryRecord rows) or the Extensions QueryHandlerFn from with_persistence; record.list answers {name, schema_type, entity} rows (TopicInfo now lives here). Errors collapse to the 3-code set; auth stays at the HTTP 401. - SnapshotProvider::snapshots(pattern) returns every cached value under the pattern, so wildcard subscriptions late-join each covered record. - Auto-subscribe seeds AimX sub frames with ids counting down from u64::MAX (cannot collide with client-chosen ids; design 045 §3.6). - ws-client builder rides AimxCodec (topic_routed_subs gone); e2e tests rewritten to drive raw AimX frames, including golden wire frames and ws↔AimX parity for ack/wildcard/snapshot/query/list. Co-Authored-By: Claude Fable 5 --- aimdb-websocket-connector/CHANGELOG.md | 25 + aimdb-websocket-connector/Cargo.toml | 1 - .../examples/ws_server.rs | 15 +- .../src/client/builder.rs | 13 +- aimdb-websocket-connector/src/client/mod.rs | 4 +- aimdb-websocket-connector/src/codec.rs | 507 ------------------ aimdb-websocket-connector/src/lib.rs | 25 +- aimdb-websocket-connector/src/protocol.rs | 10 - aimdb-websocket-connector/src/server/auth.rs | 4 +- .../src/server/builder.rs | 55 +- .../src/server/client_manager.rs | 118 ++-- .../src/server/connector.rs | 4 +- .../src/server/dispatch.rs | 129 +++-- aimdb-websocket-connector/src/server/http.rs | 10 +- aimdb-websocket-connector/src/server/mod.rs | 10 +- .../src/server/session.rs | 69 ++- aimdb-websocket-connector/src/transport.rs | 101 +--- aimdb-websocket-connector/tests/e2e.rs | 405 +++++++------- 18 files changed, 475 insertions(+), 1030 deletions(-) delete mode 100644 aimdb-websocket-connector/src/codec.rs delete mode 100644 aimdb-websocket-connector/src/protocol.rs diff --git a/aimdb-websocket-connector/CHANGELOG.md b/aimdb-websocket-connector/CHANGELOG.md index effb7bb9..7dedb374 100644 --- a/aimdb-websocket-connector/CHANGELOG.md +++ b/aimdb-websocket-connector/CHANGELOG.md @@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed (breaking) — Design 045: the WS wire is now AimX + +- **The wire protocol is AimX** (`aimdb-core::session::aimx`), one tagged JSON + frame per WS text message — the same envelope as UDS/serial/TCP. The + `aimdb-ws-protocol` crate, the 507-line `WsCodec` (and its per-connection + id↔topic maps), the multi-topic `Subscribe` split, and the `Data`-frame + pre-serialization in `ClientManager` are deleted. Subscribing to N patterns + is N `sub` frames; events carry `sub`/`seq`/`topic`; snapshots carry the + routing `sub`; errors collapse to the 3-code AimX vocabulary + (`not_found`/`denied`/`internal` — auth stays out-of-band at the HTTP 401). +- **`record.query` / `record.list` replace `Query`/`ListTopics`.** + `QueryHandler` returns the shared `aimdb_core::remote::QueryRecord` rows; + without a plugged-in handler the dispatch now falls back to the + `QueryHandlerFn` registered by `aimdb-persistence::with_persistence` + (`NoQuery` is gone). `record.list` replies with `{name, schema_type, + entity}` rows (`TopicInfo` now lives in this crate and is serialize-only). +- **`with_raw_payload` removed** — its purpose was bypassing the ws `Data` + envelope; under AimX the envelope is the protocol. +- **`SnapshotProvider::snapshot(topic)` became `snapshots(pattern)`**, + returning every cached `(topic, value)` under the pattern, so wildcard + subscriptions late-join every covered record (previously wildcard patterns + never hit the exact-key cache). +- **Auto-subscribe ids are server-chosen** (counting down from `u64::MAX`); + engine-demuxed clients should subscribe explicitly (design 045 §3.6). + ### Internal refactors - **Adjusted to core's design-036-W1 data-plane de-`Any`.** `WsDispatch`/`WsSession` carry a concrete `RuntimeContext` (was `Option` — it was always `Some`) and the inbound `Router::route` call is synchronous; the inbound route tuples and `pump_sink` routes flow through opaquely. No public API or wire change. diff --git a/aimdb-websocket-connector/Cargo.toml b/aimdb-websocket-connector/Cargo.toml index e4ba3a01..733d5462 100644 --- a/aimdb-websocket-connector/Cargo.toml +++ b/aimdb-websocket-connector/Cargo.toml @@ -34,7 +34,6 @@ tracing = ["dep:tracing"] [dependencies] aimdb-core = { version = "1.1.0", path = "../aimdb-core", default-features = false } aimdb-data-contracts = { version = "0.4.0", path = "../aimdb-data-contracts", default-features = false } -aimdb-ws-protocol = { version = "0.1.0", path = "../aimdb-ws-protocol" } # Async runtime tokio = { version = "1", features = [ diff --git a/aimdb-websocket-connector/examples/ws_server.rs b/aimdb-websocket-connector/examples/ws_server.rs index a16a0fe1..988ed139 100644 --- a/aimdb-websocket-connector/examples/ws_server.rs +++ b/aimdb-websocket-connector/examples/ws_server.rs @@ -5,16 +5,17 @@ //! ```text //! cargo run -p aimdb-websocket-connector --example ws_server //! ``` -//! Then connect a client and subscribe to the ticking `counter` record: +//! Then connect a client and subscribe to the ticking `counter` record (the +//! wire is AimX — the same tagged frames as UDS/serial/TCP): //! ```text //! wscat -c ws://127.0.0.1:8080/ws -//! > {"type":"subscribe","topics":["counter"]} -//! < {"type":"subscribed","topics":["counter"]} -//! < {"type":"data","topic":"counter","payload":{"n":1},"ts":...} +//! > {"t":"sub","id":1,"topic":"counter"} +//! < {"t":"subscribed","sub":"1"} +//! < {"t":"event","sub":"1","seq":1,"topic":"counter","data":{"n":1}} //! ``` //! Or write to the inbound `echo` record: //! ```text -//! > {"type":"write","topic":"echo","payload":{"msg":"hi"}} +//! > {"t":"write","topic":"echo","payload":{"msg":"hi"}} //! ``` use std::sync::Arc; @@ -74,7 +75,9 @@ async fn main() { tokio::spawn(runner.run()); println!("WS server listening on ws://{addr}/ws"); - println!(" subscribe: wscat -c ws://{addr}/ws → {{\"type\":\"subscribe\",\"topics\":[\"counter\"]}}"); + println!( + " subscribe: wscat -c ws://{addr}/ws → {{\"t\":\"sub\",\"id\":1,\"topic\":\"counter\"}}" + ); let mut n = 0u64; loop { diff --git a/aimdb-websocket-connector/src/client/builder.rs b/aimdb-websocket-connector/src/client/builder.rs index f40d053f..df984a83 100644 --- a/aimdb-websocket-connector/src/client/builder.rs +++ b/aimdb-websocket-connector/src/client/builder.rs @@ -19,10 +19,9 @@ use std::pin::Pin; -use aimdb_core::session::{pump_client, run_client, ClientConfig}; +use aimdb_core::session::{aimx::AimxCodec, pump_client, run_client, ClientConfig}; use aimdb_core::ConnectorBuilder; -use crate::codec::WsCodec; use crate::transport::WsDialer; // ════════════════════════════════════════════════════════════════════ @@ -140,9 +139,8 @@ impl ConnectorBuilder for WsClientConnectorBuilder { { Box::pin(async move { // ── Engine config from the WS-specific knobs (doc 039 § 5) ── - // Reconnect/keepalive/offline-queue are now `ClientConfig`/engine - // concerns; `topic_routed_subs` keys the demux by topic (the WS wire - // pushes `Data{topic}` with no id). + // Reconnect/keepalive/offline-queue are `ClientConfig`/engine + // concerns; subscriptions are id-routed like every AimX transport. let config = ClientConfig { reconnect: self.auto_reconnect, reconnect_delay: 200, @@ -154,18 +152,17 @@ impl ConnectorBuilder for WsClientConnectorBuilder { None }, max_offline_queue: self.max_offline_queue, - topic_routed_subs: true, sends_hello: false, }; // ── Drive the shared client engine + record-mirroring pumps ── // Like `UdsClient`: `run_client` owns demux/reconnect/keepalive over - // the WS `Dialer` + per-connection `WsCodec`; `pump_client` wires + // the WS `Dialer` + the shared `AimxCodec`; `pump_client` wires // `link_to`/`link_from` routes to the handle. // The runtime's clock drives reconnect backoff/keepalive. let (handle, engine_fut) = run_client( WsDialer::new(self.url.clone()), - WsCodec::new(), + AimxCodec, config, db.runtime_ops(), ); diff --git a/aimdb-websocket-connector/src/client/mod.rs b/aimdb-websocket-connector/src/client/mod.rs index 05814c8f..e4cb893c 100644 --- a/aimdb-websocket-connector/src/client/mod.rs +++ b/aimdb-websocket-connector/src/client/mod.rs @@ -18,8 +18,8 @@ //! ```text //! AimDB (local) ←─ WsClientConnector ──WebSocket──→ AimDB (remote server) //! │ │ -//! ├─ link_to → serialize → ClientMessage::Write ───→│ -//! └─ link_from ← deserialize ← ServerMessage::Data ←──│ +//! ├─ link_to → serialize → AimX `write` frame ─────→│ +//! └─ link_from ← deserialize ← AimX `event` frame ←───│ //! ``` mod builder; diff --git a/aimdb-websocket-connector/src/codec.rs b/aimdb-websocket-connector/src/codec.rs deleted file mode 100644 index c6a4ebda..00000000 --- a/aimdb-websocket-connector/src/codec.rs +++ /dev/null @@ -1,507 +0,0 @@ -//! Per-connection WS-JSON `EnvelopeCodec`. -//! -//! Maps the WS wire ([`ClientMessage`]/[`ServerMessage`]) onto the engine's -//! `Inbound`/`Outbound` so `run_session` drives a WebSocket exactly as it -//! drives AimX. -//! -//! Per-connection (not the shared `Arc` that `serve` uses) because the codec -//! holds `id↔topic` bookkeeping: `decode` is 1→1, so the transport splits a -//! multi-topic `Subscribe` (see [`crate::transport`]) and the codec synthesizes a -//! `u64` id per topic that the `Subscribed` ack and `Unsubscribe` map back. The -//! maps sit behind a `Mutex` so the codec stays `Send + Sync` with `&self` methods. -//! -//! The hot fan-out path skips the maps: [`ClientManager::broadcast`](crate::server::client_manager) -//! serializes the complete `Data` frame once (it owns the topic) and the codec -//! writes it verbatim — O(1) in subscribers. The `Subscribed` ack and late-join -//! `Snapshot` are engine emissions the codec maps to wire frames. - -use std::collections::HashMap; -use std::sync::Mutex; - -use aimdb_core::{CodecError, Inbound, Outbound, Payload, RpcError}; -use serde_json::Value; - -use crate::protocol::{ClientMessage, ErrorCode, ServerMessage}; - -/// Per-connection id bookkeeping, behind a `Mutex` so the `&self` codec methods -/// can mutate it. -#[derive(Default)] -struct WsCodecState { - /// Monotonic id allocator for engine `Subscribe`/`Request` correlation. The - /// WS client never sees these ids — they exist only for the engine's demux. - next_id: u64, - /// WS topic → engine `Subscribe.id` (synthesized on subscribe; consulted by - /// `Unsubscribe`). - topic_to_id: HashMap, - /// Engine `sub` id → wire topic (consulted when encoding `Event`→`Data` and - /// the `Subscribed` ack). - id_to_topic: HashMap, -} - -impl WsCodecState { - /// Allocate (or reuse) the engine subscribe id for `topic`. - fn alloc_sub(&mut self, topic: &str) -> u64 { - if let Some(id) = self.topic_to_id.get(topic) { - return *id; - } - self.next_id += 1; - let id = self.next_id; - self.topic_to_id.insert(topic.to_string(), id); - self.id_to_topic.insert(id, topic.to_string()); - id - } - - /// Allocate a bare correlation id (for `Query`/`ListTopics` requests). - fn alloc_req(&mut self) -> u64 { - self.next_id += 1; - self.next_id - } - - /// Drop the mapping for `topic`, returning its engine id if known. - fn remove_topic(&mut self, topic: &str) -> Option { - let id = self.topic_to_id.remove(topic)?; - self.id_to_topic.remove(&id); - Some(id) - } - - /// Resolve an engine `sub` id (as the engine's `&str` form) back to its topic. - fn topic_of(&self, sub: &str) -> Option { - let id: u64 = sub.parse().ok()?; - self.id_to_topic.get(&id).cloned() - } -} - -/// A per-connection WS-JSON codec. Construct one per accepted connection (server) -/// or per dialed connection (client). -pub struct WsCodec { - state: Mutex, -} - -impl WsCodec { - /// Build a fresh codec with empty id maps. - pub fn new() -> Self { - Self { - state: Mutex::new(WsCodecState::default()), - } - } -} - -impl Default for WsCodec { - fn default() -> Self { - Self::new() - } -} - -/// Parse record-value bytes as JSON, falling back to a JSON string (mirrors the -/// legacy `ClientManager` behavior so the wire is byte-identical). -pub(crate) fn parse_payload(bytes: &[u8]) -> Value { - serde_json::from_slice(bytes) - .unwrap_or_else(|_| Value::String(String::from_utf8_lossy(bytes).into_owned())) -} - -/// Map an engine [`RpcError`] to a wire [`ErrorCode`]. -fn rpc_to_code(err: &RpcError) -> ErrorCode { - match err { - RpcError::NotFound => ErrorCode::UnknownTopic, - RpcError::Denied => ErrorCode::Forbidden, - _ => ErrorCode::ServerError, - } -} - -/// Serialize a [`ServerMessage`] into `out` as one WS-JSON text frame. -fn write_server(out: &mut Vec, msg: &ServerMessage) -> Result<(), CodecError> { - let bytes = serde_json::to_vec(msg).map_err(|_| CodecError::Malformed)?; - out.extend_from_slice(&bytes); - Ok(()) -} - -impl aimdb_core::EnvelopeCodec for WsCodec { - // ---- server direction: read a ClientMessage, write a ServerMessage ------ - - fn decode(&self, frame: &[u8]) -> Result { - let msg: ClientMessage = - serde_json::from_slice(frame).map_err(|_| CodecError::Malformed)?; - let mut st = self.state.lock().unwrap(); - match msg { - // The transport splits multi-topic frames, so exactly one topic here. - ClientMessage::Subscribe { topics } => { - let topic = topics.into_iter().next().ok_or(CodecError::Malformed)?; - let id = st.alloc_sub(&topic); - Ok(Inbound::Subscribe { id, topic }) - } - ClientMessage::Unsubscribe { topics } => { - let topic = topics.into_iter().next().ok_or(CodecError::Malformed)?; - let id = st.remove_topic(&topic).ok_or(CodecError::Malformed)?; - Ok(Inbound::Unsubscribe { - sub: id.to_string(), - }) - } - ClientMessage::Write { topic, payload } => { - let bytes = serde_json::to_vec(&payload).map_err(|_| CodecError::Malformed)?; - Ok(Inbound::Write { - topic, - payload: Payload::from(bytes.as_slice()), - }) - } - ClientMessage::Ping => Ok(Inbound::Ping), - // The whole frame (incl. the WS `String` correlation id) rides as - // `params`; the dispatch parses it and the id round-trips in the - // response, so no per-request id map is needed. - ClientMessage::Query { .. } => Ok(Inbound::Request { - id: st.alloc_req(), - method: "query".to_string(), - params: Payload::from(frame), - }), - ClientMessage::ListTopics { .. } => Ok(Inbound::Request { - id: st.alloc_req(), - method: "list_topics".to_string(), - params: Payload::from(frame), - }), - } - } - - fn encode(&self, msg: Outbound<'_>, out: &mut Vec) -> Result<(), CodecError> { - match msg { - // The bus pre-serializes the complete `Data` frame once per broadcast, - // so fan-out is O(1) — the codec writes it verbatim. - Outbound::Event { data, .. } => { - out.extend_from_slice(&data); - Ok(()) - } - Outbound::Snapshot { topic, data } => write_server( - out, - &ServerMessage::Snapshot { - topic: topic.to_string(), - payload: Some(parse_payload(&data)), - }, - ), - Outbound::Subscribed { sub } => { - let topic = self - .state - .lock() - .unwrap() - .topic_of(sub) - .ok_or(CodecError::Malformed)?; - write_server( - out, - &ServerMessage::Subscribed { - topics: vec![topic], - }, - ) - } - Outbound::Pong => write_server(out, &ServerMessage::Pong), - // `Reply::Ok` payloads are already a complete `ServerMessage` JSON - // (`QueryResult`/`TopicList`/`Error`) built by the dispatch with the - // client's `String` id spliced in — write them verbatim. - Outbound::Reply { id, result } => match result { - Ok(payload) => { - out.extend_from_slice(&payload); - Ok(()) - } - Err(e) => { - // Restore the topic for subscribe/cap denials from the id↔topic - // map; Query/ListTopics use a bare id, so it stays `None`. - let topic = self.state.lock().unwrap().id_to_topic.get(&id).cloned(); - write_server( - out, - &ServerMessage::Error { - code: rpc_to_code(&e), - topic, - message: format!("{e:?}"), - }, - ) - } - }, - } - } - - // ---- client direction: write a ClientMessage, read a ServerMessage ------ - // Used by the WS client port; the client engine is topic-routed, so - // `Event.sub` carries the topic. - - fn encode_inbound(&self, msg: Inbound, out: &mut Vec) -> Result<(), CodecError> { - let client_msg = match msg { - Inbound::Subscribe { id, topic } => { - let mut st = self.state.lock().unwrap(); - st.topic_to_id.insert(topic.clone(), id); - st.id_to_topic.insert(id, topic.clone()); - ClientMessage::Subscribe { - topics: vec![topic], - } - } - Inbound::Unsubscribe { sub } => { - let topic = { - let mut st = self.state.lock().unwrap(); - st.topic_of(&sub) - .inspect(|t| { - st.remove_topic(t); - }) - .ok_or(CodecError::Malformed)? - }; - ClientMessage::Unsubscribe { - topics: vec![topic], - } - } - Inbound::Write { topic, payload } => ClientMessage::Write { - topic, - payload: parse_payload(&payload), - }, - Inbound::Ping => ClientMessage::Ping, - // `params` is already a complete `ClientMessage` JSON (`Query`/ - // `ListTopics`) — write it verbatim. - Inbound::Request { params, .. } => { - out.extend_from_slice(¶ms); - return Ok(()); - } - }; - let bytes = serde_json::to_vec(&client_msg).map_err(|_| CodecError::Malformed)?; - out.extend_from_slice(&bytes); - Ok(()) - } - - fn decode_outbound<'a>(&self, frame: &'a [u8]) -> Result, CodecError> { - // `Event`/`Snapshot` borrow `topic` zero-copy from the frame. serde's - // internally-tagged enums can't borrow, so peek the `type` tag, then - // deserialize a struct that borrows the topic slice. - let tag: TagOnly = serde_json::from_slice(frame).map_err(|_| CodecError::Malformed)?; - match tag.ty { - "data" => { - let d: TopicValueRef = - serde_json::from_slice(frame).map_err(|_| CodecError::Malformed)?; - Ok(Outbound::Event { - sub: d.topic, - seq: 0, - data: value_to_payload(d.payload), - }) - } - "snapshot" => { - let d: TopicValueRef = - serde_json::from_slice(frame).map_err(|_| CodecError::Malformed)?; - Ok(Outbound::Snapshot { - topic: d.topic, - data: value_to_payload(d.payload), - }) - } - // Informational; the engine ignores `Subscribed`, so `sub` is irrelevant. - "subscribed" => Ok(Outbound::Subscribed { sub: "" }), - "pong" => Ok(Outbound::Pong), - // Query/list/error replies aren't wired on the WS client (records - // mirror via Data/Snapshot), so map them to a benign Pong. - _ => Ok(Outbound::Pong), - } - } -} - -/// Zero-copy peek at the `"type"` discriminant (the tag is short ASCII — no escapes). -#[derive(serde::Deserialize)] -struct TagOnly<'a> { - #[serde(rename = "type", borrow)] - ty: &'a str, -} - -/// Zero-copy view of a `Data`/`Snapshot` frame: the `topic` borrows the frame. -#[derive(serde::Deserialize)] -struct TopicValueRef<'a> { - #[serde(borrow)] - topic: &'a str, - payload: Option, -} - -/// Serialize a WS payload `Value` back to record-value bytes. -fn value_to_payload(payload: Option) -> Payload { - let bytes = payload - .as_ref() - .and_then(|v| serde_json::to_vec(v).ok()) - .unwrap_or_default(); - Payload::from(bytes.as_slice()) -} - -#[cfg(test)] -mod tests { - use super::*; - use aimdb_core::EnvelopeCodec; - - fn sub(codec: &WsCodec, topic: &str) -> u64 { - let frame = serde_json::to_vec(&ClientMessage::Subscribe { - topics: vec![topic.to_string()], - }) - .unwrap(); - match codec.decode(&frame).unwrap() { - Inbound::Subscribe { id, topic: t } => { - assert_eq!(t, topic); - id - } - _ => panic!("expected Subscribe"), - } - } - - #[test] - fn event_is_written_verbatim() { - // The bus pre-serializes the complete Data frame; encode passes it through - // (O(1) fan-out). `sub`/`seq` are irrelevant on this path. - let codec = WsCodec::new(); - let frame = serde_json::to_vec(&ServerMessage::Data { - topic: "sensors/temp/vienna".into(), - payload: Some(serde_json::json!(22.5)), - ts: 123, - }) - .unwrap(); - let mut out = Vec::new(); - codec - .encode( - Outbound::Event { - sub: "ignored", - seq: 7, - data: Payload::from(frame.as_slice()), - }, - &mut out, - ) - .unwrap(); - assert_eq!(out, frame); - } - - #[test] - fn decode_outbound_borrows_topic_without_leaking() { - // Client direction: a Data frame decodes to a topic-routed Event whose - // `sub` is the topic, borrowed zero-copy from the frame. - let codec = WsCodec::new(); - let frame = serde_json::to_vec(&ServerMessage::Data { - topic: "weather/vienna".into(), - payload: Some(serde_json::json!("sunny")), - ts: 0, - }) - .unwrap(); - match codec.decode_outbound(&frame).unwrap() { - Outbound::Event { sub, data, .. } => { - assert_eq!(sub, "weather/vienna"); - assert_eq!(&data[..], b"\"sunny\""); - } - _ => panic!("expected Event"), - } - } - - #[test] - fn subscribed_ack_maps_to_topic() { - let codec = WsCodec::new(); - let id = sub(&codec, "a/b"); - let mut out = Vec::new(); - codec - .encode( - Outbound::Subscribed { - sub: &id.to_string(), - }, - &mut out, - ) - .unwrap(); - let v: ServerMessage = serde_json::from_slice(&out).unwrap(); - assert_eq!( - v, - ServerMessage::Subscribed { - topics: vec!["a/b".into()] - } - ); - } - - #[test] - fn unsubscribe_resolves_topic_to_id() { - let codec = WsCodec::new(); - let id = sub(&codec, "x/y"); - let frame = serde_json::to_vec(&ClientMessage::Unsubscribe { - topics: vec!["x/y".to_string()], - }) - .unwrap(); - match codec.decode(&frame).unwrap() { - Inbound::Unsubscribe { sub } => assert_eq!(sub, id.to_string()), - _ => panic!("expected Unsubscribe"), - } - } - - // Decoding many distinct-topic Data frames must not accumulate any - // process-lifetime state; the borrow is zero-copy. - #[test] - fn decode_outbound_high_cardinality_no_static_growth() { - let codec = WsCodec::new(); - for i in 0..10_000 { - let topic = format!("sensors/dev-{i}"); - let frame = serde_json::to_vec(&ServerMessage::Data { - topic: topic.clone(), - payload: Some(serde_json::json!(i)), - ts: 0, - }) - .unwrap(); - match codec.decode_outbound(&frame).unwrap() { - Outbound::Event { sub, .. } => assert_eq!(sub, topic), - _ => panic!("expected Event"), - } - } - } - - #[test] - fn write_carries_payload() { - let codec = WsCodec::new(); - let frame = serde_json::to_vec(&ClientMessage::Write { - topic: "cmd/set".to_string(), - payload: serde_json::json!({"v": 1}), - }) - .unwrap(); - match codec.decode(&frame).unwrap() { - Inbound::Write { topic, payload } => { - assert_eq!(topic, "cmd/set"); - let v: Value = serde_json::from_slice(&payload).unwrap(); - assert_eq!(v, serde_json::json!({"v": 1})); - } - _ => panic!("expected Write"), - } - } - - #[test] - fn query_passes_frame_as_params_and_reply_writes_verbatim() { - let codec = WsCodec::new(); - let frame = serde_json::to_vec(&ClientMessage::Query { - id: "q1".to_string(), - pattern: "#".to_string(), - from: None, - to: None, - limit: None, - }) - .unwrap(); - match codec.decode(&frame).unwrap() { - Inbound::Request { method, params, .. } => { - assert_eq!(method, "query"); - assert_eq!(¶ms[..], &frame[..]); - } - _ => panic!("expected Request"), - } - - // A dispatch reply (already a full ServerMessage) is written verbatim. - let reply = serde_json::to_vec(&ServerMessage::QueryResult { - id: "q1".to_string(), - records: vec![], - total: 0, - }) - .unwrap(); - let mut out = Vec::new(); - codec - .encode( - Outbound::Reply { - id: 7, - result: Ok(Payload::from(reply.as_slice())), - }, - &mut out, - ) - .unwrap(); - let v: ServerMessage = serde_json::from_slice(&out).unwrap(); - assert!(matches!(v, ServerMessage::QueryResult { id, .. } if id == "q1")); - } - - #[test] - fn ping_pong() { - let codec = WsCodec::new(); - let frame = serde_json::to_vec(&ClientMessage::Ping).unwrap(); - assert!(matches!(codec.decode(&frame).unwrap(), Inbound::Ping)); - let mut out = Vec::new(); - codec.encode(Outbound::Pong, &mut out).unwrap(); - let v: ServerMessage = serde_json::from_slice(&out).unwrap(); - assert_eq!(v, ServerMessage::Pong); - } -} diff --git a/aimdb-websocket-connector/src/lib.rs b/aimdb-websocket-connector/src/lib.rs index fcfee411..c2f8534a 100644 --- a/aimdb-websocket-connector/src/lib.rs +++ b/aimdb-websocket-connector/src/lib.rs @@ -13,7 +13,9 @@ //! `link_from("ws-client://host/topic")` for direct AimDB-to-AimDB sync //! without an intermediary broker. //! -//! Both modes share the same wire protocol from [`aimdb_ws_protocol`]. +//! Both modes speak **AimX** ([`aimdb_core::session::aimx`]) — the same NDJSON +//! tagged frames as the UDS/serial/TCP connectors, one frame per WS text +//! message (design 045 retired the separate ws wire protocol). //! //! ## Server Quick Start //! @@ -79,8 +81,9 @@ //! //! ## Wire Protocol //! -//! See [`protocol`] for the full message specification (re-exported from -//! [`aimdb_ws_protocol`]). +//! AimX-v2 tagged frames — see [`aimdb_core::session::aimx`] and design doc +//! 045 for the frame set (`req`/`reply`/`sub`/`subscribed`/`unsub`/`event`/ +//! `snap`/`write`/`ping`/`pong`). //! //! ## Authentication (server only) //! @@ -104,23 +107,12 @@ pub mod client; // Shared session-engine glue (server and/or client) // ════════════════════════════════════════════════════════════════════ -/// Per-connection WS-JSON `EnvelopeCodec` shared by the server (`run_session`) -/// and client (`run_client`) ports. -#[cfg(any(feature = "server", feature = "client"))] -pub mod codec; - /// WS transport adapters (`Connection`/`Dialer`) over a real WebSocket. #[cfg(any(feature = "server", feature = "client"))] pub mod transport; // Real-socket integration tests live in `tests/e2e.rs` (black-box, public API). -// ════════════════════════════════════════════════════════════════════ -// Protocol (always available) -// ════════════════════════════════════════════════════════════════════ - -pub mod protocol; - // ════════════════════════════════════════════════════════════════════ // Public re-exports // ════════════════════════════════════════════════════════════════════ @@ -144,7 +136,8 @@ pub use server::client_manager::ClientManager; #[cfg(feature = "client")] pub type WsClientConnector = client::WsClientConnectorBuilder; -pub use protocol::{ClientMessage, ErrorCode, QueryRecord, ServerMessage}; +/// Canonical `record.query` result row (shared AimX vocabulary, from core). +pub use aimdb_core::remote::QueryRecord; #[cfg(feature = "server")] -pub use server::session::{NoQuery, QueryFuture, QueryHandler}; +pub use server::session::{QueryFuture, QueryHandler, TopicInfo}; diff --git a/aimdb-websocket-connector/src/protocol.rs b/aimdb-websocket-connector/src/protocol.rs deleted file mode 100644 index 85b0b2e2..00000000 --- a/aimdb-websocket-connector/src/protocol.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Wire protocol types and topic matching for the WebSocket connector. -//! -//! This module re-exports all types from [`aimdb_ws_protocol`] for backwards -//! compatibility. The canonical definitions live in the shared protocol crate -//! so they can be used by both the server connector and browser/native clients. - -// Re-export everything from the shared protocol crate -pub use aimdb_ws_protocol::{ - now_ms, topic_matches, ClientMessage, ErrorCode, QueryRecord, ServerMessage, TopicInfo, -}; diff --git a/aimdb-websocket-connector/src/server/auth.rs b/aimdb-websocket-connector/src/server/auth.rs index 03d59379..d0727461 100644 --- a/aimdb-websocket-connector/src/server/auth.rs +++ b/aimdb-websocket-connector/src/server/auth.rs @@ -68,14 +68,14 @@ impl Permissions { pub fn can_subscribe(&self, topic: &str) -> bool { self.subscribe_patterns .iter() - .any(|p| crate::protocol::topic_matches(p, topic)) + .any(|p| aimdb_core::topic_matches(p, topic)) } /// Returns `true` if the client is allowed to write to `topic`. pub fn can_write(&self, topic: &str) -> bool { self.write_patterns .iter() - .any(|p| crate::protocol::topic_matches(p, topic)) + .any(|p| aimdb_core::topic_matches(p, topic)) } } diff --git a/aimdb-websocket-connector/src/server/builder.rs b/aimdb-websocket-connector/src/server/builder.rs index 67e4ccfa..9a064eb7 100644 --- a/aimdb-websocket-connector/src/server/builder.rs +++ b/aimdb-websocket-connector/src/server/builder.rs @@ -27,6 +27,8 @@ use aimdb_data_contracts::Streamable; use aimdb_core::{pump_sink, router::RouterBuilder, ConnectorBuilder, Dispatch}; use axum::Router as AxumRouter; +use aimdb_core::topic_matches; + use super::{ auth::{AuthHandler, DynAuthHandler, NoAuth}, client_manager::ClientManager, @@ -34,9 +36,8 @@ use super::{ dispatch::WsDispatch, http::{build_server_future, ServerState}, registry::StreamableRegistry, - session::{NoQuery, NoSnapshot, QueryHandler, SnapshotProvider}, + session::{NoSnapshot, QueryHandler, SnapshotProvider, TopicInfo}, }; -use aimdb_ws_protocol::TopicInfo; // ════════════════════════════════════════════════════════════════════ // Builder @@ -74,17 +75,12 @@ pub struct WebSocketConnectorBuilder { /// Topics to subscribe every new client to automatically on connect. /// /// When non-empty, clients receive data on these topics immediately after - /// the WebSocket handshake without having to send a `Subscribe` message. + /// the WebSocket handshake without having to send a subscribe frame. /// Use `["#"]` to push all topics to every client. auto_subscribe_topics: Vec, - /// When `true`, the serialized payload bytes are sent directly as the - /// WebSocket text frame — no `ServerMessage::Data` envelope. - /// - /// Combine with a serializer that produces a complete flat JSON object - /// (including `"type"` and `"node_id"`) to speak a custom protocol. - raw_payload: bool, - /// Handler for client `query` messages (history retrieval). - query_handler: Arc, + /// Custom handler for `record.query` calls. `None` falls back to the + /// `QueryHandlerFn` that `with_persistence` registers in Extensions. + query_handler: Option>, /// Registered streamable types for schema resolution. streamable_registry: StreamableRegistry, } @@ -100,8 +96,7 @@ impl Default for WebSocketConnectorBuilder { channel_capacity: 256, additional_routes: None, auto_subscribe_topics: Vec::new(), - raw_payload: false, - query_handler: Arc::new(NoQuery), + query_handler: None, streamable_registry: StreamableRegistry::new(), } } @@ -217,24 +212,13 @@ impl WebSocketConnectorBuilder { self } - /// Send serializer output directly as a WebSocket text frame, bypassing - /// the `{"type":"data","topic":…,"payload":…}` envelope. - /// - /// Use this when the record serializers already produce the complete JSON - /// expected by the client (e.g. `{"type":"temperature","node_id":…}`). - pub fn with_raw_payload(mut self, enabled: bool) -> Self { - self.raw_payload = enabled; - self - } - - /// Plug in a handler for client `query` messages (history retrieval). - /// - /// When set, clients can send `{"type":"query", "id":"…", "pattern":"*"}` - /// and receive a `{"type":"query_result", …}` response with persisted records. + /// Plug in a custom handler for `record.query` calls (history retrieval). /// - /// Without this, query messages receive a `server_error` response. + /// Without this, `record.query` delegates to the `QueryHandlerFn` that + /// `aimdb-persistence::with_persistence` registers in Extensions; with + /// neither, clients get `not_found`. pub fn with_query_handler(mut self, handler: impl QueryHandler + 'static) -> Self { - self.query_handler = Arc::new(handler); + self.query_handler = Some(Arc::new(handler)); self } @@ -288,7 +272,7 @@ impl ConnectorBuilder for WebSocketConnectorBuilder { self.late_join.then(|| Arc::new(Mutex::new(HashMap::new()))); // ── Client manager ──────────────────────────────────── - let client_mgr = ClientManager::new(self.raw_payload, self.channel_capacity.max(1)); + let client_mgr = ClientManager::new(self.channel_capacity.max(1)); // ── Build snapshot provider ────────────────────────── let snapshot_provider: Arc = match &snapshot_map { @@ -320,6 +304,7 @@ impl ConnectorBuilder for WebSocketConnectorBuilder { // ── Shared dispatch (one Arc per server) ─── let dispatch: Arc = Arc::new(WsDispatch { + db: db.clone(), client_mgr: client_mgr.clone(), snapshot_provider, query_handler: self.query_handler.clone(), @@ -371,7 +356,13 @@ impl ConnectorBuilder for WebSocketConnectorBuilder { struct DynMapSnapshot(SnapshotCache); impl SnapshotProvider for DynMapSnapshot { - fn snapshot(&self, topic: &str) -> Option> { - self.0.lock().ok()?.get(topic).cloned() + fn snapshots(&self, pattern: &str) -> Vec<(String, Vec)> { + let Ok(map) = self.0.lock() else { + return Vec::new(); + }; + map.iter() + .filter(|(topic, _)| topic_matches(pattern, topic)) + .map(|(topic, bytes)| (topic.clone(), bytes.clone())) + .collect() } } diff --git a/aimdb-websocket-connector/src/server/client_manager.rs b/aimdb-websocket-connector/src/server/client_manager.rs index 45c0ef00..ff7c06d6 100644 --- a/aimdb-websocket-connector/src/server/client_manager.rs +++ b/aimdb-websocket-connector/src/server/client_manager.rs @@ -2,10 +2,11 @@ //! //! [`ClientManager`] is the **fan-out bridge** behind `Dispatch::subscribe`: one //! record update reaches every matching subscription. Each `WsSession::subscribe` -//! registers a per-subscription channel and gets back a [`BoxStream`] of raw -//! record-value [`Payload`]s; the per-connection [`WsCodec`](crate::codec) wraps -//! each into a `ServerMessage::Data` on encode. The outbound record→broadcast -//! tasks ([`super::connector`]) feed [`broadcast`](ClientManager::broadcast). +//! registers a per-subscription channel and gets back a [`BoxStream`] of +//! topic-tagged [`SubUpdate`]s; the engine envelopes each into an AimX `event` +//! frame per connection (the payload bytes stay `Arc`-shared — only the small +//! envelope is per-subscriber). The outbound record→broadcast tasks +//! ([`super::connector`]) feed [`broadcast`](ClientManager::broadcast). //! //! Frame formatting lives in the codec; the per-connection send half is owned by //! `run_session`. @@ -15,22 +16,17 @@ use std::sync::{ Arc, }; -use aimdb_core::{BoxStream, Payload}; +use aimdb_core::{topic_matches, BoxStream, Payload, SubUpdate}; use dashmap::DashMap; use tokio::sync::mpsc; -use crate::{ - codec::parse_payload, - protocol::{now_ms, topic_matches, ServerMessage}, -}; - use super::auth::ClientId; /// One live subscription: a wildcard pattern + the channel feeding its stream. struct SubEntry { pattern: String, /// Bounded; `broadcast` drops on a full channel (slow-client protection). - tx: mpsc::Sender, + tx: mpsc::Sender, } /// Shared per-topic broadcast bus. Cloning is cheap (all clones share state). @@ -46,22 +42,17 @@ pub struct ClientManager { connections: Arc, /// Per-subscription channel bound (the builder's `with_channel_capacity`). sub_capacity: usize, - /// Mirrors the builder's `with_raw_payload`: when set, `broadcast` ships the - /// serializer bytes verbatim instead of wrapping them in a `Data` envelope. - raw_payload: bool, } impl ClientManager { - /// Create a new, empty bus. `raw_payload` mirrors the builder flag; - /// `sub_capacity` bounds each subscription's queue. - pub fn new(raw_payload: bool, sub_capacity: usize) -> Self { + /// Create a new, empty bus. `sub_capacity` bounds each subscription's queue. + pub fn new(sub_capacity: usize) -> Self { Self { subs: Arc::new(DashMap::new()), next_sub: Arc::new(AtomicU64::new(1)), next_client: Arc::new(AtomicU64::new(1)), connections: Arc::new(AtomicU64::new(0)), sub_capacity: sub_capacity.max(1), - raw_payload, } } @@ -84,11 +75,12 @@ impl ClientManager { } /// Register a subscription for `pattern`; returns its id and the stream of - /// matching record-value payloads. Dropping the stream ends the subscription; - /// the next matching [`broadcast`](Self::broadcast) lazily prunes the entry. - pub fn subscribe(&self, pattern: &str) -> (u64, BoxStream<'static, Payload>) { + /// topic-tagged record-value updates. Dropping the stream ends the + /// subscription; the next matching [`broadcast`](Self::broadcast) lazily + /// prunes the entry. + pub fn subscribe(&self, pattern: &str) -> (u64, BoxStream<'static, SubUpdate>) { let id = self.next_sub.fetch_add(1, Ordering::Relaxed); - let (tx, rx) = mpsc::channel::(self.sub_capacity); + let (tx, rx) = mpsc::channel::(self.sub_capacity); self.subs.insert( id, SubEntry { @@ -104,25 +96,13 @@ impl ClientManager { /// Fan a serialized record-value out to every subscription whose pattern /// matches `topic`. Dead subscriptions (dropped streams) are pruned. + /// + /// The payload and the topic tag are `Arc`-shared to every matching + /// subscription (refcount bumps, no per-subscriber copies); the per-frame + /// envelope is applied downstream by each connection's codec. pub async fn broadcast(&self, topic: &str, payload_bytes: &[u8]) { - // Serialize the complete wire frame **once** here — the bus is the only - // place the real topic + value meet, and doing it once (vs once per - // subscriber in the codec) keeps fan-out O(1). The codec writes the - // result verbatim. The same finished bytes are `Arc`-shared to every - // matching subscription (a refcount bump, no per-subscriber copy). - let frame = if self.raw_payload { - payload_bytes.to_vec() - } else { - match serde_json::to_vec(&ServerMessage::Data { - topic: topic.to_string(), - payload: Some(parse_payload(payload_bytes)), - ts: now_ms(), - }) { - Ok(f) => f, - Err(_) => return, - } - }; - let payload = Payload::from(frame.as_slice()); + let payload = Payload::from(payload_bytes); + let tag: Arc = Arc::from(topic); let mut dead: Vec = Vec::new(); for entry in self.subs.iter() { if !topic_matches(&entry.pattern, topic) { @@ -130,7 +110,10 @@ impl ClientManager { } // Bounded: drop on a full queue (slow-client protection), prune only // when the receiver is gone (stream dropped). - if let Err(mpsc::error::TrySendError::Closed(_)) = entry.tx.try_send(payload.clone()) { + if let Err(mpsc::error::TrySendError::Closed(_)) = entry + .tx + .try_send(SubUpdate::tagged(tag.clone(), payload.clone())) + { dead.push(*entry.key()); } } @@ -147,7 +130,7 @@ impl ClientManager { impl Default for ClientManager { fn default() -> Self { - Self::new(false, 256) + Self::new(256) } } @@ -169,27 +152,22 @@ mod tests { #[tokio::test] async fn broadcast_reaches_matching_subscriptions() { - let mgr = ClientManager::new(false, 256); + let mgr = ClientManager::new(256); let (_id, mut stream) = mgr.subscribe("sensors/#"); mgr.broadcast("sensors/temp/vienna", b"22.5").await; - // Delivery is the complete, pre-serialized `Data` frame (built once in - // broadcast) carrying the real topic — even for the wildcard sub. - let payload = stream.next().await.expect("should receive"); - match serde_json::from_slice::(&payload).unwrap() { - ServerMessage::Data { topic, payload, .. } => { - assert_eq!(topic, "sensors/temp/vienna"); - assert_eq!(payload, Some(serde_json::json!(22.5))); - } - _ => panic!("expected Data"), - } + // Delivery is the raw payload tagged with the real topic — even for the + // wildcard sub; the envelope is the per-connection codec's job. + let update = stream.next().await.expect("should receive"); + assert_eq!(update.topic.as_deref(), Some("sensors/temp/vienna")); + assert_eq!(&update.data[..], b"22.5"); } #[tokio::test] async fn non_matching_topic_is_not_delivered() { use futures_util::FutureExt; - let mgr = ClientManager::new(false, 256); + let mgr = ClientManager::new(256); let (_id, mut stream) = mgr.subscribe("commands/#"); mgr.broadcast("sensors/temp", b"22.5").await; // Nothing queued: the next() future is not ready. @@ -198,21 +176,18 @@ mod tests { #[tokio::test] async fn fan_out_to_n_subscribers() { - let mgr = ClientManager::new(false, 256); + let mgr = ClientManager::new(256); let mut streams: Vec<_> = (0..5).map(|_| mgr.subscribe("#").1).collect(); mgr.broadcast("any/topic", b"\"v\"").await; for s in &mut streams { - let frame = s.next().await.unwrap(); - assert!(matches!( - serde_json::from_slice::(&frame).unwrap(), - ServerMessage::Data { topic, .. } if topic == "any/topic" - )); + let update = s.next().await.unwrap(); + assert_eq!(update.topic.as_deref(), Some("any/topic")); } } #[tokio::test] async fn dropped_stream_is_pruned() { - let mgr = ClientManager::new(false, 256); + let mgr = ClientManager::new(256); let (_id, stream) = mgr.subscribe("#"); assert_eq!(mgr.subscription_count(), 1); drop(stream); @@ -220,20 +195,21 @@ mod tests { assert_eq!(mgr.subscription_count(), 0); } - // Layer 2.2 (#2): one broadcast → N subscribers all receive the *same* - // pre-serialized bytes (a shared `Arc`), evidencing a single serialization - // regardless of subscriber count (O(1) fan-out, not O(N)). + // One broadcast → N subscribers all receive the *same* payload allocation + // (a shared `Arc`), evidencing O(1) fan-out regardless of subscriber count. #[tokio::test] - async fn broadcast_serializes_once_and_shares_to_all() { - let mgr = ClientManager::new(false, 256); + async fn broadcast_shares_one_payload_to_all() { + let mgr = ClientManager::new(256); let mut streams: Vec<_> = (0..8).map(|_| mgr.subscribe("#").1).collect(); mgr.broadcast("t", b"123").await; - let mut frames = Vec::new(); + let mut updates = Vec::new(); for s in &mut streams { - frames.push(s.next().await.unwrap()); + updates.push(s.next().await.unwrap()); } - // Every subscriber got byte-identical content (the one serialized frame). - let first = &frames[0]; - assert!(frames.iter().all(|f| f.as_ref() == first.as_ref())); + let first = updates[0].data.as_ptr(); + assert!( + updates.iter().all(|u| u.data.as_ptr() == first), + "every subscriber shares the one payload Arc" + ); } } diff --git a/aimdb-websocket-connector/src/server/connector.rs b/aimdb-websocket-connector/src/server/connector.rs index 0a4d7774..7121a8ff 100644 --- a/aimdb-websocket-connector/src/server/connector.rs +++ b/aimdb-websocket-connector/src/server/connector.rs @@ -45,8 +45,8 @@ impl Connector for WsBusSink { if let Some(map) = &self.snapshot { map.lock().unwrap().insert(dest.clone(), bytes.clone()); } - // The per-connection `WsCodec` applies the `Data` envelope (or, in raw - // mode, sends the bytes verbatim) — so the bus carries raw bytes. + // The bus carries raw record-value bytes tagged with the topic; the + // per-connection AimX codec applies the `event` envelope downstream. self.client_mgr.broadcast(&dest, &bytes).await; Ok(()) }) diff --git a/aimdb-websocket-connector/src/server/dispatch.rs b/aimdb-websocket-connector/src/server/dispatch.rs index 08d50d0a..1af5cd40 100644 --- a/aimdb-websocket-connector/src/server/dispatch.rs +++ b/aimdb-websocket-connector/src/server/dispatch.rs @@ -6,29 +6,36 @@ //! application surface (the [`ClientManager`] bus handle, auth principal, query //! handler). //! -//! The id↔topic bookkeeping lives in the per-connection [`WsCodec`](crate::codec), -//! not here. The `Subscribed` ack and late-join `Snapshot` are engine emissions; -//! this session only supplies the snapshot bytes and the subscription stream. +//! The wire is AimX ([`aimdb_core::session::aimx::AimxCodec`]); this dispatch +//! only supplies WS-specific semantics on top of the shared vocabulary: +//! HTTP-upgrade auth, the bus-backed subscribe, and the `record.query` / +//! `record.list` calls (design 045 §3.4). The `Subscribed` ack and late-join +//! `Snapshot`s are engine emissions; this session only supplies the snapshot +//! bytes and the subscription stream. use std::sync::Arc; +use aimdb_core::remote::{QueryHandlerFn, QueryHandlerParams}; use aimdb_core::session::Session; -use aimdb_core::{AuthError, BoxFut, BoxStream, Dispatch, Payload, PeerInfo, RpcError, SessionCtx}; -use aimdb_ws_protocol::TopicInfo; - -use crate::protocol::{ClientMessage, ErrorCode, ServerMessage}; +use aimdb_core::{ + AuthError, BoxFut, BoxStream, Dispatch, Payload, PeerInfo, RpcError, SessionCtx, SubUpdate, +}; +use serde_json::Value; use super::{ auth::{AuthHandler, ClientId, ClientInfo, Permissions}, client_manager::{ClientManager, ConnectionGuard}, - session::{QueryHandler, Router, SnapshotProvider}, + session::{QueryHandler, Router, SnapshotProvider, TopicInfo}, }; /// The shared WS dispatch — one `Arc` per server. pub struct WsDispatch { + /// Cheap-clone db handle — resolves the Extensions-registered + /// `QueryHandlerFn` when no custom query handler is plugged in. + pub(crate) db: aimdb_core::builder::AimDb, pub(crate) client_mgr: ClientManager, pub(crate) snapshot_provider: Arc, - pub(crate) query_handler: Arc, + pub(crate) query_handler: Option>, pub(crate) router: Arc, pub(crate) known_topics: Arc>, pub(crate) auth: Arc, @@ -62,6 +69,7 @@ impl Dispatch for WsDispatch { }) }); Box::new(WsSession { + db: self.db.clone(), client_mgr: self.client_mgr.clone(), snapshot_provider: self.snapshot_provider.clone(), query_handler: self.query_handler.clone(), @@ -78,9 +86,10 @@ impl Dispatch for WsDispatch { /// One connection's per-session state (owned by the engine, `&mut`-threaded). struct WsSession { + db: aimdb_core::builder::AimDb, client_mgr: ClientManager, snapshot_provider: Arc, - query_handler: Arc, + query_handler: Option>, router: Arc, known_topics: Arc>, auth: Arc, @@ -98,38 +107,15 @@ impl Session for WsSession { params: Payload, ) -> BoxFut<'a, Result> { Box::pin(async move { - let msg: ClientMessage = - serde_json::from_slice(¶ms).map_err(|_| RpcError::Internal)?; - let response = match (method, msg) { - ( - "query", - ClientMessage::Query { - id, - pattern, - from, - to, - limit, - }, - ) => match self - .query_handler - .handle_query(&pattern, from, to, limit) - .await - { - Ok((records, total)) => ServerMessage::QueryResult { id, records, total }, - Err(message) => ServerMessage::Error { - code: ErrorCode::ServerError, - topic: None, - message, - }, - }, - ("list_topics", ClientMessage::ListTopics { id }) => ServerMessage::TopicList { - id, - topics: (*self.known_topics).clone(), - }, + let value = match method { + "record.list" => serde_json::json!(*self.known_topics), + "record.query" => { + let params: Value = serde_json::from_slice(¶ms).unwrap_or(Value::Null); + self.record_query(params).await? + } _ => return Err(RpcError::NotFound), }; - // The codec writes this complete `ServerMessage` verbatim. - let bytes = serde_json::to_vec(&response).map_err(|_| RpcError::Internal)?; + let bytes = serde_json::to_vec(&value).map_err(|_| RpcError::Internal)?; Ok(Payload::from(bytes.as_slice())) }) } @@ -137,7 +123,7 @@ impl Session for WsSession { fn subscribe<'a>( &'a mut self, topic: &'a str, - ) -> BoxFut<'a, Result, RpcError>> { + ) -> BoxFut<'a, Result, RpcError>> { Box::pin(async move { // Per-operation authorization via the async `AuthHandler` hook. if !self.auth.authorize_subscribe(&self.info, topic).await { @@ -149,13 +135,15 @@ impl Session for WsSession { }) } - fn snapshot(&mut self, topic: &str) -> Option { + fn snapshots(&mut self, topic: &str) -> Vec<(String, Payload)> { if !self.late_join { - return None; + return Vec::new(); } self.snapshot_provider - .snapshot(topic) - .map(|bytes| Payload::from(bytes.as_slice())) + .snapshots(topic) + .into_iter() + .map(|(topic, bytes)| (topic, Payload::from(bytes.as_slice()))) + .collect() } fn write<'a>( @@ -173,3 +161,54 @@ impl Session for WsSession { }) } } + +impl WsSession { + /// `record.query` with the shared `{name, limit, start, end}` params and + /// `{records, total}` result (design 045 §3.4): a plugged-in + /// [`QueryHandler`] wins; otherwise delegate to the Extensions-registered + /// `QueryHandlerFn` (`with_persistence`); neither → `NotFound`. + async fn record_query(&self, params: Value) -> Result { + let name = params + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or("*") + .to_string(); + let limit = params + .get("limit") + .and_then(|v| v.as_u64()) + .and_then(|v| usize::try_from(v).ok()); + let start = params.get("start").and_then(|v| v.as_u64()); + let end = params.get("end").and_then(|v| v.as_u64()); + + if let Some(handler) = &self.query_handler { + let (records, total) = handler + .handle_query(&name, start, end, limit) + .await + .map_err(|_e| { + #[cfg(feature = "tracing")] + tracing::warn!("record.query handler failed: {}", _e); + RpcError::Internal + })?; + return Ok(serde_json::json!({ "records": records, "total": total })); + } + + let handler_fut = { + let handler = self + .db + .extensions() + .get::() + .ok_or(RpcError::NotFound)?; + handler(QueryHandlerParams { + name, + limit, + start, + end, + }) + }; + handler_fut.await.map_err(|_e| { + #[cfg(feature = "tracing")] + tracing::warn!("record.query persistence handler failed: {}", _e); + RpcError::Internal + }) + } +} diff --git a/aimdb-websocket-connector/src/server/http.rs b/aimdb-websocket-connector/src/server/http.rs index a73b0749..b40e53c4 100644 --- a/aimdb-websocket-connector/src/server/http.rs +++ b/aimdb-websocket-connector/src/server/http.rs @@ -14,7 +14,7 @@ use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Instant}; use aimdb_core::{ - session::{run_session, SessionConfig}, + session::{aimx::AimxCodec, run_session, SessionConfig}, Connection, Dispatch, PeerInfo, SessionLimits, }; use axum::{ @@ -29,7 +29,7 @@ use axum::{ }; use tower_http::cors::CorsLayer; -use crate::{codec::WsCodec, transport::WsServerConnection}; +use crate::transport::WsServerConnection; use super::{ auth::{AuthError, AuthRequest, ClientInfo, DynAuthHandler}, @@ -179,9 +179,9 @@ async fn ws_upgrade_handler( let peer = PeerInfo::default().with_ext(Arc::new(info)); let conn: Box = Box::new(WsServerConnection::new(socket, peer, &auto_subscribe)); - let codec = WsCodec::new(); - // Per-connection codec + run_session drive this socket (doc 039 § 6). - run_session(conn, &codec, dispatch.as_ref(), &config).await; + // The shared AimX codec + run_session drive this socket (doc 045); + // each codec blob rides as one WS text frame. + run_session(conn, &AimxCodec, dispatch.as_ref(), &config).await; }) .into_response() } diff --git a/aimdb-websocket-connector/src/server/mod.rs b/aimdb-websocket-connector/src/server/mod.rs index d26b122c..fe0c6980 100644 --- a/aimdb-websocket-connector/src/server/mod.rs +++ b/aimdb-websocket-connector/src/server/mod.rs @@ -2,11 +2,11 @@ //! bridges records to them. //! //! The HTTP/WS accept loop is axum's ([`http`]); each upgraded socket is driven -//! by the shared `run_session` engine over the root [`codec`](crate::codec) / -//! [`transport`](crate::transport) substrate, with [`dispatch`] supplying the -//! subscribe/write/query semantics and [`client_manager`] the cross-connection -//! fan-out bus. The outbound data plane rides [`connector`]'s `WsBusSink` through -//! the core `pump_sink`. +//! by the shared `run_session` engine over the AimX codec +//! ([`aimdb_core::session::aimx`]) and the [`transport`](crate::transport) +//! substrate, with [`dispatch`] supplying the subscribe/write/query semantics +//! and [`client_manager`] the cross-connection fan-out bus. The outbound data +//! plane rides [`connector`]'s `WsBusSink` through the core `pump_sink`. pub mod auth; pub mod builder; diff --git a/aimdb-websocket-connector/src/server/session.rs b/aimdb-websocket-connector/src/server/session.rs index 9af7948b..ccf574ff 100644 --- a/aimdb-websocket-connector/src/server/session.rs +++ b/aimdb-websocket-connector/src/server/session.rs @@ -4,17 +4,43 @@ //! [`super::dispatch`]; what lives here is the pluggable application surface the //! dispatch consumes: //! -//! - [`QueryHandler`] — answers client `query` messages from a persistence backend; -//! - [`SnapshotProvider`] — supplies the late-join current value for a topic. +//! - [`QueryHandler`] — answers `record.query` calls from a persistence backend +//! (result rows are the shared [`QueryRecord`] vocabulary; without a custom +//! handler the dispatch falls back to the `QueryHandlerFn` that +//! `aimdb-persistence::with_persistence` registers in Extensions); +//! - [`SnapshotProvider`] — supplies the late-join current values for a +//! subscription pattern; +//! - [`TopicInfo`] — one `record.list` result row (topic name + schema/entity). use core::future::Future; use core::pin::Pin; -use crate::protocol::QueryRecord; +use serde::Serialize; +pub use aimdb_core::remote::QueryRecord; // Re-export so the builder/dispatch can use it easily. pub use aimdb_core::router::Router; +// ════════════════════════════════════════════════════════════════════ +// Topic metadata (record.list result rows) +// ════════════════════════════════════════════════════════════════════ + +/// Metadata for a single outbound topic served by a WebSocket endpoint — one +/// row of the `record.list` reply. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct TopicInfo { + /// Record key / topic name (e.g. `"temp.vienna"`). + pub name: String, + /// Schema type name (e.g. `"temperature"`), if known by the server. + #[serde(skip_serializing_if = "Option::is_none")] + pub schema_type: Option, + /// Entity / node identifier (e.g. `"vienna"`), extracted server-side from the + /// topic name. The server is the authority on naming conventions — clients + /// should use this field directly rather than parsing the topic name. + #[serde(skip_serializing_if = "Option::is_none")] + pub entity: Option, +} + // ════════════════════════════════════════════════════════════════════ // Query handler // ════════════════════════════════════════════════════════════════════ @@ -23,7 +49,7 @@ pub use aimdb_core::router::Router; pub type QueryFuture<'a> = Pin, usize), String>> + Send + 'a>>; -/// Trait for handling `Query` messages from WebSocket clients. +/// Trait for handling `record.query` calls from WebSocket clients. /// /// Implementations typically query a persistence backend and return matching /// records. The trait is async to support database I/O. @@ -31,7 +57,8 @@ pub trait QueryHandler: Send + Sync + 'static { /// Execute a history query and return `(records, total_count)`. /// /// - `pattern` — topic pattern (MQTT wildcards, `"*"` for all) - /// - `from` / `to` — time range in **milliseconds** since Unix epoch (inclusive) + /// - `from` / `to` — time range (inclusive; units are the handler's + /// contract — the persistence backend uses milliseconds since Unix epoch) /// - `limit` — max records per matching topic fn handle_query<'a>( &'a self, @@ -42,39 +69,23 @@ pub trait QueryHandler: Send + Sync + 'static { ) -> QueryFuture<'a>; } -/// A query handler that always returns an error (used when no persistence -/// backend is configured). -pub struct NoQuery; - -impl QueryHandler for NoQuery { - fn handle_query<'a>( - &'a self, - _pattern: &'a str, - _from: Option, - _to: Option, - _limit: Option, - ) -> QueryFuture<'a> { - Box::pin( - async move { Err("Query not supported — no persistence backend configured".into()) }, - ) - } -} - // ════════════════════════════════════════════════════════════════════ // Snapshot provider (late-join) // ════════════════════════════════════════════════════════════════════ -/// Provides the current serialized value of a record for late-join snapshots. +/// Provides the current serialized values covered by a subscription pattern for +/// late-join snapshots (one `(topic, value)` pair per covered record — a +/// wildcard pattern may cover several; an exact topic matches itself). pub trait SnapshotProvider: Send + Sync + 'static { - /// Return the latest serialized value for the given topic, if available. - fn snapshot(&self, topic: &str) -> Option>; + /// Return the latest serialized values for every topic matching `pattern`. + fn snapshots(&self, pattern: &str) -> Vec<(String, Vec)>; } -/// A snapshot provider that always returns `None` (late-join disabled or no data). +/// A snapshot provider that always returns nothing (late-join disabled or no data). pub struct NoSnapshot; impl SnapshotProvider for NoSnapshot { - fn snapshot(&self, _topic: &str) -> Option> { - None + fn snapshots(&self, _pattern: &str) -> Vec<(String, Vec)> { + Vec::new() } } diff --git a/aimdb-websocket-connector/src/transport.rs b/aimdb-websocket-connector/src/transport.rs index 82a9f73f..6dcb67d0 100644 --- a/aimdb-websocket-connector/src/transport.rs +++ b/aimdb-websocket-connector/src/transport.rs @@ -1,9 +1,10 @@ //! WS transport adapters — `Connection` (and, for the client, `Dialer`) over a //! real WebSocket, so the shared session engines drive it. //! -//! The **server** side (`WsServerConnection`) wraps axum's upgraded `WebSocket` -//! and performs the **multi-topic split**: a `Subscribe`/`Unsubscribe` carrying N -//! topics is yielded as N single-topic frames, so the codec's `decode` stays 1→1. +//! WebSocket frames are already message-delimited, so each AimX codec blob +//! rides as one WS text frame — no newline/length framing. The **server** side +//! (`WsServerConnection`) wraps axum's upgraded `WebSocket` and seeds the +//! builder's auto-subscribe patterns as synthetic AimX `sub` frames. #[cfg(feature = "server")] use std::collections::VecDeque; @@ -13,16 +14,12 @@ use aimdb_core::{BoxFut, Connection, PeerInfo, TransportError, TransportResult}; #[cfg(feature = "server")] use axum::extract::ws::{Message, WebSocket}; -#[cfg(feature = "server")] -use crate::protocol::ClientMessage; - /// A server-side [`Connection`] over an upgraded axum [`WebSocket`]. #[cfg(feature = "server")] pub struct WsServerConnection { ws: WebSocket, peer: PeerInfo, - /// Single-topic frames split off a multi-topic `Subscribe`/`Unsubscribe`, - /// drained before reading the next WS message. + /// Synthetic auto-subscribe frames, drained before reading the socket. pending: VecDeque>, } @@ -30,24 +27,29 @@ pub struct WsServerConnection { impl WsServerConnection { /// Wrap an upgraded socket with its pre-resolved peer identity. /// - /// `auto_subscribe` seeds synthetic single-topic `Subscribe` frames so the - /// engine subscribes the client to those patterns on connect, without a - /// client message. + /// `auto_subscribe` seeds synthetic AimX `sub` frames so the engine + /// subscribes the client to those patterns on connect, without a client + /// message. Their ids count down from `u64::MAX` so they cannot collide + /// with client-chosen ids (engine clients allocate from 1 upward); the + /// resulting events carry these server-side sub ids, so engine-demuxed + /// clients should subscribe explicitly instead (design 045 §3.6). pub fn new(ws: WebSocket, peer: PeerInfo, auto_subscribe: &[String]) -> Self { let pending = auto_subscribe .iter() - .filter_map(|t| { - serde_json::to_vec(&ClientMessage::Subscribe { - topics: vec![t.clone()], - }) + .enumerate() + .filter_map(|(i, t)| { + serde_json::to_vec(&serde_json::json!({ + "t": "sub", + "id": u64::MAX - i as u64, + "topic": t, + })) .ok() }) .collect(); Self { ws, peer, pending } } - /// Read one logical frame, expanding a multi-topic `Subscribe`/`Unsubscribe` - /// into one single-topic frame per call (the rest are queued in `pending`). + /// Read one logical frame, draining seeded auto-subscribe frames first. async fn next_logical(&mut self) -> TransportResult>> { loop { if let Some(frame) = self.pending.pop_front() { @@ -61,10 +63,6 @@ impl WsServerConnection { Some(Ok(Message::Close(_))) | None => return Ok(None), Some(Err(_)) => return Err(TransportError::Io), }; - if let Some(split) = split_multi_topic(&bytes) { - self.pending.extend(split); - continue; - } return Ok(Some(bytes)); } } @@ -91,31 +89,6 @@ impl Connection for WsServerConnection { } } -/// If `bytes` is a multi-topic `Subscribe`/`Unsubscribe`, return one re-serialized -/// single-topic frame per topic; otherwise `None` (the frame passes through). -#[cfg(feature = "server")] -fn split_multi_topic(bytes: &[u8]) -> Option>> { - let msg: ClientMessage = serde_json::from_slice(bytes).ok()?; - let (topics, is_sub) = match msg { - ClientMessage::Subscribe { topics } if topics.len() > 1 => (topics, true), - ClientMessage::Unsubscribe { topics } if topics.len() > 1 => (topics, false), - _ => return None, - }; - Some( - topics - .into_iter() - .filter_map(|t| { - let one = if is_sub { - ClientMessage::Subscribe { topics: vec![t] } - } else { - ClientMessage::Unsubscribe { topics: vec![t] } - }; - serde_json::to_vec(&one).ok() - }) - .collect(), - ) -} - // ════════════════════════════════════════════════════════════════════ // Client side — Dialer + Connection over tokio-tungstenite // ════════════════════════════════════════════════════════════════════ @@ -197,39 +170,3 @@ impl Connection for WsClientConnection { &self.peer } } - -#[cfg(all(test, feature = "server"))] -mod tests { - use super::*; - - #[test] - fn splits_multi_topic_subscribe() { - let frame = serde_json::to_vec(&ClientMessage::Subscribe { - topics: vec!["a".into(), "b".into(), "c".into()], - }) - .unwrap(); - let split = split_multi_topic(&frame).expect("should split"); - assert_eq!(split.len(), 3); - for (i, t) in ["a", "b", "c"].iter().enumerate() { - match serde_json::from_slice::(&split[i]).unwrap() { - ClientMessage::Subscribe { topics } => assert_eq!(topics, vec![t.to_string()]), - _ => panic!("expected single-topic Subscribe"), - } - } - } - - #[test] - fn single_topic_passes_through() { - let frame = serde_json::to_vec(&ClientMessage::Subscribe { - topics: vec!["only".into()], - }) - .unwrap(); - assert!(split_multi_topic(&frame).is_none()); - } - - #[test] - fn non_subscribe_passes_through() { - let frame = serde_json::to_vec(&ClientMessage::Ping).unwrap(); - assert!(split_multi_topic(&frame).is_none()); - } -} diff --git a/aimdb-websocket-connector/tests/e2e.rs b/aimdb-websocket-connector/tests/e2e.rs index e2e20d45..078b7bd4 100644 --- a/aimdb-websocket-connector/tests/e2e.rs +++ b/aimdb-websocket-connector/tests/e2e.rs @@ -2,10 +2,15 @@ //! //! These drive the connector through its **public API only**: a server `AimDb` //! stood up with [`WebSocketConnector`] over a real TCP socket, talked to by a -//! raw `tokio-tungstenite` client (or the public `run_client` + [`WsDialer`] -//! engine). Server→client data is pushed by *producing a record* — an "injector" -//! record whose dynamic topic + raw serializer let a test broadcast an arbitrary -//! `(topic, payload)` through the real `pump_sink` → bus → session path. +//! raw `tokio-tungstenite` client speaking AimX frames (or the public +//! `run_client` + [`WsDialer`] engine). Server→client data is pushed by +//! *producing a record* — an "injector" record whose dynamic topic + raw +//! serializer let a test broadcast an arbitrary `(topic, payload)` through the +//! real `pump_sink` → bus → session path. +//! +//! The parity block at the bottom locks the AimX WS wire to the semantics the +//! retired ws-protocol offered (subscribe ack, wildcard fan-out, late-join +//! snapshot, query, list — design 045 §2.1). //! //! Needs both halves (`server` + `client`); compiles away otherwise. @@ -19,15 +24,14 @@ use std::time::Duration; use aimdb_core::buffer::BufferCfg; use aimdb_core::connector::TopicProvider; -use aimdb_core::session::{run_client, ClientConfig}; +use aimdb_core::session::{aimx::AimxCodec, run_client, ClientConfig}; use aimdb_core::{AimDb, AimDbBuilder}; use aimdb_data_contracts::{SchemaType, Streamable}; use aimdb_tokio_adapter::{TokioAdapter, TokioRecordRegistrarExt}; -use aimdb_websocket_connector::codec::WsCodec; use aimdb_websocket_connector::transport::WsDialer; use aimdb_websocket_connector::{ - AuthError, AuthHandler, AuthRequest, ClientInfo, ClientMessage, ErrorCode, Permissions, - QueryFuture, QueryHandler, QueryRecord, ServerMessage, WebSocketConnector, + AuthError, AuthHandler, AuthRequest, ClientInfo, Permissions, QueryFuture, QueryHandler, + QueryRecord, WebSocketConnector, }; use futures_util::{SinkExt, StreamExt}; use serde::{Deserialize, Serialize}; @@ -52,7 +56,7 @@ impl TopicProvider for InjectTopic { } } -// ── A registered Streamable type (for the `list_topics` schema name) ── +// ── A registered Streamable type (for the `record.list` schema name) ── #[derive(Clone, Debug, Serialize, Deserialize)] struct Temp { @@ -187,14 +191,15 @@ async fn ws_connect(addr: SocketAddr) -> WsClient { .0 } -async fn ws_send(c: &mut WsClient, m: ClientMessage) { - c.send(Message::Text(serde_json::to_string(&m).unwrap().into())) +/// Send one raw AimX frame (a JSON value) as a WS text message. +async fn ws_send(c: &mut WsClient, frame: Value) { + c.send(Message::Text(frame.to_string().into())) .await .unwrap(); } -/// Read the next `ServerMessage`, with a timeout so a hang fails loudly. -async fn ws_recv(c: &mut WsClient) -> ServerMessage { +/// Read the next AimX frame as JSON, with a timeout so a hang fails loudly. +async fn ws_recv(c: &mut WsClient) -> Value { loop { match timeout(Duration::from_secs(3), c.next()) .await @@ -208,24 +213,15 @@ async fn ws_recv(c: &mut WsClient) -> ServerMessage { } } -/// Like [`ws_recv`] but returns the raw JSON with `ts` normalized to `0`. -async fn recv_value(c: &mut WsClient) -> Value { - loop { - match timeout(Duration::from_secs(3), c.next()) - .await - .expect("recv timed out") - { - Some(Ok(Message::Text(t))) => { - let mut v: Value = serde_json::from_str(&t).unwrap(); - if let Some(ts) = v.get_mut("ts") { - *ts = json!(0); - } - return v; - } - Some(Ok(Message::Ping(_))) | Some(Ok(Message::Pong(_))) => continue, - other => panic!("unexpected ws frame: {other:?}"), +/// Read frames until one has `"t" == tag`; panics on timeout. +async fn ws_recv_tag(c: &mut WsClient, tag: &str) -> Value { + for _ in 0..50 { + let v = ws_recv(c).await; + if v["t"] == tag { + return v; } } + panic!("no '{tag}' frame arrived"); } // ── Server e2e ─────────────────────────────────────────────────────── @@ -235,67 +231,57 @@ async fn server_subscribe_ack_and_wildcard_fanout() { let (addr, db) = spawn_default().await; let mut c = ws_connect(addr).await; - ws_send( - &mut c, - ClientMessage::Subscribe { - topics: vec!["sensors/#".into()], - }, - ) - .await; - assert!( - matches!(ws_recv(&mut c).await, ServerMessage::Subscribed { topics } if topics == vec!["sensors/#".to_string()]) + ws_send(&mut c, json!({"t":"sub","id":1,"topic":"sensors/#"})).await; + // Explicit ack (acks_subscribe:true): the sub id echoes the request id. + assert_eq!( + ws_recv(&mut c).await, + json!({"t":"subscribed","sub":"1"}), + "subscribe must be acked with the request id as sub id" ); - // The ack means the bus subscription is registered, so a fan-out reaches us. + // The ack means the bus subscription is registered, so a fan-out reaches us + // — tagged with the concrete topic the wildcard matched. inject(&db, "sensors/temp/vienna", json!(22.5)); - match ws_recv(&mut c).await { - ServerMessage::Data { topic, payload, .. } => { - assert_eq!(topic, "sensors/temp/vienna"); - assert_eq!(payload, Some(json!(22.5))); - } - other => panic!("expected Data, got {other:?}"), - } + let ev = ws_recv_tag(&mut c, "event").await; + assert_eq!(ev["sub"], "1"); + assert_eq!(ev["topic"], "sensors/temp/vienna"); + assert_eq!(ev["data"], json!(22.5)); } #[tokio::test] -async fn server_multi_topic_subscribe_and_unsubscribe() { +async fn server_two_subscriptions_and_unsubscribe() { let (addr, db) = spawn_default().await; let mut c = ws_connect(addr).await; - ws_send( - &mut c, - ClientMessage::Subscribe { - topics: vec!["a".into(), "b".into()], - }, - ) - .await; - let mut acked = Vec::new(); - for _ in 0..2 { - if let ServerMessage::Subscribed { topics } = ws_recv(&mut c).await { - acked.extend(topics); - } - } + // Two patterns are two `sub` frames (the multi-topic Subscribe is gone). + ws_send(&mut c, json!({"t":"sub","id":1,"topic":"a"})).await; + ws_send(&mut c, json!({"t":"sub","id":2,"topic":"b"})).await; + let mut acked = vec![ + ws_recv_tag(&mut c, "subscribed").await["sub"] + .as_str() + .unwrap() + .to_string(), + ws_recv_tag(&mut c, "subscribed").await["sub"] + .as_str() + .unwrap() + .to_string(), + ]; acked.sort(); - assert_eq!(acked, vec!["a".to_string(), "b".to_string()]); + assert_eq!(acked, vec!["1".to_string(), "2".to_string()]); inject(&db, "a", json!(1)); - assert!(matches!(ws_recv(&mut c).await, ServerMessage::Data { topic, .. } if topic == "a")); + let ev = ws_recv_tag(&mut c, "event").await; + assert_eq!(ev["sub"], "1"); + assert_eq!(ev["topic"], "a"); - // Unsubscribe "a"; a later "a" must not arrive, but "b" still does. - ws_send( - &mut c, - ClientMessage::Unsubscribe { - topics: vec!["a".into()], - }, - ) - .await; + // Unsubscribe "a" by its sub id; a later "a" must not arrive, but "b" does. + ws_send(&mut c, json!({"t":"unsub","sub":"1"})).await; tokio::time::sleep(Duration::from_millis(100)).await; // let the unsub settle inject(&db, "a", json!(2)); inject(&db, "b", json!(3)); - match ws_recv(&mut c).await { - ServerMessage::Data { topic, .. } => assert_eq!(topic, "b", "only 'b' should arrive"), - other => panic!("expected Data on b, got {other:?}"), - } + let ev = ws_recv_tag(&mut c, "event").await; + assert_eq!(ev["sub"], "2", "only 'b' should arrive"); + assert_eq!(ev["topic"], "b"); } #[tokio::test] @@ -306,28 +292,39 @@ async fn server_late_join_snapshot() { tokio::time::sleep(Duration::from_millis(100)).await; let mut c = ws_connect(addr).await; - ws_send( - &mut c, - ClientMessage::Subscribe { - topics: vec!["sensors/temp".into()], - }, - ) - .await; - assert!(matches!( + ws_send(&mut c, json!({"t":"sub","id":4,"topic":"sensors/temp"})).await; + assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"4"})); + // The snapshot rides between the ack and the first event, tagged with the + // subscription that triggered it. + assert_eq!( ws_recv(&mut c).await, - ServerMessage::Subscribed { .. } - )); - match ws_recv(&mut c).await { - ServerMessage::Snapshot { topic, payload } => { - assert_eq!(topic, "sensors/temp"); - assert_eq!(payload, Some(json!(99))); - } - other => panic!("expected Snapshot, got {other:?}"), + json!({"t":"snap","sub":"4","topic":"sensors/temp","data":99}) + ); +} + +#[tokio::test] +async fn server_wildcard_late_join_snapshots_per_match() { + let (addr, db) = spawn_default().await; + inject(&db, "sensors/temp", json!(1)); + inject(&db, "sensors/humidity", json!(2)); + tokio::time::sleep(Duration::from_millis(100)).await; + + let mut c = ws_connect(addr).await; + ws_send(&mut c, json!({"t":"sub","id":9,"topic":"sensors/#"})).await; + assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"9"})); + // One snapshot per cached record under the pattern (order is map order). + let mut snaps = Vec::new(); + for _ in 0..2 { + let s = ws_recv_tag(&mut c, "snap").await; + assert_eq!(s["sub"], "9"); + snaps.push(s["topic"].as_str().unwrap().to_string()); } + snaps.sort(); + assert_eq!(snaps, vec!["sensors/humidity", "sensors/temp"]); } #[tokio::test] -async fn server_query_and_list_topics() { +async fn server_query_and_record_list() { let addr = free_addr(); let mut ws = WebSocketConnector::new().with_query_handler(OneRecordQuery); ws.register::(); @@ -347,44 +344,57 @@ async fn server_query_and_list_topics() { let mut c = ws_connect(addr).await; + // record.query rides a plain AimX request; the reply carries the shared + // `{records, total}` shape. ws_send( &mut c, - ClientMessage::Query { - id: "q1".into(), - pattern: "#".into(), - from: None, - to: None, - limit: None, - }, + json!({"t":"req","id":10,"method":"record.query","params":{"name":"#"}}), ) .await; - match ws_recv(&mut c).await { - ServerMessage::QueryResult { id, records, total } => { - assert_eq!(id, "q1"); // the String id round-trips - assert_eq!(total, 1); - assert_eq!(records.len(), 1); - } - other => panic!("expected QueryResult, got {other:?}"), - } + let reply = ws_recv_tag(&mut c, "reply").await; + assert_eq!(reply["id"], 10); + assert_eq!(reply["ok"]["total"], 1); + assert_eq!( + reply["ok"]["records"], + json!([{"topic":"temp","payload":21.0,"ts":7}]) + ); - ws_send(&mut c, ClientMessage::ListTopics { id: "l1".into() }).await; - match ws_recv(&mut c).await { - ServerMessage::TopicList { id, topics } => { - assert_eq!(id, "l1"); - assert_eq!(topics.len(), 1); - assert_eq!(topics[0].name, "temp"); - assert_eq!(topics[0].schema_type.as_deref(), Some("temperature")); - } - other => panic!("expected TopicList, got {other:?}"), - } + // record.list returns `{name, schema_type, entity}` rows. + ws_send( + &mut c, + json!({"t":"req","id":11,"method":"record.list","params":null}), + ) + .await; + let reply = ws_recv_tag(&mut c, "reply").await; + assert_eq!(reply["id"], 11); + assert_eq!( + reply["ok"], + json!([{"name":"temp","schema_type":"temperature","entity":"temp"}]) + ); +} + +#[tokio::test] +async fn server_query_without_handler_is_not_found() { + let (addr, _db) = spawn_default().await; + let mut c = ws_connect(addr).await; + // No custom handler and no with_persistence → not_found (3-code vocabulary). + ws_send( + &mut c, + json!({"t":"req","id":3,"method":"record.query","params":{"name":"*"}}), + ) + .await; + assert_eq!( + ws_recv_tag(&mut c, "reply").await, + json!({"t":"reply","id":3,"err":"not_found"}) + ); } #[tokio::test] async fn server_ping_pong() { let (addr, _db) = spawn_default().await; let mut c = ws_connect(addr).await; - ws_send(&mut c, ClientMessage::Ping).await; - assert!(matches!(ws_recv(&mut c).await, ServerMessage::Pong)); + ws_send(&mut c, json!({"t":"ping"})).await; + assert_eq!(ws_recv(&mut c).await, json!({"t":"pong"})); } #[tokio::test] @@ -400,24 +410,50 @@ async fn server_survives_malformed_frame() { let (addr, db) = spawn_default().await; let mut c = ws_connect(addr).await; - // Garbage that is not a ClientMessage — the session must skip it, not die. + // Garbage that is not an AimX frame — the session must skip it, not die. c.send(Message::Text("{not valid".to_string().into())) .await .unwrap(); // The connection is still usable afterwards. + ws_send(&mut c, json!({"t":"sub","id":1,"topic":"x"})).await; + assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"1"})); + inject(&db, "x", json!(1)); + let ev = ws_recv_tag(&mut c, "event").await; + assert_eq!(ev["topic"], "x"); +} + +#[tokio::test] +async fn server_write_reaches_inbound_record() { + let addr = free_addr(); + let mut sb = AimDbBuilder::new() + .runtime(Arc::new(TokioAdapter)) + .with_connector(WebSocketConnector::new().bind(addr).path("/ws")); + sb.configure::("cfg", |reg| { + reg.buffer(BufferCfg::SingleLatest) + .with_remote_access() + .link_from("ws://cfg") + .with_deserializer(|_ctx, d: &[u8]| { + serde_json::from_slice::(d).map_err(|e| e.to_string()) + }) + .finish(); + }); + let (db, runner) = sb.build().await.expect("build db"); + let db = Arc::new(db); + tokio::spawn(runner.run()); + wait_for_listen(addr).await; + + let mut c = ws_connect(addr).await; + // Fire-and-forget write: the payload is the raw record value (no wrapper). ws_send( &mut c, - ClientMessage::Subscribe { - topics: vec!["x".into()], - }, + json!({"t":"write","topic":"cfg","payload":{"c":7.5}}), ) .await; - assert!(matches!( - ws_recv(&mut c).await, - ServerMessage::Subscribed { .. } - )); - inject(&db, "x", json!(1)); - assert!(matches!(ws_recv(&mut c).await, ServerMessage::Data { topic, .. } if topic == "x")); + // FIFO on the one connection: the pong proves the write frame was processed. + ws_send(&mut c, json!({"t":"ping"})).await; + assert_eq!(ws_recv(&mut c).await, json!({"t":"pong"})); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(db.try_latest_as_json("cfg"), Some(json!({"c":7.5}))); } // ── Client engine e2e (run_client + WsDialer over a real socket) ───── @@ -427,13 +463,12 @@ async fn client_engine_receives_broadcast_over_real_socket() { let (addr, db) = spawn_default().await; let config = ClientConfig { - topic_routed_subs: true, reconnect: false, ..ClientConfig::default() }; let (handle, engine) = run_client( WsDialer::new(format!("ws://{addr}/ws")), - WsCodec::new(), + AimxCodec, config, Arc::new(TokioAdapter), ); @@ -450,8 +485,10 @@ async fn client_engine_receives_broadcast_over_real_socket() { break; } } - // The record value round-trips: Data{payload:42} → engine stream yields b"42". - assert_eq!(&got.expect("a value")[..], b"42"); + // The record value round-trips; the bus tags every event with its topic. + let update = got.expect("a value"); + assert_eq!(&update.data[..], b"42"); + assert_eq!(update.topic.as_deref(), Some("sensors/temp")); drop(handle); drop(stream); @@ -467,24 +504,16 @@ async fn many_clients_fanout() { let mut clients = Vec::new(); for _ in 0..20 { let mut c = ws_connect(addr).await; - ws_send( - &mut c, - ClientMessage::Subscribe { - topics: vec!["evt/#".into()], - }, - ) - .await; - assert!(matches!( - ws_recv(&mut c).await, - ServerMessage::Subscribed { .. } - )); + ws_send(&mut c, json!({"t":"sub","id":1,"topic":"evt/#"})).await; + assert_eq!(ws_recv(&mut c).await["t"], "subscribed"); clients.push(c); } // One broadcast reaches all 20. inject(&db, "evt/x", json!(1)); for c in &mut clients { - assert!(matches!(ws_recv(c).await, ServerMessage::Data { topic, .. } if topic == "evt/x")); + let ev = ws_recv_tag(c, "event").await; + assert_eq!(ev["topic"], "evt/x"); } } @@ -494,26 +523,11 @@ async fn stalled_client_does_not_block_a_healthy_one() { // Stalled: subscribes but never reads — its socket backpressures. let mut stalled = ws_connect(addr).await; - ws_send( - &mut stalled, - ClientMessage::Subscribe { - topics: vec!["x".into()], - }, - ) - .await; + ws_send(&mut stalled, json!({"t":"sub","id":1,"topic":"x"})).await; let mut healthy = ws_connect(addr).await; - ws_send( - &mut healthy, - ClientMessage::Subscribe { - topics: vec!["x".into()], - }, - ) - .await; - assert!(matches!( - ws_recv(&mut healthy).await, - ServerMessage::Subscribed { .. } - )); + ws_send(&mut healthy, json!({"t":"sub","id":1,"topic":"x"})).await; + assert_eq!(ws_recv(&mut healthy).await["t"], "subscribed"); tokio::time::sleep(Duration::from_millis(100)).await; // let the stalled sub register // Flood well past the bounded funnel (256). This also overruns the injector @@ -524,9 +538,10 @@ async fn stalled_client_does_not_block_a_healthy_one() { inject(&db, "x", json!(i)); } - assert!( - matches!(ws_recv(&mut healthy).await, ServerMessage::Data { topic, .. } if topic == "x"), - "healthy client must keep receiving past a stalled peer", + let ev = ws_recv_tag(&mut healthy, "event").await; + assert_eq!( + ev["topic"], "x", + "healthy client must keep receiving past a stalled peer" ); let _ = stalled.close(None).await; } @@ -540,30 +555,21 @@ async fn golden_wire_frames() { tokio::time::sleep(Duration::from_millis(100)).await; let mut c = ws_connect(addr).await; - ws_send( - &mut c, - ClientMessage::Subscribe { - topics: vec!["t".into()], - }, - ) - .await; + ws_send(&mut c, json!({"t":"sub","id":1,"topic":"t"})).await; + assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"1"})); assert_eq!( - recv_value(&mut c).await, - json!({"type": "subscribed", "topics": ["t"]}) - ); - assert_eq!( - recv_value(&mut c).await, - json!({"type": "snapshot", "topic": "t", "payload": 5}) + ws_recv(&mut c).await, + json!({"t":"snap","sub":"1","topic":"t","data":5}) ); inject(&db, "t", json!(42)); assert_eq!( - recv_value(&mut c).await, - json!({"type": "data", "topic": "t", "payload": 42, "ts": 0}) + ws_recv(&mut c).await, + json!({"t":"event","sub":"1","seq":1,"topic":"t","data":42}) ); - ws_send(&mut c, ClientMessage::Ping).await; - assert_eq!(recv_value(&mut c).await, json!({"type": "pong"})); + ws_send(&mut c, json!({"t":"ping"})).await; + assert_eq!(ws_recv(&mut c).await, json!({"t":"pong"})); } // ── Async authorization over a real socket ─────────────────────────── @@ -574,32 +580,17 @@ async fn async_authorize_subscribe_gates_despite_allow_all_permissions() { let mut c = ws_connect(addr).await; // Denied topic: permissions are allow-all, but the *async* hook says no. - ws_send( - &mut c, - ClientMessage::Subscribe { - topics: vec!["secret/x".into()], - }, - ) - .await; - match ws_recv(&mut c).await { - ServerMessage::Error { code, .. } => assert!(matches!(code, ErrorCode::Forbidden)), - other => panic!("expected Forbidden Error for denied subscribe, got {other:?}"), - } + // The refusal is a `reply` carrying the subscribe id + the 3-code error. + ws_send(&mut c, json!({"t":"sub","id":1,"topic":"secret/x"})).await; + assert_eq!( + ws_recv(&mut c).await, + json!({"t":"reply","id":1,"err":"denied"}) + ); // An allowed topic still works end-to-end. - ws_send( - &mut c, - ClientMessage::Subscribe { - topics: vec!["public/x".into()], - }, - ) - .await; - assert!(matches!( - ws_recv(&mut c).await, - ServerMessage::Subscribed { .. } - )); + ws_send(&mut c, json!({"t":"sub","id":2,"topic":"public/x"})).await; + assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"2"})); inject(&db, "public/x", json!(1)); - assert!( - matches!(ws_recv(&mut c).await, ServerMessage::Data { topic, .. } if topic == "public/x") - ); + let ev = ws_recv_tag(&mut c, "event").await; + assert_eq!(ev["topic"], "public/x"); } From 2ea53d21e6e6dce764e42b2ec7f51e139ca085a7 Mon Sep 17 00:00:00 2001 From: test Date: Fri, 17 Jul 2026 14:05:16 +0000 Subject: [PATCH 04/45] feat(wasm-adapter): rewrite WsBridge on run_client with the AimX codec (design 045) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser is now a first-class engine client: a web_sys::WebSocket- backed Connection/Dialer pair (JS events funneled into the engine's async recv; single-threaded wasm Send shims, same pattern and compile guard as SendFuture) drives run_client + AimxCodec, so reply and subscription correlation, reconnect backoff, keepalive, and the offline queue exist exactly once — in core. The hand-rolled 835-line demux is gone. The #[wasm_bindgen] surface is preserved (write, query, listTopics, onStatusChange, status, disconnect, connectBridge options; lateJoin is retained for option-shape compatibility — snapshots are server-driven under AimX). Subscription pumps mirror topic-tagged updates into local records and re-subscribe when a stream ends; the queued subscribe replays after the engine redials. WasmDb.discover speaks a one-shot record.list request and resolves {name, schema_type, entity} rows. Depends on aimdb-core's connector-session + remote features (the engines cross-compile to wasm32); the aimdb-ws-protocol dep is gone. Co-Authored-By: Claude Fable 5 --- aimdb-wasm-adapter/CHANGELOG.md | 17 + aimdb-wasm-adapter/Cargo.toml | 8 +- aimdb-wasm-adapter/src/bindings.rs | 44 +- aimdb-wasm-adapter/src/ws_bridge.rs | 940 +++++++++++++--------------- 4 files changed, 471 insertions(+), 538 deletions(-) diff --git a/aimdb-wasm-adapter/CHANGELOG.md b/aimdb-wasm-adapter/CHANGELOG.md index cae60a3f..b82b0090 100644 --- a/aimdb-wasm-adapter/CHANGELOG.md +++ b/aimdb-wasm-adapter/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed (breaking) — Design 045: `WsBridge` is an AimX engine client + +- **`WsBridge` rewritten on `run_client` + `ClientHandle`** over a + `web_sys::WebSocket`-backed `Connection`/`Dialer`: reply/subscription + correlation, reconnect backoff, keepalive, and the offline queue now live in + `aimdb-core`'s client engine — the hand-rolled 835-line demux is gone. The + `#[wasm_bindgen]` surface is preserved (`write`, `query`, `listTopics`, + `onStatusChange`, `status`, `disconnect`, `connectBridge` options), but the + wire is AimX, so the bridge only talks to design-045 servers. + `BridgeOptions.lateJoin` is retained for option-shape compatibility + (snapshots are server-driven under AimX). +- **`WasmDb.discover` / the raw discovery path speak `record.list`** and + resolve with `{name, schema_type, entity}` rows. +- Depends on `aimdb-core`'s `connector-session` + `remote` features (the + engines cross-compile to wasm32); the `aimdb-ws-protocol` dependency is + gone. + ### Fixed - **SingleLatest fresh-subscriber parity (Design 040).** A subscriber created diff --git a/aimdb-wasm-adapter/Cargo.toml b/aimdb-wasm-adapter/Cargo.toml index 625aa2d2..a9b4ad77 100644 --- a/aimdb-wasm-adapter/Cargo.toml +++ b/aimdb-wasm-adapter/Cargo.toml @@ -26,14 +26,14 @@ wasm-runtime = [ ] [dependencies] -# Core AimDB types (alloc only — no std, no tokio) +# Core AimDB types (no std, no tokio — the connector-session engines and the +# AimX codec cross-compile to wasm32; the bridge rides `run_client`) aimdb-core = { version = "1.1.0", path = "../aimdb-core", default-features = false, features = [ "alloc", + "connector-session", + "remote", ] } -# Shared WebSocket wire protocol -aimdb-ws-protocol = { version = "0.1.0", path = "../aimdb-ws-protocol" } - # Data contracts (alloc only — no std) aimdb-data-contracts = { version = "0.4.0", path = "../aimdb-data-contracts", default-features = false, features = [ "alloc", diff --git a/aimdb-wasm-adapter/src/bindings.rs b/aimdb-wasm-adapter/src/bindings.rs index c33f1bb3..e6b0577c 100644 --- a/aimdb-wasm-adapter/src/bindings.rs +++ b/aimdb-wasm-adapter/src/bindings.rs @@ -33,8 +33,6 @@ use aimdb_core::buffer::BufferCfg; use aimdb_core::builder::{AimDb, AimDbBuilder}; use aimdb_core::record_id::StringKey; -use aimdb_ws_protocol::{ClientMessage, ServerMessage}; - use crate::schema_registry::{SchemaOps, SchemaRegistry}; use crate::ws_bridge::WsBridge; use crate::WasmAdapter; @@ -248,7 +246,7 @@ impl WasmDb { /// Discover topics served at `url` without building a full database. /// - /// Opens a one-shot WebSocket, sends `ListTopics`, and resolves with + /// Opens a one-shot WebSocket, sends an AimX `record.list` request, and resolves with /// `TopicInfo[]` once the server responds. Rejects after 30 s if no /// response arrives, or immediately on connection error. /// @@ -344,6 +342,10 @@ impl WasmDb { /// Build a one-shot WebSocket promise that resolves with `TopicInfo[]`. /// +/// Speaks one raw AimX exchange: `{"t":"req","id":1,"method":"record.list"}` +/// → `{"t":"reply","id":1,"ok":[{name, schema_type, entity}, …]}` (design 045 +/// §2.1 — discovery is a plain `record.list` request). +/// /// Each callback pair (resolve, reject) is stored in an `Rc>` /// so that whichever event fires first wins and subsequent events are no-ops. fn discover_impl(url: String) -> js_sys::Promise { @@ -363,29 +365,25 @@ fn discover_impl(url: String) -> js_sys::Promise { Rc::new(RefCell::new(Some(resolve))); let reject_rc: Rc>> = Rc::new(RefCell::new(Some(reject))); - // on_open: send ListTopics + // on_open: send the record.list request { let ws_clone = ws.clone(); let on_open = Closure::wrap(Box::new(move || { - let msg = ClientMessage::ListTopics { - id: "discover".to_string(), - }; - if let Ok(json) = serde_json::to_string(&msg) { - let _ = ws_clone.send_with_str(&json); - } + let _ = ws_clone + .send_with_str(r#"{"t":"req","id":1,"method":"record.list","params":null}"#); }) as Box); ws.set_onopen(Some(on_open.as_ref().unchecked_ref())); on_open.forget(); } - // on_message: parse TopicList, resolve, close socket + // on_message: parse the reply, resolve, close socket { let ws_clone = ws.clone(); let resolve_clone = resolve_rc.clone(); let reject_clone = reject_rc.clone(); let on_message = Closure::wrap(Box::new(move |event: web_sys::MessageEvent| { - let _ = ws_clone.close(); let Some(text) = event.data().as_string() else { + let _ = ws_clone.close(); if let Some(rej) = reject_clone.borrow_mut().take() { let _ = rej.call1( &JsValue::NULL, @@ -394,11 +392,19 @@ fn discover_impl(url: String) -> js_sys::Promise { } return; }; - match serde_json::from_str::(&text) { - Ok(ServerMessage::TopicList { topics, .. }) => { + let Ok(frame) = serde_json::from_str::(&text) else { + return; // skip non-JSON noise; the timeout guards a dead peer + }; + // Ignore unrelated frames (auto-subscribe events, acks, …). + if frame["t"] != "reply" || frame["id"] != 1 { + return; + } + let _ = ws_clone.close(); + match frame.get("ok").and_then(|v| v.as_array()) { + Some(topics) => { let serializer = serde_wasm_bindgen::Serializer::json_compatible(); let arr = js_sys::Array::new(); - for topic in &topics { + for topic in topics { if let Ok(js_val) = topic.serialize(&serializer) { arr.push(&js_val); } @@ -407,11 +413,11 @@ fn discover_impl(url: String) -> js_sys::Promise { let _ = res.call1(&JsValue::NULL, &arr); } } - _ => { + None => { if let Some(rej) = reject_clone.borrow_mut().take() { let _ = rej.call1( &JsValue::NULL, - &JsValue::from_str("Unexpected server message"), + &JsValue::from_str("record.list failed on server"), ); } } @@ -436,7 +442,7 @@ fn discover_impl(url: String) -> js_sys::Promise { on_error.forget(); } - // on_close: reject if server closed before we got TopicList + // on_close: reject if server closed before the record.list reply // (no-op if on_message already resolved) { let reject_clone = reject_rc.clone(); @@ -444,7 +450,7 @@ fn discover_impl(url: String) -> js_sys::Promise { if let Some(rej) = reject_clone.borrow_mut().take() { let _ = rej.call1( &JsValue::NULL, - &JsValue::from_str("Connection closed before TopicList received"), + &JsValue::from_str("Connection closed before record.list reply"), ); } }) as Box); diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index 58f935ef..cf599da2 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -1,15 +1,17 @@ //! WebSocket bridge connecting browser AimDB to a server instance. //! -//! `WsBridge` wraps `web_sys::WebSocket` and speaks the shared wire protocol -//! from [`aimdb_ws_protocol`] (`ServerMessage` / `ClientMessage`). It maps -//! incoming server data to local buffer pushes and forwards local writes to -//! the server. +//! `WsBridge` rides the shared session engine: a [`web_sys::WebSocket`]-backed +//! [`Connection`]/[`Dialer`] pair drives [`run_client`] with the AimX codec +//! ([`AimxCodec`]), so reply/subscription correlation, reconnect backoff, +//! keepalive, and the offline queue all live in +//! `aimdb-core/src/session/client.rs` — exactly once (design 045 Phase 3). +//! This module only adapts the JS-event-driven socket to the engine's async +//! `recv`/`send` interface and maps incoming updates to local buffer pushes. //! //! # Modes //! //! - **Synchronized** — browser instance mirrors server records. //! - **Hybrid** — works offline with local records, syncs when connected. -//! extern crate alloc; @@ -18,16 +20,26 @@ use alloc::collections::BTreeMap; use alloc::format; use alloc::rc::Rc; use alloc::string::{String, ToString}; +use alloc::sync::Arc; use alloc::vec::Vec; -use core::cell::RefCell; -use core::fmt::Debug; +use core::cell::{Cell, RefCell}; +use core::future::Future; +use core::pin::Pin; +use core::task::{Context, Poll}; -use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; +use futures_util::StreamExt; +use serde::Deserialize; use wasm_bindgen::prelude::*; -use crate::schema_registry::SchemaRegistry; use aimdb_core::builder::AimDb; +use aimdb_core::session::aimx::AimxCodec; +use aimdb_core::session::{run_client, ClientConfig, ClientHandle}; +use aimdb_core::{ + BoxFut, Connection, Dialer, PeerInfo, RpcError, RuntimeOps, TransportError, TransportResult, +}; + +use crate::schema_registry::SchemaRegistry; +use crate::WasmAdapter; // ─── Connection status ──────────────────────────────────────────────────── @@ -51,10 +63,6 @@ impl ConnectionStatus { } } -// ─── Wire protocol (shared with aimdb-websocket-connector) ─────────────── - -use aimdb_ws_protocol::{ClientMessage, ServerMessage}; - // ─── Bridge configuration ───────────────────────────────────────────────── /// Configuration for `WasmDb.connectBridge()`. @@ -67,10 +75,13 @@ pub struct BridgeOptions { /// Re-connect automatically on close (default: true). #[serde(default = "default_true")] pub auto_reconnect: bool, - /// Request snapshots on (re)connect (default: true). + /// Retained for option-shape compatibility: late-join snapshots are + /// server-driven under AimX (`with_late_join` on the server builder) and + /// arrive automatically on subscribe. #[serde(default = "default_true")] pub late_join: bool, - /// Maximum queued writes while disconnected (default: 256). + /// Maximum queued commands (writes/subscribes) while disconnected + /// (default: 256). #[serde(default = "default_queue_size")] pub max_offline_queue: usize, /// Keepalive interval in milliseconds (default: 30 000). @@ -108,58 +119,226 @@ impl Default for BridgeOptions { } } -// ─── Bridge state ───────────────────────────────────────────────────────── - -struct BridgeState { - status: ConnectionStatus, - pending_writes: alloc::collections::VecDeque, - backoff_index: usize, - /// Active keepalive interval ID (cleared on close/disconnect). - keepalive_id: Option, - /// Keepalive ping closure (prevent GC while interval is active). - _ping_closure: Option>, - /// Closures retained to prevent GC. - _on_open: Option>, - _on_message: Option>, - _on_close: Option>, - _on_error: Option>, +// ─── Single-threaded `Send` shims ───────────────────────────────────────── + +/// Wraps a JS-touching future so it satisfies the engine's `Send` bounds. +/// +/// # Safety +/// +/// Only ever executed on `wasm32-unknown-unknown` without the `atomics` +/// proposal — single-threaded by construction (the compile guard in +/// [`crate::time`] rejects wasm threads). The unconditional impl exists so the +/// crate also *compiles* in the native test lane, where this code never runs — +/// the same rationale as [`WsBridge`]'s `Send`/`Sync` impls below. +struct JsSendFut(F); + +unsafe impl Send for JsSendFut {} + +impl Future for JsSendFut { + type Output = F::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // SAFETY: pure pin projection to the inner field. + let inner = unsafe { self.map_unchecked_mut(|s| &mut s.0) }; + inner.poll(cx) + } } -// ─── Shared reconnect context ───────────────────────────────────────────── +// ─── Shared bridge state ────────────────────────────────────────────────── + +/// State shared between the JS-facing [`WsBridge`], the transport wrappers, +/// and the subscription pumps. +struct BridgeShared { + status: Cell, + on_status: RefCell>, + /// Set by `disconnect()` — stops redials and ends the pumps. + stopped: Cell, + /// Whether a connection ever succeeded (Connecting vs Reconnecting). + ever_connected: Cell, + auto_reconnect: bool, + /// The live socket, for a prompt close on `disconnect()`. + ws: RefCell>, +} -/// Tag for routing server responses to the correct pending promise. -#[derive(Debug, Clone, Copy)] -enum RequestKind { - Query, - ListTopics, +impl BridgeShared { + /// Transition the observable status (deduplicated) and notify JS. + fn set_status(&self, status: ConnectionStatus) { + if self.status.get() == status { + return; + } + self.status.set(status); + emit_status(&self.on_status, status); + } + + /// The status to report when a connection ends. + fn drop_status(&self) -> ConnectionStatus { + if self.stopped.get() || !self.auto_reconnect { + ConnectionStatus::Disconnected + } else { + ConnectionStatus::Reconnecting + } + } } -/// A pending request waiting for a server response. -struct PendingRequest { - /// Reserved for future use when response routing depends on request type. - _kind: RequestKind, - resolve: js_sys::Function, - reject: js_sys::Function, +// ─── Transport: web_sys::WebSocket as Connection/Dialer ────────────────── + +/// Frames arriving from JS event callbacks, funneled to the engine's `recv`. +type FrameRx = futures_channel::mpsc::UnboundedReceiver>; + +/// A dialed browser WebSocket serving the engine's [`Connection`] contract. +struct WasmWsConnection { + ws: web_sys::WebSocket, + frames: FrameRx, + shared: Rc, + peer: PeerInfo, + /// JS callbacks kept alive for the socket's lifetime. + _callbacks: Vec>, + _plain_callbacks: Vec>, } -/// Shared state needed by both the initial connect and reconnect paths. -/// -/// Wrapped in `Rc` so closures can cheaply reference it without cloning -/// every field individually (reduces parameter explosion). -struct SharedCtx { - db: AimDb, - schema_map: BTreeMap, - registry: SchemaRegistry, - state: Rc>, - on_status: Rc>>, - config: BridgeOptions, - backoff: Vec, +// SAFETY: wasm32 without atomics is single-threaded; see `JsSendFut`. +unsafe impl Send for WasmWsConnection {} + +impl Connection for WasmWsConnection { + fn recv(&mut self) -> BoxFut<'_, TransportResult>>> { + // `Ok(None)` when the frame funnel closes (socket closed/errored). + Box::pin(JsSendFut(async move { Ok(self.frames.next().await) })) + } + + fn send<'a>(&'a mut self, frame: &'a [u8]) -> BoxFut<'a, TransportResult<()>> { + // `send_with_str` is synchronous — resolve before the future so no JS + // reference is held across an await. + let result = core::str::from_utf8(frame) + .map_err(|_| TransportError::Io) + .and_then(|text| self.ws.send_with_str(text).map_err(|_| TransportError::Io)); + Box::pin(async move { result }) + } + + fn peer(&self) -> &PeerInfo { + &self.peer + } +} + +impl Drop for WasmWsConnection { + fn drop(&mut self) { + // Detach the JS callbacks before they drop, and close the socket if the + // engine lets go of a still-open connection (graceful stop). + self.ws.set_onopen(None); + self.ws.set_onmessage(None); + self.ws.set_onclose(None); + self.ws.set_onerror(None); + let _ = self.ws.close(); + if self + .shared + .ws + .borrow() + .as_ref() + .is_some_and(|current| current == &self.ws) + { + self.shared.ws.borrow_mut().take(); + } + } +} + +/// Dials `url` with a fresh `web_sys::WebSocket`, resolving once `onopen` +/// fires. `run_client` calls this on every (re)dial; the backoff between +/// attempts is the engine's. +struct WasmWsDialer { url: String, - ws_cell: Rc>, - /// Pending request-response pairs: correlation ID → pending request. - pending_requests: Rc>>, - /// Counter for generating unique request IDs. - request_id_counter: Rc>, + shared: Rc, +} + +// SAFETY: wasm32 without atomics is single-threaded; see `JsSendFut`. +unsafe impl Send for WasmWsDialer {} + +impl Dialer for WasmWsDialer { + fn connect(&self) -> BoxFut<'_, TransportResult>> { + let url = self.url.clone(); + let shared = self.shared.clone(); + Box::pin(JsSendFut(async move { + if shared.stopped.get() { + return Err(TransportError::Closed); + } + if shared.ever_connected.get() { + shared.set_status(ConnectionStatus::Reconnecting); + } + + let ws = web_sys::WebSocket::new(&url).map_err(|_| TransportError::Io)?; + + // Frame funnel: onmessage pushes text frames; onclose closes it so + // the engine's `recv` observes end-of-stream. + let (frame_tx, frames) = futures_channel::mpsc::unbounded::>(); + // Open handshake: whichever of onopen/onclose fires first wins. + let opened: Rc>>> = + Rc::new(RefCell::new(None)); + let (open_tx, open_rx) = futures_channel::oneshot::channel::(); + *opened.borrow_mut() = Some(open_tx); + + let mut msg_callbacks = Vec::new(); + let mut plain_callbacks = Vec::new(); + + let on_open = { + let opened = opened.clone(); + Closure::wrap(Box::new(move || { + if let Some(tx) = opened.borrow_mut().take() { + let _ = tx.send(true); + } + }) as Box) + }; + ws.set_onopen(Some(on_open.as_ref().unchecked_ref())); + plain_callbacks.push(on_open); + + let on_message = { + let frame_tx = frame_tx.clone(); + Closure::wrap(Box::new(move |event: web_sys::MessageEvent| { + if let Some(text) = event.data().as_string() { + let _ = frame_tx.unbounded_send(text.into_bytes()); + } + }) as Box) + }; + ws.set_onmessage(Some(on_message.as_ref().unchecked_ref())); + msg_callbacks.push(on_message); + + let on_close = { + let opened = opened.clone(); + let shared = shared.clone(); + Closure::wrap(Box::new(move || { + // Before open: fail the dial. After: end the frame stream. + if let Some(tx) = opened.borrow_mut().take() { + let _ = tx.send(false); + } + frame_tx.close_channel(); + shared.set_status(shared.drop_status()); + }) as Box) + }; + ws.set_onclose(Some(on_close.as_ref().unchecked_ref())); + plain_callbacks.push(on_close); + + // The browser always follows `error` with `close`; nothing to do. + let on_error = Closure::wrap(Box::new(move || { + web_sys::console::warn_1(&"WsBridge: WebSocket error".into()); + }) as Box); + ws.set_onerror(Some(on_error.as_ref().unchecked_ref())); + plain_callbacks.push(on_error); + + if open_rx.await != Ok(true) { + return Err(TransportError::Io); + } + + shared.ever_connected.set(true); + *shared.ws.borrow_mut() = Some(ws.clone()); + shared.set_status(ConnectionStatus::Connected); + + Ok(Box::new(WasmWsConnection { + ws, + frames, + shared, + peer: PeerInfo::default(), + _callbacks: msg_callbacks, + _plain_callbacks: plain_callbacks, + }) as Box) + })) + } } // ─── WsBridge ───────────────────────────────────────────────────────────── @@ -182,7 +361,10 @@ struct SharedCtx { /// ``` #[wasm_bindgen] pub struct WsBridge { - ctx: Rc, + shared: Rc, + /// Dropped on `disconnect()` — stopping the engine gracefully. + handle: Rc>>, + query_timeout_ms: u32, } // SAFETY: wasm32-unknown-unknown is single-threaded. @@ -200,76 +382,45 @@ impl WsBridge { pub fn on_status_change(&self, callback: js_sys::Function) { // Immediately replay current status so late registrations don't // miss the "connected" transition that may have already fired. - let current = self.ctx.state.borrow().status; + let current = self.shared.status.get(); let _ = callback.call1(&JsValue::NULL, &JsValue::from_str(current.as_str())); // Store for subsequent status changes - *self.ctx.on_status.borrow_mut() = Some(callback); + *self.shared.on_status.borrow_mut() = Some(callback); } - /// Send a value to the server for a given topic. + /// Send a value to the server for a given topic (AimX `write` frame). /// - /// If the WebSocket is disconnected, the message is queued (up to - /// `maxOfflineQueue`). Queued messages are flushed on reconnect. + /// While disconnected the command is queued by the engine (up to + /// `maxOfflineQueue`) and flushed on reconnect. pub fn write(&self, topic: &str, payload: JsValue) -> Result<(), JsError> { let json_payload: serde_json::Value = serde_wasm_bindgen::from_value(payload) .map_err(|e| JsError::new(&format!("Payload serialization failed: {e}")))?; - - let msg = ClientMessage::Write { - topic: topic.to_string(), - payload: json_payload, - }; - - let state = self.ctx.state.borrow(); - if state.status == ConnectionStatus::Connected { - drop(state); - send_json(&self.ctx.ws_cell.borrow(), &msg)?; - } else { - drop(state); - let mut state = self.ctx.state.borrow_mut(); - if state.pending_writes.len() < self.ctx.config.max_offline_queue { - state.pending_writes.push_back(msg); - } else { - web_sys::console::warn_1( - &format!( - "[WsBridge] Offline queue full ({} messages), dropping write for topic '{}'", - self.ctx.config.max_offline_queue, topic - ) - .into(), - ); - } - } - Ok(()) + let bytes = serde_json::to_vec(&json_payload) + .map_err(|e| JsError::new(&format!("Payload serialization failed: {e}")))?; + self.with_handle(|handle| { + handle + .write(topic, aimdb_core::Payload::from(bytes.as_slice())) + .map_err(|e| JsError::new(&format!("write failed: {}", rpc_err_str(&e)))) + }) } /// Close the WebSocket and stop reconnection attempts. pub fn disconnect(&self) { - let mut state = self.ctx.state.borrow_mut(); - state.status = ConnectionStatus::Disconnected; - // Clear keepalive timer - if let Some(id) = state.keepalive_id.take() { - if let Some(window) = web_sys::window() { - window.clear_interval_with_handle(id); - } + self.shared.stopped.set(true); + // Dropping the handle stops the engine (pending calls reject). + self.handle.borrow_mut().take(); + if let Some(ws) = self.shared.ws.borrow_mut().take() { + let _ = ws.close(); } - // Drop closures to break Rc cycles - state._ping_closure = None; - state._on_open = None; - state._on_message = None; - state._on_close = None; - state._on_error = None; - drop(state); - - let _ = self.ctx.ws_cell.borrow().close(); - reject_all_pending(&self.ctx.pending_requests, "Disconnected"); - emit_status(&self.ctx.on_status, ConnectionStatus::Disconnected); + self.shared.set_status(ConnectionStatus::Disconnected); } /// Current connection status as a string. pub fn status(&self) -> String { - self.ctx.state.borrow().status.as_str().to_string() + self.shared.status.get().as_str().to_string() } - /// Query historical / persisted records over the WebSocket connection. + /// Query historical / persisted records (AimX `record.query`). /// /// Returns a `Promise` that resolves with `{ records, total }`. /// @@ -289,19 +440,16 @@ impl WsBridge { serde_wasm_bindgen::from_value(options).unwrap_or_default() }; - let pattern = pattern.to_string(); - send_request(&self.ctx, RequestKind::Query, move |id| { - ClientMessage::Query { - id, - pattern, - from: opts.from, - to: opts.to, - limit: opts.limit, - } - }) + let params = serde_json::json!({ + "name": pattern, + "start": opts.from, + "end": opts.to, + "limit": opts.limit, + }); + self.call_as_promise("record.query", params) } - /// List all topics served by the WebSocket endpoint. + /// List all topics served by the endpoint (AimX `record.list`). /// /// Returns a `Promise` that resolves with `TopicInfo[]`. /// @@ -310,9 +458,7 @@ impl WsBridge { /// ``` #[wasm_bindgen(js_name = "listTopics")] pub fn list_topics(&self) -> js_sys::Promise { - send_request(&self.ctx, RequestKind::ListTopics, |id| { - ClientMessage::ListTopics { id } - }) + self.call_as_promise("record.list", serde_json::Value::Null) } } @@ -340,404 +486,177 @@ impl WsBridge { .map_err(|e| JsError::new(&format!("Invalid bridge options: {e}")))? }; - let ws = web_sys::WebSocket::new(url) - .map_err(|e| JsError::new(&format!("WebSocket open failed: {e:?}")))?; - - let ws_cell = Rc::new(RefCell::new(ws)); - let state = Rc::new(RefCell::new(BridgeState { - status: ConnectionStatus::Connecting, - pending_writes: alloc::collections::VecDeque::new(), - backoff_index: 0, - keepalive_id: None, - _ping_closure: None, - _on_open: None, - _on_message: None, - _on_close: None, - _on_error: None, - })); - let on_status: Rc>> = Rc::new(RefCell::new(None)); - let backoff = alloc::vec![500, 1_000, 2_000, 4_000, 8_000]; - - let ctx = Rc::new(SharedCtx { - db, - schema_map, - registry, - state, - on_status, - config, - backoff, - url: url.to_string(), - ws_cell, - pending_requests: Rc::new(RefCell::new(BTreeMap::new())), - request_id_counter: Rc::new(RefCell::new(0)), + let shared = Rc::new(BridgeShared { + status: Cell::new(ConnectionStatus::Connecting), + on_status: RefCell::new(None), + stopped: Cell::new(false), + ever_connected: Cell::new(false), + auto_reconnect: config.auto_reconnect, + ws: RefCell::new(None), }); - install_ws_callbacks(&ctx); - - Ok(WsBridge { ctx }) - } -} - -// ─── Unified request helper ──────────────────────────────────────────────── - -/// Send a request-response message and return a JS Promise. -/// -/// Generates a unique ID, registers a pending request, sends the message, -/// and optionally schedules a timeout that rejects the promise if the -/// server doesn't respond in time. -fn send_request( - ctx: &Rc, - kind: RequestKind, - build_msg: impl FnOnce(String) -> ClientMessage, -) -> js_sys::Promise { - let ctx = ctx.clone(); - - let id = { - let mut counter = ctx.request_id_counter.borrow_mut(); - *counter += 1; - format!("r{}", *counter) - }; - - let id_for_promise = id.clone(); - let mut build_msg = Some(build_msg); - js_sys::Promise::new(&mut move |resolve, reject| { - let build_msg = build_msg.take().expect("Promise executor called twice"); - ctx.pending_requests.borrow_mut().insert( - id_for_promise.clone(), - PendingRequest { - _kind: kind, - resolve, - reject: reject.clone(), - }, - ); - - let msg = build_msg(id_for_promise.clone()); - - let state = ctx.state.borrow(); - if state.status != ConnectionStatus::Connected { - drop(state); - ctx.pending_requests.borrow_mut().remove(&id_for_promise); - let _ = reject.call1(&JsValue::NULL, &JsValue::from_str("Not connected")); - return; + // Reconnect/keepalive/offline-queue are engine concerns; the backoff + // ladder mirrors the retired hand-rolled bridge (500 ms → 8 s). + let engine_config = ClientConfig { + reconnect: config.auto_reconnect, + reconnect_delay: 500, + max_reconnect_delay: 8_000, + max_reconnect_attempts: 0, + keepalive_interval: (config.keepalive_ms > 0).then_some(config.keepalive_ms as u64), + max_offline_queue: config.max_offline_queue, + sends_hello: false, + }; + let dialer = WasmWsDialer { + url: url.to_string(), + shared: shared.clone(), + }; + let (handle, engine_fut) = + run_client(dialer, AimxCodec, engine_config, Arc::new(WasmAdapter)); + wasm_bindgen_futures::spawn_local(engine_fut); + + // One pump per configured pattern: mirror tagged updates into local + // records, re-subscribing when a stream ends (disconnect) — the + // subscribe command queues offline and replays after the redial. + for pattern in &config.subscribe_topics { + wasm_bindgen_futures::spawn_local(pump_pattern( + shared.clone(), + handle.clone(), + db.clone(), + schema_map.clone(), + registry.clone(), + pattern.clone(), + )); } - drop(state); - if let Err(e) = send_json(&ctx.ws_cell.borrow(), &msg) { - ctx.pending_requests.borrow_mut().remove(&id_for_promise); - let _ = reject.call1( - &JsValue::NULL, - &JsValue::from_str(&format!("Send failed: {e:?}")), - ); - return; - } + Ok(WsBridge { + shared, + handle: Rc::new(RefCell::new(Some(handle))), + query_timeout_ms: config.query_timeout_ms, + }) + } - // Schedule timeout if configured - if ctx.config.query_timeout_ms > 0 { - let pending = ctx.pending_requests.clone(); - let timeout_id = id_for_promise.clone(); - let timeout_closure = Closure::once(move || { - if let Some(req) = pending.borrow_mut().remove(&timeout_id) { - let _ = req - .reject - .call1(&JsValue::NULL, &JsValue::from_str("Request timed out")); - } - }); - if let Some(window) = web_sys::window() { - let _ = window.set_timeout_with_callback_and_timeout_and_arguments_0( - timeout_closure.as_ref().unchecked_ref(), - ctx.config.query_timeout_ms as i32, - ); - } - timeout_closure.forget(); + /// Run `f` with the live engine handle, or fail if `disconnect()` ran. + fn with_handle( + &self, + f: impl FnOnce(&ClientHandle) -> Result, + ) -> Result { + match self.handle.borrow().as_ref() { + Some(handle) => f(handle), + None => Err(JsError::new("Bridge is disconnected")), } - }) -} - -/// Reject all pending requests with the given reason. -/// -/// Called on disconnect and on_close to prevent hung promises. -fn reject_all_pending(pending: &Rc>>, reason: &str) { - let mut map = pending.borrow_mut(); - let drained: Vec = core::mem::take(&mut *map).into_values().collect(); - for req in drained { - let _ = req.reject.call1(&JsValue::NULL, &JsValue::from_str(reason)); } -} - -// ─── Callback installation (shared by connect + reconnect) ──────────────── - -/// Install WebSocket event callbacks on the current socket in `ctx.ws_cell`. -/// -/// Extracted as a free function so both the initial `connect` and -/// `schedule_reconnect` paths can call it. -fn install_ws_callbacks(ctx: &Rc) { - let ws = ctx.ws_cell.borrow(); - - // on_open - let on_open = { - let ctx = ctx.clone(); - Closure::wrap(Box::new(move || { - { - let mut s = ctx.state.borrow_mut(); - s.status = ConnectionStatus::Connected; - s.backoff_index = 0; - - // Flush pending writes - let ws = ctx.ws_cell.borrow(); - while let Some(msg) = s.pending_writes.pop_front() { - if let Ok(json) = serde_json::to_string(&msg) { - let _ = ws.send_with_str(&json); - } - } - } - - // Subscribe to configured topics - let topics = &ctx.config.subscribe_topics; - if !topics.is_empty() { - let sub = ClientMessage::Subscribe { - topics: topics.clone(), - }; - if let Ok(json) = serde_json::to_string(&sub) { - let _ = ctx.ws_cell.borrow().send_with_str(&json); - } - } - - // Start keepalive ping timer - if ctx.config.keepalive_ms > 0 { - let ws_for_ping = ctx.ws_cell.clone(); - let ping_closure = Closure::wrap(Box::new(move || { - let ping = ClientMessage::Ping; - if let Ok(json) = serde_json::to_string(&ping) { - let _ = ws_for_ping.borrow().send_with_str(&json); - } - }) as Box); - - if let Some(window) = web_sys::window() { - if let Ok(id) = window.set_interval_with_callback_and_timeout_and_arguments_0( - ping_closure.as_ref().unchecked_ref(), - ctx.config.keepalive_ms as i32, - ) { - let mut s = ctx.state.borrow_mut(); - s.keepalive_id = Some(id); - s._ping_closure = Some(ping_closure); - } - } - } - - let has_cb = ctx.on_status.borrow().is_some(); - web_sys::console::log_1( - &format!("[WsBridge] on_open: status=connected, callback={has_cb}").into(), - ); - emit_status(&ctx.on_status, ConnectionStatus::Connected); - }) as Box) - }; - ws.set_onopen(Some(on_open.as_ref().unchecked_ref())); - - // on_message — route server data to local buffers (no JsValue hop) - let on_message = { - let ctx = ctx.clone(); - Closure::wrap(Box::new(move |event: web_sys::MessageEvent| { - if let Some(text) = event.data().as_string() { - if let Ok(msg) = serde_json::from_str::(&text) { - handle_server_message(&ctx, msg); - } - } - }) as Box) - }; - ws.set_onmessage(Some(on_message.as_ref().unchecked_ref())); - - // on_close — reconnect with exponential backoff - let on_close = { - let ctx = ctx.clone(); - Closure::wrap(Box::new(move || { - let current = ctx.state.borrow().status; - if current == ConnectionStatus::Disconnected { - return; // user-initiated disconnect — don't reconnect - } - // Clear keepalive timer and closure - { - let mut s = ctx.state.borrow_mut(); - if let Some(id) = s.keepalive_id.take() { - if let Some(window) = web_sys::window() { - window.clear_interval_with_handle(id); + /// Issue an AimX call and expose it as a JS Promise (with the configured + /// timeout), resolving with the JSON result converted to a JS value. + fn call_as_promise(&self, method: &'static str, params: serde_json::Value) -> js_sys::Promise { + let handle = self.handle.borrow().clone(); + let timeout_ms = self.query_timeout_ms; + wasm_bindgen_futures::future_to_promise(async move { + let handle = handle.ok_or_else(|| JsValue::from_str("Bridge is disconnected"))?; + let params = serde_json::to_vec(¶ms) + .map_err(|e| JsValue::from_str(&format!("params serialization failed: {e}")))?; + let call = handle.call(method, aimdb_core::Payload::from(params.as_slice())); + + let reply = if timeout_ms > 0 { + let timeout = + WasmAdapter.sleep(core::time::Duration::from_millis(timeout_ms as u64)); + futures_util::pin_mut!(call); + match futures_util::future::select(call, timeout).await { + futures_util::future::Either::Left((reply, _)) => reply, + futures_util::future::Either::Right(((), _)) => { + return Err(JsValue::from_str("Request timed out")) } } - s._ping_closure = None; - } - - // Reject pending requests — correlation IDs are per-socket and - // won't match on a reconnected connection. - reject_all_pending(&ctx.pending_requests, "Connection lost"); - - if ctx.config.auto_reconnect { - let delay = { - let mut s = ctx.state.borrow_mut(); - s.status = ConnectionStatus::Reconnecting; - let d = ctx - .backoff - .get(s.backoff_index) - .copied() - .unwrap_or(*ctx.backoff.last().unwrap_or(&8_000)); - s.backoff_index = (s.backoff_index + 1).min(ctx.backoff.len() - 1); - d - }; - - emit_status(&ctx.on_status, ConnectionStatus::Reconnecting); - schedule_reconnect(ctx.clone(), delay); } else { - ctx.state.borrow_mut().status = ConnectionStatus::Disconnected; - emit_status(&ctx.on_status, ConnectionStatus::Disconnected); - } - }) as Box) - }; - ws.set_onclose(Some(on_close.as_ref().unchecked_ref())); + call.await + }; - // on_error — WebSocket always fires `close` after `error`, so just log. - let on_error = { - Closure::wrap(Box::new(move || { - web_sys::console::warn_1(&"WsBridge: WebSocket error".into()); - }) as Box) - }; - ws.set_onerror(Some(on_error.as_ref().unchecked_ref())); - - // Store closures to prevent GC - let mut state = ctx.state.borrow_mut(); - state._on_open = Some(on_open); - state._on_message = Some(on_message); - state._on_close = Some(on_close); - state._on_error = Some(on_error); + let payload = + reply.map_err(|e| JsValue::from_str(&format!("{method}: {}", rpc_err_str(&e))))?; + let value: serde_json::Value = serde_json::from_slice(&payload) + .map_err(|e| JsValue::from_str(&format!("{method}: malformed reply: {e}")))?; + let serializer = serde_wasm_bindgen::Serializer::json_compatible(); + serde::Serialize::serialize(&value, &serializer) + .map_err(|e| JsValue::from_str(&format!("{method}: {e}"))) + }) + } +} + +/// Human-readable form of the engine's 3-code error vocabulary. +fn rpc_err_str(e: &RpcError) -> &'static str { + match e { + RpcError::NotFound => "not_found", + RpcError::Denied => "denied", + _ => "internal (engine stopped, disconnected, or server error)", + } } -// ─── Reconnect ───────────────────────────────────────────────────────────── +// ─── Subscription pump ───────────────────────────────────────────────────── -/// Schedule a reconnect attempt after `delay_ms` using `setTimeout`. -/// -/// On success: swaps the new socket into `ctx.ws_cell` and re-installs -/// callbacks. On failure: increments backoff and schedules the next attempt. -/// -/// Uses `Closure::once` + `.forget()` — each attempt leaks a few bytes. -/// With 5 backoff steps capped at 8 s, worst case is ~5 closures in flight. -fn schedule_reconnect(ctx: Rc, delay_ms: u32) { - let closure = Closure::once(move || { - // Guard: user may have called disconnect() during the delay - if ctx.state.borrow().status == ConnectionStatus::Disconnected { +/// Subscribe to `pattern` and mirror every tagged update into the local +/// database; when the stream ends (disconnect or server rejection), +/// re-subscribe until the bridge is stopped. +async fn pump_pattern( + shared: Rc, + handle: ClientHandle, + db: AimDb, + schema_map: BTreeMap, + registry: SchemaRegistry, + pattern: String, +) { + loop { + if shared.stopped.get() { return; } - - match web_sys::WebSocket::new(&ctx.url) { - Ok(new_ws) => { - *ctx.ws_cell.borrow_mut() = new_ws; - install_ws_callbacks(&ctx); - } - Err(e) => { - web_sys::console::error_1(&format!("WsBridge: reconnect failed: {e:?}").into()); - // Try again with increased backoff - let next_delay = { - let mut s = ctx.state.borrow_mut(); - let d = ctx - .backoff - .get(s.backoff_index) - .copied() - .unwrap_or(*ctx.backoff.last().unwrap_or(&8_000)); - s.backoff_index = (s.backoff_index + 1).min(ctx.backoff.len() - 1); - d - }; - schedule_reconnect(ctx, next_delay); - } + let mut stream = match handle.subscribe(&pattern) { + Ok(stream) => stream, + Err(_) => return, // engine stopped + }; + while let Some(update) = stream.next().await { + // Wildcard events carry the concrete record topic; an exact-topic + // subscription may leave it implicit. + let topic = update.topic.as_deref().unwrap_or(&pattern); + route_update(&db, &schema_map, ®istry, topic, &update.data); } - }); - - if let Some(window) = web_sys::window() { - let _ = window.set_timeout_with_callback_and_timeout_and_arguments_0( - closure.as_ref().unchecked_ref(), - delay_ms as i32, - ); + // Stream ended — disconnected or rejected. Pace the re-subscribe so a + // rejecting server doesn't spin; the engine queues it while offline. + WasmAdapter + .sleep(core::time::Duration::from_millis(500)) + .await; } - closure.forget(); // prevent GC — fires once, then drops itself } -// ─── Server message handling ─────────────────────────────────────────────── - -/// Route a decoded server message to the local database. -/// -/// Uses direct `serde_json::from_value::()` → buffer push, bypassing the -/// `JsValue` intermediary that the old code path used. -fn handle_server_message(ctx: &SharedCtx, msg: ServerMessage) { - match msg { - ServerMessage::Data { topic, payload, .. } | ServerMessage::Snapshot { topic, payload } => { - if let Some(payload) = payload { - let schema = ctx.schema_map.get(&topic).map(|v| v.as_str()); - - match schema { - Some(schema) => { - if let Some(ops) = ctx.registry.get(schema) { - (ops.produce_from_json)(&ctx.db, &topic, payload.clone()); - } else { - web_sys::console::warn_1( - &format!( - "[WsBridge] unknown schema='{}' topic='{}'", - schema, topic - ) - .into(), - ); - } - } - None => { - web_sys::console::warn_1( - &format!( - "[WsBridge] No schema mapping for topic='{}' (schema_map has {} entries)", - topic, - ctx.schema_map.len() - ).into(), - ); - } - } - } - } - ServerMessage::Subscribed { .. } => { - // ACK — no action needed beyond status change (already handled in on_open). - } - ServerMessage::Error { message, topic, .. } => { - let detail = match topic { - Some(t) => format!("WsBridge error on topic '{t}': {message}"), - None => format!("WsBridge error: {message}"), - }; - web_sys::console::error_1(&detail.into()); - } - ServerMessage::Pong => { - // Keepalive ACK — reset timer if needed. - } - ServerMessage::QueryResult { id, records, total } => { - if let Some(req) = ctx.pending_requests.borrow_mut().remove(&id) { - let serializer = serde_wasm_bindgen::Serializer::json_compatible(); - let arr = js_sys::Array::new(); - for rec in &records { - if let Ok(js_val) = rec.serialize(&serializer) { - arr.push(&js_val); - } - } - let result_obj = js_sys::Object::new(); - let _ = js_sys::Reflect::set(&result_obj, &"records".into(), &arr); - let _ = js_sys::Reflect::set( - &result_obj, - &"total".into(), - &JsValue::from_f64(total as f64), - ); - let _ = req.resolve.call1(&JsValue::NULL, &result_obj); - } - } - ServerMessage::TopicList { id, topics } => { - if let Some(req) = ctx.pending_requests.borrow_mut().remove(&id) { - let serializer = serde_wasm_bindgen::Serializer::json_compatible(); - let arr = js_sys::Array::new(); - for topic in &topics { - if let Ok(js_val) = topic.serialize(&serializer) { - arr.push(&js_val); - } - } - let _ = req.resolve.call1(&JsValue::NULL, &arr); - } +/// Route one serialized record value into the local database. +fn route_update( + db: &AimDb, + schema_map: &BTreeMap, + registry: &SchemaRegistry, + topic: &str, + data: &[u8], +) { + let Some(schema) = schema_map.get(topic) else { + web_sys::console::warn_1( + &format!( + "[WsBridge] No schema mapping for topic='{}' (schema_map has {} entries)", + topic, + schema_map.len() + ) + .into(), + ); + return; + }; + let Some(ops) = registry.get(schema) else { + web_sys::console::warn_1( + &format!("[WsBridge] unknown schema='{schema}' topic='{topic}'").into(), + ); + return; + }; + match serde_json::from_slice::(data) { + Ok(payload) => (ops.produce_from_json)(db, topic, payload), + Err(e) => { + web_sys::console::warn_1( + &format!("[WsBridge] malformed payload for topic='{topic}': {e}").into(), + ); } } } @@ -747,7 +666,7 @@ fn handle_server_message(ctx: &SharedCtx, msg: ServerMessage) { /// This is the fast path for incoming server data — no `JsValue` hop. pub(crate) fn produce_from_json(db: &AimDb, key: &str, json: serde_json::Value) where - T: Send + Sync + 'static + Debug + Clone + DeserializeOwned, + T: Send + Sync + 'static + core::fmt::Debug + Clone + serde::de::DeserializeOwned, { match serde_json::from_value::(json) { Ok(val) => { @@ -779,16 +698,7 @@ where } } -// ─── Helpers ─────────────────────────────────────────────────────────────── - -/// Serialize a `ClientMessage` and send as text frame. -fn send_json(ws: &web_sys::WebSocket, msg: &ClientMessage) -> Result<(), JsError> { - let json = serde_json::to_string(msg) - .map_err(|e| JsError::new(&format!("JSON serialization failed: {e}")))?; - ws.send_with_str(&json) - .map_err(|e| JsError::new(&format!("WebSocket send failed: {e:?}")))?; - Ok(()) -} +// ─── Status emission ─────────────────────────────────────────────────────── /// Emit status change to the registered JS callback **and** via DOM /// `CustomEvent` on `window` (secondary channel for non-React consumers). @@ -800,7 +710,7 @@ fn send_json(ws: &web_sys::WebSocket, msg: &ClientMessage) -> Result<(), JsError /// returns `Ok` but the JS function body never runs. Yielding once via /// `Promise.resolve().await` puts us in a clean microtask context (same /// as `subscribe_typed`), where React state updates flush correctly. -fn emit_status(on_status: &Rc>>, status: ConnectionStatus) { +fn emit_status(on_status: &RefCell>, status: ConnectionStatus) { // Primary: deferred callback via microtask let cb = on_status.borrow().as_ref().cloned(); if let Some(cb) = cb { From a8a40e276efdc47571e7ee2b191dd7f001396292 Mon Sep 17 00:00:00 2001 From: test Date: Fri, 17 Jul 2026 14:05:30 +0000 Subject: [PATCH 05/45] chore(workspace): delete aimdb-ws-protocol and retire the ws wire residue (design 045) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With no consumers left, the fork is gone: - aimdb-ws-protocol crate deleted (353 lines) and de-registered from the workspace members, Makefile build/test/clippy/doc targets, and the publish sequence (17 → 16 crates). - aimdb-client: dead NDJSON helpers retired with the hand-rolled client in PR #124 are deleted (RequestExt/ResponseExt/serialize_message/ parse_message/EventMessage/cli_hello — grep-confirmed zero consumers); the protocol re-export keeps RecordMetadata/WelcomeMessage/Request/ Response/Event. - design 038 §2.5/§3.9 annotated as resolved by 045; root and per-crate CHANGELOGs describe the breaking release. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 17 +- Cargo.lock | 10 - Cargo.toml | 1 - Makefile | 47 +-- aimdb-client/src/lib.rs | 3 +- aimdb-client/src/protocol.rs | 108 +----- aimdb-ws-protocol/CHANGELOG.md | 22 -- aimdb-ws-protocol/Cargo.toml | 11 - aimdb-ws-protocol/src/lib.rs | 353 ------------------ ...echnical-debt-and-simplification-review.md | 8 +- 10 files changed, 46 insertions(+), 534 deletions(-) delete mode 100644 aimdb-ws-protocol/CHANGELOG.md delete mode 100644 aimdb-ws-protocol/Cargo.toml delete mode 100644 aimdb-ws-protocol/src/lib.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 66132e53..bdd8807e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > - [aimdb-uds-connector/CHANGELOG.md](aimdb-uds-connector/CHANGELOG.md) > - [aimdb-serial-connector/CHANGELOG.md](aimdb-serial-connector/CHANGELOG.md) > - [aimdb-tcp-connector/CHANGELOG.md](aimdb-tcp-connector/CHANGELOG.md) -> - [aimdb-ws-protocol/CHANGELOG.md](aimdb-ws-protocol/CHANGELOG.md) > - [aimdb-wasm-adapter/CHANGELOG.md](aimdb-wasm-adapter/CHANGELOG.md) > - [aimdb-sync/CHANGELOG.md](aimdb-sync/CHANGELOG.md) > - [aimdb-client/CHANGELOG.md](aimdb-client/CHANGELOG.md) @@ -30,6 +29,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed — Design 045: one wire protocol (AimX) for every transport (breaking) + +Implementation of [design 045](docs/design/045-retire-ws-protocol-converge-on-aimx.md) +(the 038 §3.9 / 036 A2 protocol unification). The `aimdb-ws-protocol` crate is +**deleted**; the WebSocket connector and the browser `WsBridge` now speak the +same AimX tagged frames as UDS/serial/TCP, over the same session engines. +AimX gained the two features that justified the ws fork: wildcard / +multi-record subscribe (one subscription fans in every matching record, events +tagged with the record that fired, one late-join snapshot per match) and the +shared `record.query`/`record.list` result shapes (`{records, total}` with +`QueryRecord{topic, payload, ts}` rows; `{name, schema_type, entity}` topic +rows). **Breaking for browser clients** (the JS `WsBridge` API is unchanged, +but the wire is new); breaking Rust API changes are listed per crate +(`aimdb-core`, `aimdb-websocket-connector`, `aimdb-wasm-adapter`, +`aimdb-client`, `aimdb-persistence`). + ### Changed — Design 038 simplification pass (breaking) Implementation of the accepted items of [design 038](docs/design/038-technical-debt-and-simplification-review.md) diff --git a/Cargo.lock b/Cargo.lock index 61b2e854..820654b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -386,7 +386,6 @@ version = "0.3.0" dependencies = [ "aimdb-core", "aimdb-data-contracts", - "aimdb-ws-protocol", "futures", "futures-channel", "futures-util", @@ -407,7 +406,6 @@ dependencies = [ "aimdb-core", "aimdb-data-contracts", "aimdb-tokio-adapter", - "aimdb-ws-protocol", "axum", "dashmap", "futures-util", @@ -419,14 +417,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "aimdb-ws-protocol" -version = "0.1.0" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "aligned" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index cc6cf7bf..c91ecbbe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,6 @@ members = [ "aimdb-uds-connector", "aimdb-serial-connector", "aimdb-tcp-connector", - "aimdb-ws-protocol", "aimdb-wasm-adapter", "tools/aimdb-cli", "tools/aimdb-mcp", diff --git a/Makefile b/Makefile index 859f1d75..97f6aaeb 100644 --- a/Makefile +++ b/Makefile @@ -96,8 +96,6 @@ build: cargo build --package aimdb-persistence-sqlite @printf "$(YELLOW) → Building KNX connector$(NC)\n" cargo build --package aimdb-knx-connector --features "std,tokio-runtime" - @printf "$(YELLOW) → Building WS protocol$(NC)\n" - cargo build --package aimdb-ws-protocol @printf "$(YELLOW) → Building WebSocket connector (server + client)$(NC)\n" cargo build --package aimdb-websocket-connector --features "server,client" @printf "$(YELLOW) → Building UDS connector$(NC)\n" @@ -165,8 +163,6 @@ test: cargo test --package aimdb-mqtt-connector --features "std,tokio-runtime" @printf "$(YELLOW) → Testing KNX connector$(NC)\n" cargo test --package aimdb-knx-connector --features "std,tokio-runtime" - @printf "$(YELLOW) → Testing WS protocol$(NC)\n" - cargo test --package aimdb-ws-protocol @printf "$(YELLOW) → Testing WebSocket connector (server + client: unit, real-socket e2e, AimDB round-trip)$(NC)\n" cargo test --package aimdb-websocket-connector --features "server,client" @printf "$(YELLOW) → Testing WebSocket connector client-only build$(NC)\n" @@ -184,7 +180,7 @@ test: fmt: @printf "$(GREEN)Formatting code (workspace members only)...$(NC)\n" - @for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-ws-protocol aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring-async aimdb-bench; do \ + @for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring-async aimdb-bench; do \ printf "$(YELLOW) → Formatting $$pkg$(NC)\n"; \ cargo fmt -p $$pkg 2>/dev/null || true; \ done @@ -193,7 +189,7 @@ fmt: fmt-check: @printf "$(GREEN)Checking code formatting (workspace members only)...$(NC)\n" @FAILED=0; \ - for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-ws-protocol aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring-async aimdb-bench; do \ + for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring-async aimdb-bench; do \ printf "$(YELLOW) → Checking $$pkg$(NC)\n"; \ if ! cargo fmt -p $$pkg -- --check 2>&1; then \ printf "$(RED)❌ Formatting check failed for $$pkg$(NC)\n"; \ @@ -264,8 +260,6 @@ clippy: cargo clippy --package aimdb-mqtt-connector --target thumbv7em-none-eabihf --no-default-features --features "embassy-runtime,embassy-tls,defmt" -- -D warnings @printf "$(YELLOW) → Clippy on KNX connector (embassy + defmt)$(NC)\n" cargo clippy --package aimdb-knx-connector --target thumbv7em-none-eabihf --no-default-features --features "embassy-runtime,defmt" -- -D warnings - @printf "$(YELLOW) → Clippy on WS protocol$(NC)\n" - cargo clippy --package aimdb-ws-protocol --all-targets -- -D warnings @printf "$(YELLOW) → Clippy on WebSocket connector$(NC)\n" cargo clippy --package aimdb-websocket-connector --features "tokio-runtime,client" --all-targets -- -D warnings @printf "$(YELLOW) → Clippy on UDS connector$(NC)\n" @@ -306,7 +300,6 @@ doc: cargo doc --package aimdb-mcp --no-deps cargo doc --package aimdb-persistence --no-deps cargo doc --package aimdb-persistence-sqlite --no-deps - cargo doc --package aimdb-ws-protocol --no-deps cargo doc --package aimdb-websocket-connector --features "tokio-runtime" --no-deps @cp -r target/doc/* target/doc-final/cloud/ @printf "$(YELLOW) → Building embedded documentation$(NC)\n" @@ -485,71 +478,67 @@ publish: else \ printf "$(BLUE)Running in CI mode - skipping confirmation$(NC)\n"; \ fi - @printf "$(YELLOW) → Publishing aimdb-derive (1/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-derive (1/16)$(NC)\n" @cargo publish -p aimdb-derive @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-codegen (2/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-codegen (2/16)$(NC)\n" @cargo publish -p aimdb-codegen @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-core (3/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-core (3/16)$(NC)\n" @cargo publish -p aimdb-core @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-data-contracts (4/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-data-contracts (4/16)$(NC)\n" @cargo publish -p aimdb-data-contracts @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-tokio-adapter (5/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-tokio-adapter (5/16)$(NC)\n" @cargo publish -p aimdb-tokio-adapter @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-embassy-adapter (6/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-embassy-adapter (6/16)$(NC)\n" @cargo publish -p aimdb-embassy-adapter --no-verify @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-client (7/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-client (7/16)$(NC)\n" @cargo publish -p aimdb-client @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-sync (8/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-sync (8/16)$(NC)\n" @cargo publish -p aimdb-sync @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-persistence (9/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-persistence (9/16)$(NC)\n" @cargo publish -p aimdb-persistence @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-persistence-sqlite (10/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-persistence-sqlite (10/16)$(NC)\n" @cargo publish -p aimdb-persistence-sqlite @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-mqtt-connector (11/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-mqtt-connector (11/16)$(NC)\n" @cargo publish -p aimdb-mqtt-connector @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-knx-connector (12/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-knx-connector (12/16)$(NC)\n" @cargo publish -p aimdb-knx-connector @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-ws-protocol (13/17)$(NC)\n" - @cargo publish -p aimdb-ws-protocol - @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" - @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-websocket-connector (14/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-websocket-connector (13/16)$(NC)\n" @cargo publish -p aimdb-websocket-connector @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-wasm-adapter (15/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-wasm-adapter (14/16)$(NC)\n" @cargo publish -p aimdb-wasm-adapter --no-verify @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-cli (16/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-cli (15/16)$(NC)\n" @cargo publish -p aimdb-cli @printf "$(YELLOW) → Waiting 10s for crates.io propagation...$(NC)\n" @sleep 10 - @printf "$(YELLOW) → Publishing aimdb-mcp (17/17)$(NC)\n" + @printf "$(YELLOW) → Publishing aimdb-mcp (16/16)$(NC)\n" @cargo publish -p aimdb-mcp @printf "$(GREEN)✓ All 17 crates published successfully!$(NC)\n" @printf "$(BLUE)🎉 AimDB v$(shell grep '^version' Cargo.toml | head -1 | cut -d '"' -f 2) is now live on crates.io!$(NC)\n" diff --git a/aimdb-client/src/lib.rs b/aimdb-client/src/lib.rs index b57d65d3..d7e0bca3 100644 --- a/aimdb-client/src/lib.rs +++ b/aimdb-client/src/lib.rs @@ -52,6 +52,5 @@ pub use endpoint::{dial, parse_endpoint, ParsedEndpoint, Scheme}; pub use engine::{AimxConnection, DrainResponse}; pub use error::{ClientError, ClientResult}; pub use protocol::{ - cli_hello, parse_message, serialize_message, Event, EventMessage, RecordMetadata, Request, - RequestExt, Response, ResponseExt, WelcomeMessage, CLIENT_NAME, PROTOCOL_VERSION, + Event, RecordMetadata, Request, Response, WelcomeMessage, CLIENT_NAME, PROTOCOL_VERSION, }; diff --git a/aimdb-client/src/protocol.rs b/aimdb-client/src/protocol.rs index ec6f6530..69a56051 100644 --- a/aimdb-client/src/protocol.rs +++ b/aimdb-client/src/protocol.rs @@ -1,9 +1,9 @@ -//! AimX Protocol Types and Utilities +//! AimX Protocol Types //! -//! This module re-exports protocol types from `aimdb-core` and adds CLI-specific -//! utilities for NDJSON serialization and convenience methods. - -use serde::{Deserialize, Serialize}; +//! Re-exports the protocol types from `aimdb-core` that the client surface +//! uses. The NDJSON helper functions and legacy `Request`/`Response` +//! extension traits were retired with the hand-rolled client — the engine +//! ([`crate::engine::AimxConnection`]) owns framing and correlation. // Re-export protocol types from aimdb-core pub use aimdb_core::remote::{ @@ -13,101 +13,3 @@ pub use aimdb_core::remote::{ /// Client identifier pub const CLIENT_NAME: &str = "aimdb-cli"; - -/// CLI-specific convenience methods for Request -pub trait RequestExt { - /// Create a new request - #[allow(clippy::new_ret_no_self)] - fn new(id: u64, method: impl Into) -> Request; - - /// Create a request with parameters - fn with_params(id: u64, method: impl Into, params: serde_json::Value) -> Request; -} - -impl RequestExt for Request { - fn new(id: u64, method: impl Into) -> Request { - Request { - id, - method: method.into(), - params: None, - } - } - - fn with_params(id: u64, method: impl Into, params: serde_json::Value) -> Request { - Request { - id, - method: method.into(), - params: Some(params), - } - } -} - -/// CLI-specific convenience methods for Response -pub trait ResponseExt { - /// Extract result or return error - fn into_result(self) -> Result; -} - -impl ResponseExt for Response { - fn into_result(self) -> Result { - match self { - Response::Success { result, .. } => Ok(result), - Response::Error { error, .. } => Err(error), - } - } -} - -/// Create default CLI hello message -pub fn cli_hello() -> HelloMessage { - HelloMessage { - version: PROTOCOL_VERSION.to_string(), - client: CLIENT_NAME.to_string(), - capabilities: None, - auth_token: None, - } -} - -/// Event message wrapper from subscription (for NDJSON parsing) -#[derive(Debug, Clone, Deserialize)] -pub struct EventMessage { - pub event: Event, -} - -/// Helper to serialize a message as NDJSON line -pub fn serialize_message(msg: &T) -> Result { - Ok(format!("{}\n", serde_json::to_string(msg)?)) -} - -/// Helper to parse NDJSON line into message -pub fn parse_message Deserialize<'de>>(line: &str) -> Result { - serde_json::from_str(line.trim()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_request_serialization() { - let req = Request::new(1, "record.list"); - let json = serialize_message(&req).unwrap(); - assert!(json.contains("\"method\":\"record.list\"")); - assert!(json.ends_with('\n')); - } - - #[test] - fn test_response_parsing() { - let json = r#"{"id":1,"result":{"status":"ok"}}"#; - let resp: Response = parse_message(json).unwrap(); - let result = resp.into_result().unwrap(); - assert_eq!(result["status"], "ok"); - } - - #[test] - fn test_cli_hello() { - let hello = cli_hello(); - assert_eq!(hello.version, PROTOCOL_VERSION); - assert_eq!(hello.client, CLIENT_NAME); - assert!(hello.auth_token.is_none()); - } -} diff --git a/aimdb-ws-protocol/CHANGELOG.md b/aimdb-ws-protocol/CHANGELOG.md deleted file mode 100644 index ab0b4d0c..00000000 --- a/aimdb-ws-protocol/CHANGELOG.md +++ /dev/null @@ -1,22 +0,0 @@ -# Changelog - -All notable changes to `aimdb-ws-protocol` will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -No changes yet. - -## [0.1.0] - 2026-03-11 - -### Added - -- Initial release of the shared WebSocket wire protocol -- `ServerMessage` enum: `Data`, `Snapshot`, `Subscribed`, `Error`, `Pong`, `QueryResult` -- `ClientMessage` enum: `Subscribe`, `Unsubscribe`, `Write`, `Ping`, `Query` -- JSON-encoded wire format with `"type"` discriminant tag -- MQTT-style wildcard topic matching (`#` multi-level, `*` single-level) -- Timestamp-tagged data messages for ordering -- Late-join snapshot support diff --git a/aimdb-ws-protocol/Cargo.toml b/aimdb-ws-protocol/Cargo.toml deleted file mode 100644 index 15881210..00000000 --- a/aimdb-ws-protocol/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "aimdb-ws-protocol" -version = "0.1.0" -edition = "2021" -license.workspace = true -description = "Shared wire protocol types for AimDB WebSocket connector and clients" -keywords = ["websocket", "protocol", "aimdb"] - -[dependencies] -serde = { workspace = true } -serde_json = { workspace = true } diff --git a/aimdb-ws-protocol/src/lib.rs b/aimdb-ws-protocol/src/lib.rs deleted file mode 100644 index 02bfac40..00000000 --- a/aimdb-ws-protocol/src/lib.rs +++ /dev/null @@ -1,353 +0,0 @@ -//! # aimdb-ws-protocol -//! -//! Shared wire protocol types for the AimDB WebSocket connector ecosystem. -//! -//! Used by: -//! -//! - **`aimdb-websocket-connector`** — the server side (Axum/Tokio) -//! - **`aimdb-wasm-adapter`** — the browser client (`WsBridge`) -//! -//! # Wire Protocol -//! -//! All messages are JSON-encoded with a `"type"` discriminant tag: -//! -//! ## Server → Client ([`ServerMessage`]) -//! -//! - `data` — live record push with timestamp -//! - `snapshot` — late-join current value -//! - `subscribed` — subscription acknowledgement -//! - `error` — per-operation error -//! - `pong` — response to client ping -//! - `query_result` — response to a client query request -//! -//! ## Client → Server ([`ClientMessage`]) -//! -//! - `subscribe` — subscribe to one or more topics (supports MQTT wildcards) -//! - `unsubscribe` — cancel subscriptions -//! - `write` — inbound value for a `link_from("ws://…")` record -//! - `ping` — keepalive ping -//! - `query` — query historical / persisted records -//! -//! # Topic Matching -//! -//! [`topic_matches`] implements MQTT-style wildcard matching (`#` for -//! multi-level, `*` for single-level). - -use serde::{Deserialize, Serialize}; - -// ════════════════════════════════════════════════════════════════════ -// Server → Client -// ════════════════════════════════════════════════════════════════════ - -/// A message sent from the server to a connected WebSocket client. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ServerMessage { - /// Live data push from an outbound route. - Data { - topic: String, - #[serde(skip_serializing_if = "Option::is_none")] - payload: Option, - /// Server-side dispatch timestamp (milliseconds since Unix epoch). - ts: u64, - }, - - /// Late-join snapshot — current value sent when a client subscribes. - Snapshot { - topic: String, - #[serde(skip_serializing_if = "Option::is_none")] - payload: Option, - }, - - /// Confirmation sent once subscriptions are recorded. - Subscribed { topics: Vec }, - - /// Per-operation error. - Error { - code: ErrorCode, - #[serde(skip_serializing_if = "Option::is_none")] - topic: Option, - message: String, - }, - - /// Response to a client `ping` message. - Pong, - - /// Response to a client `query` request. - /// - /// Contains the matching historical records and metadata. - QueryResult { - /// Correlation ID echoed from the client request. - id: String, - /// Matching records, ordered by timestamp ascending. - records: Vec, - /// Total number of records matched (before any limit). - total: usize, - }, - - /// Response to a client `list_topics` request. - TopicList { - /// Correlation ID echoed from the client request. - id: String, - /// All outbound topics served by this endpoint. - topics: Vec, - }, -} - -/// A single record returned in a [`ServerMessage::QueryResult`]. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct QueryRecord { - /// Topic / record name (e.g. `"temp.vienna"`). - pub topic: String, - /// Deserialized record value. - pub payload: serde_json::Value, - /// Storage timestamp (milliseconds since Unix epoch). - pub ts: u64, -} - -/// Metadata for a single outbound topic served by a WebSocket endpoint. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct TopicInfo { - /// Record key / topic name (e.g. `"temp.vienna"`). - pub name: String, - /// Schema type name (e.g. `"temperature"`), if known by the server. - #[serde(skip_serializing_if = "Option::is_none")] - pub schema_type: Option, - /// Entity / node identifier (e.g. `"vienna"`), extracted server-side from the - /// topic name. The server is the authority on naming conventions — clients - /// should use this field directly rather than parsing the topic name. - #[serde(skip_serializing_if = "Option::is_none")] - pub entity: Option, -} - -/// Machine-readable error codes sent in `ServerMessage::Error`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub enum ErrorCode { - Unauthorized, - Forbidden, - UnknownTopic, - SerializationError, - WriteError, - ServerError, -} - -// ════════════════════════════════════════════════════════════════════ -// Client → Server -// ════════════════════════════════════════════════════════════════════ - -/// A message received from a WebSocket client. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ClientMessage { - /// Subscribe to one or more topics (wildcards supported). - Subscribe { topics: Vec }, - - /// Unsubscribe from one or more topics. - Unsubscribe { topics: Vec }, - - /// Write a value to an inbound record (`link_from("ws://…")`). - Write { - topic: String, - payload: serde_json::Value, - }, - - /// Keepalive ping. - Ping, - - /// Query historical / persisted records. - /// - /// The server responds with [`ServerMessage::QueryResult`] carrying the - /// same `id` for correlation. - Query { - /// Client-generated correlation ID (echoed in the response). - id: String, - /// Topic pattern to match (MQTT wildcards supported, `"*"` for all). - pattern: String, - /// Start of time range (milliseconds since Unix epoch), inclusive. Defaults to 1 h ago. - #[serde(skip_serializing_if = "Option::is_none")] - from: Option, - /// End of time range (milliseconds since Unix epoch), inclusive. Defaults to now. - #[serde(skip_serializing_if = "Option::is_none")] - to: Option, - /// Maximum number of records to return per matching topic. - #[serde(skip_serializing_if = "Option::is_none")] - limit: Option, - }, - - /// Request the list of topics served by this WebSocket endpoint. - /// - /// The server responds with [`ServerMessage::TopicList`] carrying the - /// same `id` for correlation. - ListTopics { - /// Client-generated correlation ID (echoed in the response). - id: String, - }, -} - -// ════════════════════════════════════════════════════════════════════ -// Topic matching -// ════════════════════════════════════════════════════════════════════ - -/// Returns `true` if `topic` matches `pattern`. -/// -/// Follows MQTT wildcard conventions: -/// -/// | Pattern | Semantics | -/// |----------|-----------------------------------| -/// | `#` | Multi-level wildcard (all topics) | -/// | `a/#` | Everything under `a/` | -/// | `a/*/c` | Single-level wildcard in segment | -/// | `a/b/c` | Exact match | -pub fn topic_matches(pattern: &str, topic: &str) -> bool { - // Fast path: exact match - if pattern == topic { - return true; - } - - // Multi-level wildcard: `#` matches everything - if pattern == "#" { - return true; - } - - // `prefix/#` matches everything under prefix — only when prefix is literal - // (no wildcards in the prefix). When wildcards are present, fall through to - // the segment loop which handles `#` at any position. - if let Some(prefix) = pattern.strip_suffix("/#") { - if !prefix.contains('*') && !prefix.contains('#') { - return topic.starts_with(prefix) - && (topic.len() == prefix.len() - || topic.as_bytes().get(prefix.len()) == Some(&b'/')); - } - } - - // Segment-by-segment matching with `*` single-level wildcard - let mut pattern_parts = pattern.split('/'); - let mut topic_parts = topic.split('/'); - - loop { - match (pattern_parts.next(), topic_parts.next()) { - (Some("#"), _) => return true, - (Some("*"), Some(_)) => {} // single-level wildcard — consume one segment - (Some(p), Some(t)) if p == t => {} // literal match - (None, None) => return true, // both exhausted at the same time - _ => return false, - } - } -} - -/// Returns the current milliseconds since the Unix epoch (for `ts` fields). -pub fn now_ms() -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64 -} - -// ════════════════════════════════════════════════════════════════════ -// Tests -// ════════════════════════════════════════════════════════════════════ - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn exact_match() { - assert!(topic_matches("a/b/c", "a/b/c")); - assert!(!topic_matches("a/b/c", "a/b/d")); - } - - #[test] - fn hash_wildcard() { - assert!(topic_matches("#", "anything/goes/here")); - assert!(topic_matches("#", "a")); - } - - #[test] - fn prefix_hash_wildcard() { - assert!(topic_matches("sensors/#", "sensors/temperature/vienna")); - assert!(topic_matches("sensors/#", "sensors/humidity/berlin")); - assert!(!topic_matches("sensors/#", "commands/setpoint")); - // Edge: prefix itself - assert!(topic_matches("sensors/#", "sensors")); - } - - #[test] - fn star_wildcard() { - assert!(topic_matches( - "sensors/temperature/*", - "sensors/temperature/vienna" - )); - assert!(topic_matches( - "sensors/temperature/*", - "sensors/temperature/berlin" - )); - assert!(!topic_matches( - "sensors/temperature/*", - "sensors/humidity/vienna" - )); - assert!(!topic_matches( - "sensors/temperature/*", - "sensors/temperature/a/b" - )); - } - - #[test] - fn mixed_wildcards() { - assert!(topic_matches("a/*/c/#", "a/b/c/d/e/f")); - assert!(!topic_matches("a/*/c/#", "a/b/x/d")); - } - - #[test] - fn serde_server_message_roundtrip() { - let msg = ServerMessage::Data { - topic: "sensors/temp".into(), - payload: Some(serde_json::json!({"celsius": 21.5})), - ts: 1234567890, - }; - let json = serde_json::to_string(&msg).unwrap(); - let parsed: ServerMessage = serde_json::from_str(&json).unwrap(); - match parsed { - ServerMessage::Data { topic, ts, .. } => { - assert_eq!(topic, "sensors/temp"); - assert_eq!(ts, 1234567890); - } - _ => panic!("Expected Data variant"), - } - } - - #[test] - fn serde_client_message_roundtrip() { - let msg = ClientMessage::Subscribe { - topics: vec!["sensors/#".into()], - }; - let json = serde_json::to_string(&msg).unwrap(); - let parsed: ClientMessage = serde_json::from_str(&json).unwrap(); - match parsed { - ClientMessage::Subscribe { topics } => { - assert_eq!(topics, vec!["sensors/#".to_string()]); - } - _ => panic!("Expected Subscribe variant"), - } - } - - #[test] - fn serde_error_code_roundtrip() { - let msg = ServerMessage::Error { - code: ErrorCode::UnknownTopic, - topic: Some("foo/bar".into()), - message: "not found".into(), - }; - let json = serde_json::to_string(&msg).unwrap(); - assert!(json.contains("UNKNOWN_TOPIC")); - let parsed: ServerMessage = serde_json::from_str(&json).unwrap(); - match parsed { - ServerMessage::Error { code, .. } => { - assert!(matches!(code, ErrorCode::UnknownTopic)); - } - _ => panic!("Expected Error variant"), - } - } -} diff --git a/docs/design/038-technical-debt-and-simplification-review.md b/docs/design/038-technical-debt-and-simplification-review.md index 0b4e5715..a5c40933 100644 --- a/docs/design/038-technical-debt-and-simplification-review.md +++ b/docs/design/038-technical-debt-and-simplification-review.md @@ -89,6 +89,8 @@ The knock-on structure this forces: ### 2.5 The remote-access stack: one excellent engine, two protocols, three clients +> **Status update:** resolved by [design 045](./045-retire-ws-protocol-converge-on-aimx.md) — the table below is the *pre-045* picture; every transport (including the browser) now speaks AimX over the shared engines. + The session substrate (`session/mod.rs` + server/client engines + pumps, ~2,300 lines) is genuinely good work: dyn-safe layering (Connection/Listener/Dialer → EnvelopeCodec → Dispatch/Session), runtime-neutral (`futures` channels + `RuntimeOps` clock), and it made `aimdb-uds-connector` a 391-line crate. That is the target picture. But the workspace ended up with: @@ -201,9 +203,11 @@ Validate "at most one of {source, transform, link_from}" in exactly one place: ` ### 3.9 One remote-access protocol — **~1,200–1,800 lines, breaking for browser clients, real work** +> **Status update: implemented via [design 045](./045-retire-ws-protocol-converge-on-aimx.md)** (the design-doc gate 036 A2 asked for). `aimdb-ws-protocol`, the WS codec, and the hand-rolled `WsBridge` demux are gone; AimX gained wildcard subscribe (optional `Event.topic`, `Snapshot.sub`, the `subscribed` ack frame) and the shared `record.query`/`record.list` result shapes. Two scope corrections vs. the paragraph below: the `aimdb-client` demux fold had **already** happened (PR #124 — only dead re-exports remained), and query passthrough already existed (`record.query` + `QueryHandlerFn`); what was missing was the result shape, not the passthrough. + Port the WebSocket connector to speak AimX over its existing WS transport (it already runs on the session engine; the delta is the envelope), extend AimX with the two features that justified the fork (wildcard subscribe, query passthrough — both fit the `method`/`topic` model), and rewrite `WsBridge` as an AimX client. Then delete `aimdb-ws-protocol` (353), the WS codec (507), and the protocol-specific halves of `server/` and `WsBridge`. End state: **one protocol, one snapshot/ack/auth semantics, every transport including the browser**. This also collapses the documentation story ("AimX is how you talk to AimDB — over UDS, serial, TCP, or WebSocket"). -Follow-up in the same theme: fold `aimdb-client`'s hand-rolled demux onto `session::client::run_client` so reply correlation exists once. +Follow-up in the same theme: fold `aimdb-client`'s hand-rolled demux onto `session::client::run_client` so reply correlation exists once. *(Already done before 045 — see the status note above.)* ### 3.10 Extract the architecture agent — **~4,000 lines out of this repo, organizational** @@ -248,7 +252,7 @@ Respecting the 034 Phase-4 decision to stay monolithic: at minimum move the 5-cr | §3.3 DbError diet | ~450 | match sites | low | **implemented** | | §3.4–§3.6 registry/traits/validation | ~520 | no | low | **implemented** | | §3.7–§3.8 API pruning | ~450 | mechanical | low | **implemented** | -| §3.9 one protocol | ~1,500 | browser clients | medium–high | open (no decision yet; needs design doc) | +| §3.9 one protocol | ~1,500 | browser clients | medium–high | **implemented (design 045)** | | §3.10 extract agent | ~4,000 (relocated) | no | organizational | **deferred** — CI drift check implemented instead | | §3.11 features | (gates, not lines) | feature selectors | low | **implemented** as amended | | §3.12 examples/workspace | (build time) | no | low | **discarded for now** | From 377869c5aac396778ebc67c85c574c4ba41d90f4 Mon Sep 17 00:00:00 2001 From: test Date: Fri, 17 Jul 2026 17:27:30 +0000 Subject: [PATCH 06/45] chore(external): update embassy subproject commit reference --- Cargo.lock | 2 +- _external/embassy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 820654b6..b14c23e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3977,7 +3977,7 @@ dependencies = [ [[package]] name = "stm32-metapac" version = "21.0.0" -source = "git+https://github.com/embassy-rs/stm32-data-generated?tag=stm32-data-43d482dc249c6755fe990b67e86be4fdc9092909#1d8486efb2e9226b08ddfcd424841a11fb0620a0" +source = "git+https://github.com/embassy-rs/stm32-data-generated?tag=stm32-data-1912ac7d73c27f03863fb2d71fbfddf0d9fce062#27bdd2e4982cbc04ba4f4ad9ee7c8824a797944b" dependencies = [ "cortex-m", "cortex-m-rt", diff --git a/_external/embassy b/_external/embassy index e5921a9d..158325bd 160000 --- a/_external/embassy +++ b/_external/embassy @@ -1 +1 @@ -Subproject commit e5921a9d3a4a138f724ea40662599ff0fc42e6c7 +Subproject commit 158325bd4cfc13ace5b09fb572104b292f8442e6 From c49e9ef0d3f83d297939b5648d269b685a6f9aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 18 Jul 2026 21:09:34 +0000 Subject: [PATCH 07/45] feat: retire aimdb-ws-protocol in favor of AimX, update related components and documentation --- aimdb-client/src/engine.rs | 6 +- aimdb-client/src/lib.rs | 4 +- aimdb-client/src/protocol.rs | 5 +- aimdb-core/src/lib.rs | 5 ++ aimdb-core/src/remote/metadata.rs | 13 ++-- aimdb-core/src/remote/mod.rs | 27 ++++++++ aimdb-core/src/remote/query.rs | 2 +- aimdb-core/src/session/client.rs | 60 +++++++++++------ aimdb-core/src/session/mod.rs | 2 +- aimdb-core/src/session/topic_match.rs | 3 +- aimdb-core/tests/session_engine.rs | 43 ++++++++---- aimdb-wasm-adapter/src/bindings.rs | 55 ++++++++++++---- aimdb-wasm-adapter/src/time.rs | 34 +++++----- aimdb-wasm-adapter/src/ws_bridge.rs | 65 ++++++------------- .../src/server/builder.rs | 11 ++-- aimdb-websocket-connector/tests/e2e.rs | 2 +- ...045-retire-ws-protocol-converge-on-aimx.md | 4 +- 17 files changed, 209 insertions(+), 132 deletions(-) diff --git a/aimdb-client/src/engine.rs b/aimdb-client/src/engine.rs index 95a9522b..fad86208 100644 --- a/aimdb-client/src/engine.rs +++ b/aimdb-client/src/engine.rs @@ -195,9 +195,10 @@ impl AimxConnection { pub fn subscribe(&self, name: &str) -> ClientResult> { let raw = self.handle.subscribe(name).map_err(rpc_err)?; // Decode each update's payload into a JSON value; drop any that fail to - // parse. For the per-record topic (wildcard subscriptions), see + // parse. A terminal rejection item ends the stream. For the per-record + // topic (wildcard subscriptions), see // [`subscribe_with_topics`](Self::subscribe_with_topics). - let decoded = raw.filter_map(|u| async move { serde_json::from_slice(&u.data).ok() }); + let decoded = raw.filter_map(|u| async move { serde_json::from_slice(&u.ok()?.data).ok() }); Ok(Box::pin(decoded)) } @@ -211,6 +212,7 @@ impl AimxConnection { ) -> ClientResult, serde_json::Value)>> { let raw = self.handle.subscribe(pattern).map_err(rpc_err)?; let decoded = raw.filter_map(|u| async move { + let u = u.ok()?; let value = serde_json::from_slice(&u.data).ok()?; Some((u.topic.as_deref().map(String::from), value)) }); diff --git a/aimdb-client/src/lib.rs b/aimdb-client/src/lib.rs index d7e0bca3..869c0af1 100644 --- a/aimdb-client/src/lib.rs +++ b/aimdb-client/src/lib.rs @@ -51,6 +51,4 @@ pub use discovery::{discover_instances, find_instance, InstanceInfo}; pub use endpoint::{dial, parse_endpoint, ParsedEndpoint, Scheme}; pub use engine::{AimxConnection, DrainResponse}; pub use error::{ClientError, ClientResult}; -pub use protocol::{ - Event, RecordMetadata, Request, Response, WelcomeMessage, CLIENT_NAME, PROTOCOL_VERSION, -}; +pub use protocol::{RecordMetadata, WelcomeMessage, CLIENT_NAME, PROTOCOL_VERSION}; diff --git a/aimdb-client/src/protocol.rs b/aimdb-client/src/protocol.rs index 69a56051..5b013206 100644 --- a/aimdb-client/src/protocol.rs +++ b/aimdb-client/src/protocol.rs @@ -1,9 +1,8 @@ //! AimX Protocol Types //! //! Re-exports the protocol types from `aimdb-core` that the client surface -//! uses. The NDJSON helper functions and legacy `Request`/`Response` -//! extension traits were retired with the hand-rolled client — the engine -//! ([`crate::engine::AimxConnection`]) owns framing and correlation. +//! uses. Framing and correlation live in the engine +//! ([`crate::engine::AimxConnection`]), not here. // Re-export protocol types from aimdb-core pub use aimdb_core::remote::{ diff --git a/aimdb-core/src/lib.rs b/aimdb-core/src/lib.rs index 84cd218e..4f6c11cb 100644 --- a/aimdb-core/src/lib.rs +++ b/aimdb-core/src/lib.rs @@ -75,6 +75,11 @@ pub use typed_record::{AnyRecord, AnyRecordExt, TypedRecord}; #[cfg(feature = "remote")] pub use codec::{JsonCodec, RemoteSerialize, SerdeJsonCodec}; +// Record-key leaf/entity derivation, shared by record metadata and the +// connectors that surface the `entity` field. +#[cfg(feature = "remote")] +pub use remote::topic_leaf; + // connector-session contracts (feature `connector-session`, no_std + alloc // compatible). See docs/design/remote-access-via-connectors.md. #[cfg(feature = "connector-session")] diff --git a/aimdb-core/src/remote/metadata.rs b/aimdb-core/src/remote/metadata.rs index 7e408faa..cc3c951c 100644 --- a/aimdb-core/src/remote/metadata.rs +++ b/aimdb-core/src/remote/metadata.rs @@ -66,9 +66,10 @@ pub struct RecordMetadata { #[serde(default, skip_serializing_if = "Option::is_none")] pub schema_type: Option, - /// Entity / node identifier (e.g. `"vienna"` for `"temp.vienna"`), derived - /// from the record key's final `.` segment. The server is the authority on - /// naming conventions — clients use this field instead of parsing keys. + /// Entity / node identifier (e.g. `"vienna"` for `"temp.vienna"` or + /// `"sensors/temp/vienna"`), the record key's leaf segment after the last + /// `.` or `/` separator. The server is the authority on naming conventions — + /// clients use this field instead of parsing keys. #[serde(default, skip_serializing_if = "Option::is_none")] pub entity: Option, @@ -138,11 +139,7 @@ impl RecordMetadata { writable: bool, outbound_connector_count: usize, ) -> Self { - let entity = record_key - .as_str() - .rsplit('.') - .next() - .map(|s| s.to_string()); + let entity = Some(super::topic_leaf(record_key.as_str()).to_string()); Self { record_id: record_id.raw(), record_key: record_key.as_str().to_string(), diff --git a/aimdb-core/src/remote/mod.rs b/aimdb-core/src/remote/mod.rs index e3c7ac15..65eaeedd 100644 --- a/aimdb-core/src/remote/mod.rs +++ b/aimdb-core/src/remote/mod.rs @@ -61,3 +61,30 @@ pub use query::{QueryHandlerFn, QueryHandlerParams, QueryRecord}; // Internal exports for implementation #[cfg(feature = "connector-session")] pub(crate) mod stream; + +/// The leaf (entity) segment of a record key — the trailing component after the +/// last separator. Both `.` and `/` are treated as delimiters, since keys use +/// either convention (`temp.vienna` or `sensors/temp/vienna`), and both yield +/// `vienna`. Servers report this as a record's `entity` so clients trust the +/// field instead of parsing keys themselves. +pub fn topic_leaf(key: &str) -> &str { + key.rsplit(['.', '/']).next().unwrap_or(key) +} + +#[cfg(test)] +mod topic_leaf_tests { + use super::topic_leaf; + + #[test] + fn leaf_handles_both_separators() { + // Dot convention. + assert_eq!(topic_leaf("temp.vienna"), "vienna"); + // Slash convention (the bug: `rsplit('.')` returned the whole key). + assert_eq!(topic_leaf("sensors/temp/vienna"), "vienna"); + // Mixed — the last separator of either kind wins. + assert_eq!(topic_leaf("sensors/temp.vienna"), "vienna"); + assert_eq!(topic_leaf("a.b/c"), "c"); + // No separator: the whole key is its own leaf. + assert_eq!(topic_leaf("vienna"), "vienna"); + } +} diff --git a/aimdb-core/src/remote/query.rs b/aimdb-core/src/remote/query.rs index f2dea0d7..f51690ea 100644 --- a/aimdb-core/src/remote/query.rs +++ b/aimdb-core/src/remote/query.rs @@ -25,7 +25,7 @@ pub type QueryHandlerFn = Box< >; /// One row of a `record.query` result — the canonical shape every transport -/// shares (the retired ws-protocol `QueryRecord`, now core vocabulary). +/// shares. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct QueryRecord { /// Record key / topic the value was stored under (e.g. `"temp.vienna"`). diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index e5f432f4..5b1c2cf9 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -24,7 +24,7 @@ use hashbrown::HashMap; use super::{ BoxFut, BoxStream, Connection, Dialer, EnvelopeCodec, Inbound, Outbound, Payload, RpcError, - SubUpdate, + SubUpdate, TransportError, }; use crate::router::RouterBuilder; use crate::AimDb; @@ -103,7 +103,7 @@ enum ClientCmd { }, Subscribe { topic: String, - events: Sender, + events: Sender>, }, Write { topic: String, @@ -137,16 +137,20 @@ impl ClientHandle { /// Open a subscription; returns the stream of updates immediately (the engine /// sends the `Subscribe` request asynchronously). Dropping the stream stops - /// local delivery. The stream ends on disconnect and is not re-subscribed on - /// reconnect (see [`ClientConfig::reconnect`]) — re-call to resume. + /// local delivery. The stream is not re-subscribed on reconnect (see + /// [`ClientConfig::reconnect`]) — re-call to resume. /// - /// Each [`SubUpdate`] carries the concrete record topic when the server tags - /// it (wildcard subscriptions fan in many records under this one stream). + /// Each item is `Ok(`[`SubUpdate`]`)` for a delivered update, carrying the + /// concrete record topic when the server tags it (wildcard subscriptions fan + /// in many records under this one stream). A server rejection surfaces as a + /// terminal `Err(`[`RpcError`]`)` item, letting the caller distinguish a + /// denied subscription (do not retry) from a disconnect-shaped stream end + /// (retry to resume). pub fn subscribe( &self, topic: impl Into, - ) -> Result, RpcError> { - let (events, rx) = async_channel::unbounded::(); + ) -> Result>, RpcError> { + let (events, rx) = async_channel::unbounded::>(); self.enqueue(ClientCmd::Subscribe { topic: topic.into(), events, @@ -244,8 +248,16 @@ async fn client_loop( attempt = 0; conn } - Err(_e) => { - log_warn!("client dial failed: {:?}", _e); + Err(e) => { + log_warn!("client dial failed: {:?}", e); + // `Closed` is terminal — the dialer signals it will never succeed + // again (e.g. the caller stopped the bridge), so retrying would + // spin a permanently-failing redial forever. Only transient + // failures (`Io`) earn a backoff+retry. Other transports' dialers + // map connect failures to `Io`, never `Closed`, so this is safe. + if e == TransportError::Closed { + return; + } match reconnect_after(&mut attempt, &config, &cmd_rx, &*clock).await { true => continue, false => return, @@ -313,7 +325,7 @@ where let mut pending: HashMap>> = HashMap::new(); // sub-id → event sink. The sub-id is `id.to_string()` of the opening // request, matching the server's derivation so `Event.sub` routes back. - let mut subs: HashMap> = HashMap::new(); + let mut subs: HashMap>> = HashMap::new(); let mut out = Vec::new(); let keepalive_ms = config.keepalive_interval; // Keepalive is deadline-based: activity only records a timestamp (one dyn @@ -380,12 +392,16 @@ where Ok(Outbound::Reply { id, result }) => { if let Some(tx) = pending.remove(&id) { let _ = tx.send(result); - } else if result.is_err() { + } else if let Err(err) = result { // A subscribe is acked implicitly by its events; the // server replies only on failure, carrying the subscribe - // `id` (never a pending call). Drop the event sink so the - // stream ends instead of hanging. - subs.remove(&id.to_string()); + // `id` (never a pending call). Surface the rejection as a + // terminal error item before dropping the sink, so the + // subscriber can tell "denied" from a disconnect-shaped + // stream end instead of re-subscribing forever. + if let Some(tx) = subs.remove(&id.to_string()) { + let _ = tx.try_send(Err(err)); + } } } Ok(Outbound::Event { @@ -396,10 +412,10 @@ where }) => { let dead = match subs.get(sub) { Some(tx) => tx - .try_send(SubUpdate { + .try_send(Ok(SubUpdate { topic: topic.map(Arc::from), data, - }) + })) .is_err(), None => false, // late event for a dropped sub — ignore }; @@ -409,7 +425,7 @@ where } Ok(Outbound::Snapshot { sub, topic, data }) => { if let Some(tx) = subs.get(sub) { - let _ = tx.try_send(SubUpdate::tagged(Arc::from(topic), data)); + let _ = tx.try_send(Ok(SubUpdate::tagged(Arc::from(topic), data))); } } Ok(Outbound::Pong) => {} @@ -572,7 +588,13 @@ pub fn pump_client(db: &AimDb, scheme: &str, handle: &ClientHandle) -> Vec return, }; while let Some(update) = stream.next().await { - let _ = router.route(id.as_ref(), &update.data, &ctx); + match update { + Ok(update) => { + let _ = router.route(id.as_ref(), &update.data, &ctx); + } + // Server rejected the subscription — terminal, not replayed. + Err(_e) => break, + } } })); } diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index b9d6bf7a..f4823f4b 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -72,7 +72,7 @@ pub type Payload = Arc<[u8]>; /// any transport that tags every event, like the WS bus); `None` where the /// subscription is exact-topic and the wire stays minimal. `Arc` so /// per-event tagging is a refcount bump, not a string allocation. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct SubUpdate { /// Concrete record topic that fired, when the producer side tags it. pub topic: Option>, diff --git a/aimdb-core/src/session/topic_match.rs b/aimdb-core/src/session/topic_match.rs index 26f29d7d..922af7fa 100644 --- a/aimdb-core/src/session/topic_match.rs +++ b/aimdb-core/src/session/topic_match.rs @@ -3,8 +3,7 @@ //! Shared by every transport that supports wildcard subscriptions: the AimX //! wildcard subscribe ([`crate::session::aimx`]) matches a pattern against the //! registry once at subscribe time, and the WebSocket connector's fan-out bus -//! matches per broadcast. Moved here from the retired `aimdb-ws-protocol` -//! crate (design 045). +//! matches per broadcast. /// Returns `true` if `topic` matches `pattern`. /// diff --git a/aimdb-core/tests/session_engine.rs b/aimdb-core/tests/session_engine.rs index d05ad04c..e8797317 100644 --- a/aimdb-core/tests/session_engine.rs +++ b/aimdb-core/tests/session_engine.rs @@ -382,9 +382,9 @@ async fn echo_roundtrip_rpc_streaming_and_write() { // 2) Streaming: subscribe → three events routed back by sub id. let mut stream = handle.subscribe("temp").unwrap(); - let e1 = stream.next().await.expect("event 1"); - let e2 = stream.next().await.expect("event 2"); - let e3 = stream.next().await.expect("event 3"); + let e1 = stream.next().await.expect("event 1").expect("ok event 1"); + let e2 = stream.next().await.expect("event 2").expect("ok event 2"); + let e3 = stream.next().await.expect("event 3").expect("ok event 3"); assert_eq!(&*e1.data, b"temp#1"); assert_eq!(&*e2.data, b"temp#2"); assert_eq!(&*e3.data, b"temp#3"); @@ -411,8 +411,10 @@ async fn echo_roundtrip_rpc_streaming_and_write() { server.abort(); } -/// Subscribe-ack: a subscribe the server rejects must surface as a stream that -/// *ends* (`None`) rather than one that hangs forever (the pre-fix behavior). +/// Subscribe-ack: a subscribe the server rejects must surface as a terminal +/// `Err` item followed by end-of-stream — distinguishable from a disconnect (a +/// plain `None`) so a caller can stop instead of re-subscribing forever, and +/// never hanging (the original pre-fix behavior). #[tokio::test] async fn failed_subscribe_ends_stream_via_ack() { let (listener, dialer) = transport_pair(); @@ -441,13 +443,26 @@ async fn failed_subscribe_ends_stream_via_ack() { ); let client = tokio::spawn(client_fut); - // The "bad" topic is rejected server-side; the failure Reply must close the - // stream. A generous timeout guards against the old hang-forever behavior. + // The "bad" topic is rejected server-side; the failure Reply must surface as + // a terminal `Err` item, then the stream ends. A generous timeout guards + // against the old hang-forever behavior. let mut stream = handle.subscribe("bad").unwrap(); + let rejected = tokio::time::timeout(Duration::from_secs(2), stream.next()) + .await + .expect("failed subscribe must yield an item, not hang") + .expect("rejected subscribe should yield a terminal item"); + assert_eq!( + rejected.unwrap_err(), + RpcError::NotFound, + "rejection should carry the server's error code" + ); let ended = tokio::time::timeout(Duration::from_secs(2), stream.next()) .await - .expect("failed subscribe must end the stream, not hang"); - assert!(ended.is_none(), "rejected subscribe should yield no events"); + .expect("stream must end after the rejection, not hang"); + assert!( + ended.is_none(), + "stream should end after the terminal error" + ); drop(handle); drop(stream); @@ -501,12 +516,16 @@ async fn ended_subscription_frees_its_cap_slot() { // Drain a subscription's three echo updates; the server-side stream then // ends, so its pump finishes and is reaped. - async fn drain_three(stream: &mut BoxStream<'static, SubUpdate>, topic: &str) { + async fn drain_three( + stream: &mut BoxStream<'static, Result>, + topic: &str, + ) { for i in 1..=3 { let ev = tokio::time::timeout(Duration::from_secs(2), stream.next()) .await .expect("event should arrive") - .expect("an accepted subscription must yield its events"); + .expect("an accepted subscription must yield its events") + .expect("an accepted subscription must not be rejected"); assert_eq!(&*ev.data, format!("{topic}#{i}").as_bytes()); } } @@ -528,7 +547,7 @@ async fn ended_subscription_frees_its_cap_slot() { .await .expect("third subscribe must not hang"); assert_eq!( - first.map(|u| u.data.to_vec()), + first.map(|u| u.expect("third subscribe must be accepted").data.to_vec()), Some(b"c#1".to_vec()), "an ended subscription must free its cap slot; the third subscribe was refused" ); diff --git a/aimdb-wasm-adapter/src/bindings.rs b/aimdb-wasm-adapter/src/bindings.rs index e6b0577c..5d98363f 100644 --- a/aimdb-wasm-adapter/src/bindings.rs +++ b/aimdb-wasm-adapter/src/bindings.rs @@ -32,6 +32,8 @@ use wasm_bindgen::prelude::*; use aimdb_core::buffer::BufferCfg; use aimdb_core::builder::{AimDb, AimDbBuilder}; use aimdb_core::record_id::StringKey; +use aimdb_core::session::aimx::AimxCodec; +use aimdb_core::{EnvelopeCodec, Inbound, Outbound, Payload}; use crate::schema_registry::{SchemaOps, SchemaRegistry}; use crate::ws_bridge::WsBridge; @@ -342,9 +344,9 @@ impl WasmDb { /// Build a one-shot WebSocket promise that resolves with `TopicInfo[]`. /// -/// Speaks one raw AimX exchange: `{"t":"req","id":1,"method":"record.list"}` -/// → `{"t":"reply","id":1,"ok":[{name, schema_type, entity}, …]}` (design 045 -/// §2.1 — discovery is a plain `record.list` request). +/// Speaks one AimX exchange via the shared [`AimxCodec`]: a `record.list` +/// request (`id` 1) whose reply carries `[{name, schema_type, entity}, …]` +/// (design 045 §2.1 — discovery is a plain `record.list` request). /// /// Each callback pair (resolve, reject) is stored in an `Rc>` /// so that whichever event fires first wins and subsequent events are no-ops. @@ -365,12 +367,36 @@ fn discover_impl(url: String) -> js_sys::Promise { Rc::new(RefCell::new(Some(resolve))); let reject_rc: Rc>> = Rc::new(RefCell::new(Some(reject))); + // Encode the request frame once via the shared codec. + let request = { + let mut buf = Vec::new(); + if AimxCodec + .encode_inbound( + Inbound::Request { + id: 1, + method: "record.list".into(), + params: Payload::from(&b"null"[..]), + }, + &mut buf, + ) + .is_err() + { + if let Some(rej) = reject_rc.borrow_mut().take() { + let _ = rej.call1( + &JsValue::NULL, + &JsValue::from_str("failed to encode record.list request"), + ); + } + return; + } + String::from_utf8(buf).unwrap_or_default() + }; + // on_open: send the record.list request { let ws_clone = ws.clone(); let on_open = Closure::wrap(Box::new(move || { - let _ = ws_clone - .send_with_str(r#"{"t":"req","id":1,"method":"record.list","params":null}"#); + let _ = ws_clone.send_with_str(&request); }) as Box); ws.set_onopen(Some(on_open.as_ref().unchecked_ref())); on_open.forget(); @@ -392,19 +418,22 @@ fn discover_impl(url: String) -> js_sys::Promise { } return; }; - let Ok(frame) = serde_json::from_str::(&text) else { - return; // skip non-JSON noise; the timeout guards a dead peer + // Decode via the shared codec; ignore anything that isn't our + // `record.list` reply (auto-subscribe events, acks, other ids, + // non-JSON noise) — the timeout guards a dead peer. + let result = match AimxCodec.decode_outbound(text.as_bytes()) { + Ok(Outbound::Reply { id: 1, result }) => result, + _ => return, }; - // Ignore unrelated frames (auto-subscribe events, acks, …). - if frame["t"] != "reply" || frame["id"] != 1 { - return; - } let _ = ws_clone.close(); - match frame.get("ok").and_then(|v| v.as_array()) { + let topics = result + .ok() + .and_then(|ok| serde_json::from_slice::>(&ok).ok()); + match topics { Some(topics) => { let serializer = serde_wasm_bindgen::Serializer::json_compatible(); let arr = js_sys::Array::new(); - for topic in topics { + for topic in &topics { if let Ok(js_val) = topic.serialize(&serializer) { arr.push(&js_val); } diff --git a/aimdb-wasm-adapter/src/time.rs b/aimdb-wasm-adapter/src/time.rs index 21e615a1..ab14e78a 100644 --- a/aimdb-wasm-adapter/src/time.rs +++ b/aimdb-wasm-adapter/src/time.rs @@ -8,24 +8,27 @@ //! accessing `globalThis` via `js_sys::global()` instead of `web_sys::window()`. use crate::runtime::WasmAdapter; -#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] +#[cfg(feature = "wasm-runtime")] use core::future::Future; -#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] +#[cfg(feature = "wasm-runtime")] use core::pin::Pin; -#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] +#[cfg(feature = "wasm-runtime")] use core::task::{Context, Poll}; -/// A wrapper that unsafely implements `Send` for a future. +/// A wrapper that unconditionally implements `Send` for a future, so a +/// JS-touching (`!Send`) future can satisfy the engine's `Send` bounds and box +/// into the `dyn Future + Send` transport shapes. /// /// # Safety /// -/// Only safe on `wasm32-unknown-unknown` where all execution is single-threaded -/// **without** the `atomics` / shared-memory proposal enabled. -/// The inner future will never actually be sent between threads. -/// -/// The `Send` impl is gated on `target_arch = "wasm32"` so this type cannot -/// accidentally satisfy a `Send` bound when cross-compiled for a native target. -#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] +/// Only ever polled on `wasm32-unknown-unknown` without the `atomics` / +/// shared-memory proposal — single-threaded by construction (see the compile +/// guard below), so the inner future is never actually sent between threads. +/// The `Send` impl is unconditional (not gated to `target_arch = "wasm32"`) so +/// this type is also `Send` in the native test lane, where the WASM bridge +/// compiles — to satisfy the same `dyn Future + Send` coercions — but never +/// runs. +#[cfg(feature = "wasm-runtime")] pub(crate) struct SendFuture(pub(crate) F); // Guard: detect wasm32 + threads (atomics target feature). The shared-memory @@ -36,13 +39,12 @@ compile_error!( Disable the `atomics` target feature or provide a thread-safe implementation." ); -// SAFETY: wasm32 (without atomics) is single-threaded — the future cannot be -// sent to another thread. On non-wasm targets this impl is absent, so -// SendFuture is only Send when F: Send, which is the correct default. -#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] +// SAFETY: see the type's docs — wasm32 (without atomics) is single-threaded, and +// the native lane only compiles (never polls) this type. +#[cfg(feature = "wasm-runtime")] unsafe impl Send for SendFuture {} -#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] +#[cfg(feature = "wasm-runtime")] impl Future for SendFuture { type Output = F::Output; diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index cf599da2..d2ec74b2 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -23,9 +23,6 @@ use alloc::string::{String, ToString}; use alloc::sync::Arc; use alloc::vec::Vec; use core::cell::{Cell, RefCell}; -use core::future::Future; -use core::pin::Pin; -use core::task::{Context, Poll}; use futures_util::StreamExt; use serde::Deserialize; @@ -39,6 +36,7 @@ use aimdb_core::{ }; use crate::schema_registry::SchemaRegistry; +use crate::time::SendFuture; use crate::WasmAdapter; // ─── Connection status ──────────────────────────────────────────────────── @@ -75,11 +73,6 @@ pub struct BridgeOptions { /// Re-connect automatically on close (default: true). #[serde(default = "default_true")] pub auto_reconnect: bool, - /// Retained for option-shape compatibility: late-join snapshots are - /// server-driven under AimX (`with_late_join` on the server builder) and - /// arrive automatically on subscribe. - #[serde(default = "default_true")] - pub late_join: bool, /// Maximum queued commands (writes/subscribes) while disconnected /// (default: 256). #[serde(default = "default_queue_size")] @@ -111,7 +104,6 @@ impl Default for BridgeOptions { Self { subscribe_topics: Vec::new(), auto_reconnect: true, - late_join: true, max_offline_queue: 256, keepalive_ms: 30_000, query_timeout_ms: 30_000, @@ -119,31 +111,6 @@ impl Default for BridgeOptions { } } -// ─── Single-threaded `Send` shims ───────────────────────────────────────── - -/// Wraps a JS-touching future so it satisfies the engine's `Send` bounds. -/// -/// # Safety -/// -/// Only ever executed on `wasm32-unknown-unknown` without the `atomics` -/// proposal — single-threaded by construction (the compile guard in -/// [`crate::time`] rejects wasm threads). The unconditional impl exists so the -/// crate also *compiles* in the native test lane, where this code never runs — -/// the same rationale as [`WsBridge`]'s `Send`/`Sync` impls below. -struct JsSendFut(F); - -unsafe impl Send for JsSendFut {} - -impl Future for JsSendFut { - type Output = F::Output; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - // SAFETY: pure pin projection to the inner field. - let inner = unsafe { self.map_unchecked_mut(|s| &mut s.0) }; - inner.poll(cx) - } -} - // ─── Shared bridge state ────────────────────────────────────────────────── /// State shared between the JS-facing [`WsBridge`], the transport wrappers, @@ -196,13 +163,13 @@ struct WasmWsConnection { _plain_callbacks: Vec>, } -// SAFETY: wasm32 without atomics is single-threaded; see `JsSendFut`. +// SAFETY: wasm32 without atomics is single-threaded; see [`SendFuture`]. unsafe impl Send for WasmWsConnection {} impl Connection for WasmWsConnection { fn recv(&mut self) -> BoxFut<'_, TransportResult>>> { // `Ok(None)` when the frame funnel closes (socket closed/errored). - Box::pin(JsSendFut(async move { Ok(self.frames.next().await) })) + Box::pin(SendFuture(async move { Ok(self.frames.next().await) })) } fn send<'a>(&'a mut self, frame: &'a [u8]) -> BoxFut<'a, TransportResult<()>> { @@ -248,14 +215,14 @@ struct WasmWsDialer { shared: Rc, } -// SAFETY: wasm32 without atomics is single-threaded; see `JsSendFut`. +// SAFETY: wasm32 without atomics is single-threaded; see [`SendFuture`]. unsafe impl Send for WasmWsDialer {} impl Dialer for WasmWsDialer { fn connect(&self) -> BoxFut<'_, TransportResult>> { let url = self.url.clone(); let shared = self.shared.clone(); - Box::pin(JsSendFut(async move { + Box::pin(SendFuture(async move { if shared.stopped.get() { return Err(TransportError::Closed); } @@ -353,7 +320,6 @@ impl Dialer for WasmWsDialer { /// const bridge = db.connectBridge('wss://api.example.com/ws', { /// subscribeTopics: ['sensors/#'], /// autoReconnect: true, -/// lateJoin: true, /// }); /// bridge.onStatusChange((status) => updateIndicator(status)); /// // ... @@ -407,7 +373,10 @@ impl WsBridge { /// Close the WebSocket and stop reconnection attempts. pub fn disconnect(&self) { self.shared.stopped.set(true); - // Dropping the handle stops the engine (pending calls reject). + // Setting `stopped` makes the dialer return a terminal `Closed` on its + // next redial, which stops the engine — dropping this handle alone would + // not, since the subscription pumps hold their own clones. Dropping it + // still rejects pending calls once the engine drains its command channel. self.handle.borrow_mut().take(); if let Some(ws) = self.shared.ws.borrow_mut().take() { let _ = ws.close(); @@ -594,8 +563,10 @@ fn rpc_err_str(e: &RpcError) -> &'static str { // ─── Subscription pump ───────────────────────────────────────────────────── /// Subscribe to `pattern` and mirror every tagged update into the local -/// database; when the stream ends (disconnect or server rejection), -/// re-subscribe until the bridge is stopped. +/// database; when the stream ends on a disconnect, re-subscribe until the +/// bridge is stopped. A server rejection (a terminal error item) is permanent, +/// so the pump stops rather than spinning re-subscribe attempts the server will +/// keep denying. async fn pump_pattern( shared: Rc, handle: ClientHandle, @@ -613,13 +584,19 @@ async fn pump_pattern( Err(_) => return, // engine stopped }; while let Some(update) = stream.next().await { + let update = match update { + Ok(update) => update, + // Terminal rejection (e.g. the server denied the pattern): + // re-subscribing would only be denied again — stop this pump. + Err(_rejected) => return, + }; // Wildcard events carry the concrete record topic; an exact-topic // subscription may leave it implicit. let topic = update.topic.as_deref().unwrap_or(&pattern); route_update(&db, &schema_map, ®istry, topic, &update.data); } - // Stream ended — disconnected or rejected. Pace the re-subscribe so a - // rejecting server doesn't spin; the engine queues it while offline. + // Stream ended without a rejection — a disconnect. Pace the re-subscribe + // so we don't spin; the engine queues it while offline. WasmAdapter .sleep(core::time::Duration::from_millis(500)) .await; diff --git a/aimdb-websocket-connector/src/server/builder.rs b/aimdb-websocket-connector/src/server/builder.rs index 9a064eb7..9ca7f907 100644 --- a/aimdb-websocket-connector/src/server/builder.rs +++ b/aimdb-websocket-connector/src/server/builder.rs @@ -27,7 +27,7 @@ use aimdb_data_contracts::Streamable; use aimdb_core::{pump_sink, router::RouterBuilder, ConnectorBuilder, Dispatch}; use axum::Router as AxumRouter; -use aimdb_core::topic_matches; +use aimdb_core::{topic_leaf, topic_matches}; use super::{ auth::{AuthHandler, DynAuthHandler, NoAuth}, @@ -290,10 +290,11 @@ impl ConnectorBuilder for WebSocketConnectorBuilder { .streamable_registry .resolve_name(&type_id) .map(|s| s.to_string()); - // Extract entity from topic name: "temp.vienna" → "vienna". - // The server owns the naming convention — clients receive - // the entity as a first-class field and never parse topics. - let entity = topic.rsplit('.').next().map(|s| s.to_string()); + // Leaf segment of the topic ("temp.vienna" or + // "sensors/temp/vienna" → "vienna"). The server owns the + // naming convention — clients receive the entity as a + // first-class field and never parse topics. + let entity = Some(topic_leaf(&topic).to_string()); TopicInfo { name: topic, schema_type, diff --git a/aimdb-websocket-connector/tests/e2e.rs b/aimdb-websocket-connector/tests/e2e.rs index 078b7bd4..dd900d83 100644 --- a/aimdb-websocket-connector/tests/e2e.rs +++ b/aimdb-websocket-connector/tests/e2e.rs @@ -480,7 +480,7 @@ async fn client_engine_receives_broadcast_over_real_socket() { let mut got = None; for _ in 0..100 { inject(&db, "sensors/temp", json!(42)); - if let Ok(Some(item)) = timeout(Duration::from_millis(20), stream.next()).await { + if let Ok(Some(Ok(item))) = timeout(Duration::from_millis(20), stream.next()).await { got = Some(item); break; } diff --git a/docs/design/045-retire-ws-protocol-converge-on-aimx.md b/docs/design/045-retire-ws-protocol-converge-on-aimx.md index 62e735c2..d0212922 100644 --- a/docs/design/045-retire-ws-protocol-converge-on-aimx.md +++ b/docs/design/045-retire-ws-protocol-converge-on-aimx.md @@ -1,7 +1,7 @@ # 045 — Retire `aimdb-ws-protocol`: every transport speaks AimX -**Status:** 🚧 Proposed (mapping-table gate from 036 A2; implementation bound -to the next protocol-breaking release) +**Status:** ✅ Implemented (mapping-table gate from 036 A2; shipped on +`feat/retire-aimdb-ws-protocol`) **Scope:** delete the `aimdb-ws-protocol` crate and the WS-JSON envelope it defines, porting the WebSocket connector (`aimdb-websocket-connector`), the From 9fb3be1b7f263eb6192df877c64dda7580b08558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 18 Jul 2026 21:27:22 +0000 Subject: [PATCH 08/45] feat: add embassy-futures dependency and update embassy submodule --- Cargo.lock | 1 + _external/embassy | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 56c2cceb..456b551d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1446,6 +1446,7 @@ version = "0.9.1" dependencies = [ "defmt 1.0.1", "document-features", + "embassy-futures", "embassy-net-driver", "embassy-sync", "embassy-time", diff --git a/_external/embassy b/_external/embassy index 158325bd..44200294 160000 --- a/_external/embassy +++ b/_external/embassy @@ -1 +1 @@ -Subproject commit 158325bd4cfc13ace5b09fb572104b292f8442e6 +Subproject commit 4420029493483dfb7be20e1f9dccbd75c954ec4a From 94fd894fe98d4e3f05be1ee8f1494e9750dc0efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sun, 19 Jul 2026 08:11:46 +0000 Subject: [PATCH 09/45] feat: remove aimdb-ws-protocol and update related components to AimX --- Makefile | 6 ++---- ...n-aimx.md => 047-retire-ws-protocol-converge-on-aimx.md} | 0 2 files changed, 2 insertions(+), 4 deletions(-) rename docs/design/{045-retire-ws-protocol-converge-on-aimx.md => 047-retire-ws-protocol-converge-on-aimx.md} (100%) diff --git a/Makefile b/Makefile index c81eb5f8..da9d0dce 100644 --- a/Makefile +++ b/Makefile @@ -186,8 +186,7 @@ test: fmt: @printf "$(GREEN)Formatting code (workspace members only)...$(NC)\n" - @for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-ws-protocol aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring hello-spmc-ring-async aimdb-bench; do \ - @for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring-async aimdb-bench; do \ + @for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring hello-spmc-ring-async aimdb-bench; do \ printf "$(YELLOW) → Formatting $$pkg$(NC)\n"; \ cargo fmt -p $$pkg 2>/dev/null || true; \ done @@ -196,8 +195,7 @@ fmt: fmt-check: @printf "$(GREEN)Checking code formatting (workspace members only)...$(NC)\n" @FAILED=0; \ - for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-ws-protocol aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring hello-spmc-ring-async aimdb-bench; do \ - for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring-async aimdb-bench; do \ + for pkg in aimdb-derive aimdb-data-contracts aimdb-core aimdb-client aimdb-embassy-adapter aimdb-tokio-adapter aimdb-wasm-adapter aimdb-sync aimdb-persistence aimdb-persistence-sqlite aimdb-mqtt-connector aimdb-knx-connector aimdb-websocket-connector aimdb-uds-connector aimdb-serial-connector aimdb-tcp-connector aimdb-codegen aimdb-cli aimdb-mcp sync-api-demo tokio-mqtt-connector-demo embassy-mqtt-connector-demo tokio-knx-connector-demo embassy-knx-connector-demo embassy-serial-connector-demo embassy-bench-stm32h5 weather-mesh-common weather-hub weather-station-alpha weather-station-beta hello-mailbox hello-mailbox-async hello-single-latest hello-single-latest-async hello-spmc-ring hello-spmc-ring-async aimdb-bench; do \ printf "$(YELLOW) → Checking $$pkg$(NC)\n"; \ if ! cargo fmt -p $$pkg -- --check 2>&1; then \ printf "$(RED)❌ Formatting check failed for $$pkg$(NC)\n"; \ diff --git a/docs/design/045-retire-ws-protocol-converge-on-aimx.md b/docs/design/047-retire-ws-protocol-converge-on-aimx.md similarity index 100% rename from docs/design/045-retire-ws-protocol-converge-on-aimx.md rename to docs/design/047-retire-ws-protocol-converge-on-aimx.md From cbf715bfafc98f79fad64fc2904e51e918d3f52c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sun, 19 Jul 2026 08:20:42 +0000 Subject: [PATCH 10/45] feat: enhance safety for wasm32 by feature-gating Send and Sync for CancelToken, CancelHandle, WasmWsConnection, WasmWsDialer and WsBridge --- aimdb-wasm-adapter/src/buffer.rs | 8 +++++++- aimdb-wasm-adapter/src/ws_bridge.rs | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/aimdb-wasm-adapter/src/buffer.rs b/aimdb-wasm-adapter/src/buffer.rs index bb96107f..79a53aa0 100644 --- a/aimdb-wasm-adapter/src/buffer.rs +++ b/aimdb-wasm-adapter/src/buffer.rs @@ -411,10 +411,16 @@ pub(crate) struct CancelHandle { inner: Rc, } -// SAFETY: wasm32 is single-threaded — no concurrent access possible +// SAFETY: wasm32 is single-threaded — no concurrent access possible. +// Gated to wasm32 — `buffer.rs` is not feature-gated and compiles on any +// host target, where `Rc`-backed types are not actually Send/Sync. +#[cfg(target_arch = "wasm32")] unsafe impl Send for CancelToken {} +#[cfg(target_arch = "wasm32")] unsafe impl Sync for CancelToken {} +#[cfg(target_arch = "wasm32")] unsafe impl Send for CancelHandle {} +#[cfg(target_arch = "wasm32")] unsafe impl Sync for CancelHandle {} /// Create a linked cancel token/handle pair. diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index d2ec74b2..26accdef 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -164,6 +164,10 @@ struct WasmWsConnection { } // SAFETY: wasm32 without atomics is single-threaded; see [`SendFuture`]. +// Gated to wasm32 — this crate's `wasm-runtime` feature can be enabled on a +// native host build (e.g. for testing), where the single-threaded argument +// does not hold. +#[cfg(target_arch = "wasm32")] unsafe impl Send for WasmWsConnection {} impl Connection for WasmWsConnection { @@ -216,6 +220,8 @@ struct WasmWsDialer { } // SAFETY: wasm32 without atomics is single-threaded; see [`SendFuture`]. +// Gated to wasm32 — see the identical note on `WasmWsConnection` above. +#[cfg(target_arch = "wasm32")] unsafe impl Send for WasmWsDialer {} impl Dialer for WasmWsDialer { @@ -334,7 +340,10 @@ pub struct WsBridge { } // SAFETY: wasm32-unknown-unknown is single-threaded. +// Gated to wasm32 — see the identical note on `WasmWsConnection` above. +#[cfg(target_arch = "wasm32")] unsafe impl Send for WsBridge {} +#[cfg(target_arch = "wasm32")] unsafe impl Sync for WsBridge {} #[wasm_bindgen] From 8d2d6ac651106c4e30ae6a2f24ec144429ae503a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sun, 19 Jul 2026 11:43:21 +0000 Subject: [PATCH 11/45] feat: update AimX protocol to version 3.0 with breaking changes in handshake and compatibility checks --- CHANGELOG.md | 23 +++++++ aimdb-client/src/engine.rs | 33 +++++++-- aimdb-client/src/protocol.rs | 4 +- aimdb-core/src/remote/mod.rs | 3 +- aimdb-core/src/remote/protocol.rs | 52 ++++++++++++-- aimdb-core/src/session/aimx/codec.rs | 9 ++- aimdb-core/src/session/aimx/dispatch.rs | 23 ++++++- aimdb-core/src/session/mod.rs | 6 ++ .../tests/handshake_version.rs | 69 +++++++++++++++++++ 9 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 aimdb-uds-connector/tests/handshake_version.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 70e8e3ee..0b57b204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed — Design 048 WI1: AimX protocol version handshake gate (breaking) + +`PROTOCOL_VERSION` is bumped **`"2.0"` → `"3.0"`** to mark the design-047/048 +convergence as a breaking wire change (`record.query` results moved from +`{values, count}` to `{records, total}`; wildcard subscribe / auto-subscribe +added). The `hello` handshake now **negotiates** the version instead of ignoring +it: the server refuses a client whose declared major version is incompatible — +or absent — with a new `RpcError::VersionMismatch` (wire code +`version_mismatch`), and the client rejects a pre-3.x server's `welcome` +symmetrically. Compatibility is by **major** version (`version_compatible()`, +exported from `aimdb_core::remote`); a missing/malformed version fails closed. + +- **Breaking:** a pre-3.x client (including any hand-parsing the old + `ServerMessage` shape) is refused at `hello` and can no longer connect — it + fails fast at the handshake rather than tripping over the new reply shapes on + its first call. First-party clients (`aimdb-client`, and the `aimdb-cli` / + `aimdb-mcp` tools riding it) declare `"3.0"` and are unaffected. +- **Caveat carried from design 047 §3.6:** server-seeded auto-subscriptions are + invisible to `run_client`-based consumers. Client authors upgrading to 3.0 + must drive their own `subscribe` for records they want streamed. +- Schema-level record migration over AimX (the `Migratable` trait) is + **out of scope** here and tracked as a follow-up (`with_migration`). + ### Changed — Design 045: one wire protocol (AimX) for every transport (breaking) Implementation of [design 045](docs/design/045-retire-ws-protocol-converge-on-aimx.md) diff --git a/aimdb-client/src/engine.rs b/aimdb-client/src/engine.rs index fad86208..49d27fdb 100644 --- a/aimdb-client/src/engine.rs +++ b/aimdb-client/src/engine.rs @@ -27,7 +27,7 @@ use aimdb_core::session::{ use aimdb_tokio_adapter::TokioAdapter; use crate::error::{ClientError, ClientResult}; -use crate::protocol::{RecordMetadata, WelcomeMessage}; +use crate::protocol::{version_compatible, RecordMetadata, WelcomeMessage, PROTOCOL_VERSION}; /// Default deadline for the connect handshake (dial + `hello`/Welcome). Bounds /// the case where a peer accepts the socket but never replies — the engine has @@ -114,17 +114,38 @@ impl AimxConnection { let engine = tokio::spawn(engine_fut); let server_info = async { - let hello = json!({ "client": "aimdb-client" }); + let hello = json!({ "client": "aimdb-client", "version": PROTOCOL_VERSION }); let reply = timeout(connect_timeout, handle.call("hello", to_payload(&hello)?)) .await .map_err(|_| ClientError::connection_failed(label, "handshake timed out"))? - .map_err(|_| { - ClientError::connection_failed( + .map_err(|e| match e { + // The server refused our declared version (design 048 WI1) — + // report it as such, not as an unreachable-engine failure. + RpcError::VersionMismatch => ClientError::connection_failed( + label, + format!( + "protocol version mismatch: server rejected client version {PROTOCOL_VERSION}" + ), + ), + _ => ClientError::connection_failed( label, "handshake failed (engine could not reach server)", - ) + ), })?; - from_payload::(&reply) + let welcome = from_payload::(&reply)?; + // Reverse gate: a pre-3.x server ignores our `version` and completes + // the handshake with an old-shaped `welcome`. Detect that here so a + // new client fails fast instead of tripping over the old reply shapes. + if !version_compatible(&welcome.version) { + return Err(ClientError::connection_failed( + label, + format!( + "protocol version mismatch: server speaks {}, client speaks {PROTOCOL_VERSION}", + welcome.version + ), + )); + } + Ok(welcome) } .await; diff --git a/aimdb-client/src/protocol.rs b/aimdb-client/src/protocol.rs index 5b013206..585c610b 100644 --- a/aimdb-client/src/protocol.rs +++ b/aimdb-client/src/protocol.rs @@ -6,8 +6,8 @@ // Re-export protocol types from aimdb-core pub use aimdb_core::remote::{ - ErrorObject, Event, HelloMessage, RecordMetadata, Request, Response, WelcomeMessage, - PROTOCOL_VERSION, + version_compatible, ErrorObject, Event, HelloMessage, RecordMetadata, Request, Response, + WelcomeMessage, PROTOCOL_VERSION, }; /// Client identifier diff --git a/aimdb-core/src/remote/mod.rs b/aimdb-core/src/remote/mod.rs index 65eaeedd..6fdbf9f0 100644 --- a/aimdb-core/src/remote/mod.rs +++ b/aimdb-core/src/remote/mod.rs @@ -54,7 +54,8 @@ pub use config::{AimxConfig, SecurityPolicy}; pub use error::{RemoteError, RemoteResult}; pub use metadata::RecordMetadata; pub use protocol::{ - ErrorObject, Event, HelloMessage, Request, Response, WelcomeMessage, PROTOCOL_VERSION, + version_compatible, ErrorObject, Event, HelloMessage, Request, Response, WelcomeMessage, + PROTOCOL_VERSION, }; pub use query::{QueryHandlerFn, QueryHandlerParams, QueryRecord}; diff --git a/aimdb-core/src/remote/protocol.rs b/aimdb-core/src/remote/protocol.rs index 09cb427a..264abaa6 100644 --- a/aimdb-core/src/remote/protocol.rs +++ b/aimdb-core/src/remote/protocol.rs @@ -11,9 +11,35 @@ use alloc::vec::Vec; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; -/// Version of the AimX wire protocol spoken by this crate (v2 NDJSON tagged -/// frames; not backward-compatible with the legacy v1 framing). -pub const PROTOCOL_VERSION: &str = "2.0"; +/// Version of the AimX wire protocol spoken by this crate. +pub const PROTOCOL_VERSION: &str = "3.0"; + +/// Major-version component of a `"MAJOR.MINOR"` protocol string, or `None` if it +/// is empty or not in that shape. +fn protocol_major(version: &str) -> Option<&str> { + match version.split('.').next() { + Some(major) if !major.is_empty() => Some(major), + _ => None, + } +} + +/// Whether a peer advertising `their_version` is wire-compatible with this +/// crate's [`PROTOCOL_VERSION`]. +/// +/// Compatibility is by **major** version — a breaking wire change bumps the +/// major (2.x → 3.0), a compatible additive change bumps only the minor. A +/// missing or malformed version string is treated as **incompatible** (fail +/// closed): a peer that cannot state a version it speaks is exactly the legacy +/// client this gate exists to turn away (design 048 WI1). +pub fn version_compatible(their_version: &str) -> bool { + match ( + protocol_major(their_version), + protocol_major(PROTOCOL_VERSION), + ) { + (Some(theirs), Some(ours)) => theirs == ours, + _ => false, + } +} /// Client hello message #[derive(Debug, Clone, Serialize, Deserialize)] @@ -177,10 +203,28 @@ mod tests { }; let json = serde_json::to_string(&hello).unwrap(); - assert!(json.contains("\"version\":\"2.0\"")); + assert!(json.contains("\"version\":\"3.0\"")); assert!(json.contains("\"client\":\"test-client\"")); } + #[test] + fn version_compatible_matches_on_major() { + // Same major (current + a hypothetical future minor) → compatible. + assert!(version_compatible(PROTOCOL_VERSION)); + assert!(version_compatible("3.0")); + assert!(version_compatible("3.7")); + assert!(version_compatible("3")); // bare major + + // Older major (the legacy WS-era wire) → refused. + assert!(!version_compatible("2.0")); + assert!(!version_compatible("1.0")); + + // Missing / malformed → fail closed. + assert!(!version_compatible("")); + assert!(!version_compatible(".0")); + assert!(!version_compatible("garbage")); + } + #[test] fn test_request_serialization() { let request = Request { diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index 9e4fde36..74bb6247 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -101,6 +101,7 @@ fn err_code(e: &RpcError) -> &'static str { RpcError::NotFound => "not_found", RpcError::Denied => "denied", RpcError::Internal => "internal", + RpcError::VersionMismatch => "version_mismatch", } } @@ -108,6 +109,7 @@ fn code_err(s: &str) -> RpcError { match s { "not_found" => RpcError::NotFound, "denied" => RpcError::Denied, + "version_mismatch" => RpcError::VersionMismatch, _ => RpcError::Internal, } } @@ -368,7 +370,12 @@ mod tests { #[test] fn reply_err_codes_roundtrip() { - for err in [RpcError::NotFound, RpcError::Denied, RpcError::Internal] { + for err in [ + RpcError::NotFound, + RpcError::Denied, + RpcError::Internal, + RpcError::VersionMismatch, + ] { let frame = encode_outbound(Outbound::Reply { id: 1, result: Err(err.clone()), diff --git a/aimdb-core/src/session/aimx/dispatch.rs b/aimdb-core/src/session/aimx/dispatch.rs index d3287a23..d6c75ddb 100644 --- a/aimdb-core/src/session/aimx/dispatch.rs +++ b/aimdb-core/src/session/aimx/dispatch.rs @@ -196,7 +196,7 @@ impl AimxSession { /// Match the method and produce its JSON result (or an [`RpcError`]). async fn dispatch_call(&mut self, method: &str, params: Value) -> Result { match method { - "hello" => Ok(self.welcome()), + "hello" => self.hello(params), "record.list" => Ok(json!(self.db.list_records())), "record.get" => { let name = str_field(¶ms, "name").ok_or(RpcError::NotFound)?; @@ -360,6 +360,27 @@ impl AimxSession { })) } + /// `hello`: the version-gated handshake. + /// + /// The client's declared `version` must be major-compatible with this + /// server's [`PROTOCOL_VERSION`] ([`version_compatible`](crate::remote::version_compatible)). + /// A pre-3.x — or version-less — client is refused **here** with + /// [`RpcError::VersionMismatch`], rather than completing the handshake and + /// letting the client trip over the new `reply`/`event` shapes on its first + /// `record.query` (design 048 WI1, decision 2: hard refusal at `hello`). + fn hello(&self, params: Value) -> Result { + let version = str_field(¶ms, "version").unwrap_or_default(); + if !crate::remote::version_compatible(&version) { + log_warn!( + "refusing handshake: client protocol version {:?} incompatible with server {}", + version, + PROTOCOL_VERSION + ); + return Err(RpcError::VersionMismatch); + } + Ok(self.welcome()) + } + /// Build the `Welcome` from the security policy + writable records. /// /// `writable_records` is derived from the policy directly and intersected with diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index f4823f4b..bfea43b8 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -219,6 +219,12 @@ pub enum RpcError { Denied, /// The handler failed. Internal, + /// The peer's declared protocol version is incompatible with this server's + /// [`PROTOCOL_VERSION`](crate::remote::PROTOCOL_VERSION). Raised at the + /// handshake so an old-version client is refused fast, rather than + /// completing `hello` and tripping over the new reply/event shapes on its + /// first call. + VersionMismatch, } /// Authentication failure raised by [`Dispatch::authenticate`]. diff --git a/aimdb-uds-connector/tests/handshake_version.rs b/aimdb-uds-connector/tests/handshake_version.rs new file mode 100644 index 00000000..337ae4de --- /dev/null +++ b/aimdb-uds-connector/tests/handshake_version.rs @@ -0,0 +1,69 @@ +//! End-to-end handshake version gate (design 048 WI1). +//! +//! The AimX server must refuse a client whose declared protocol version is +//! major-incompatible — or absent — rather than completing `hello` into a +//! `welcome`/`reply` shape the client can't parse and letting it panic on its +//! first `record.query`. These drive the real [`AimxDispatch`] `hello` handler +//! through the [`Session`] surface (no socket needed) and assert the refusal. + +use std::sync::Arc; + +use aimdb_core::remote::{AimxConfig, PROTOCOL_VERSION}; +use aimdb_core::session::aimx::AimxDispatch; +use aimdb_core::session::{Dispatch, Payload, RpcError, Session, SessionCtx}; +use aimdb_core::AimDbBuilder; +use aimdb_tokio_adapter::TokioAdapter; +use serde_json::json; + +/// A dispatch session over an empty db — enough to exercise the handshake, which +/// gates on the declared version before it ever touches the record set. +async fn open_session() -> Box { + let (db, _runner) = AimDbBuilder::new() + .runtime(Arc::new(TokioAdapter)) + .build() + .await + .expect("build empty db"); + let dispatch = AimxDispatch::new(Arc::new(db), AimxConfig::uds_default()); + dispatch.open(&SessionCtx::default()) +} + +fn hello_payload(value: serde_json::Value) -> Payload { + Arc::from(serde_json::to_vec(&value).unwrap().as_slice()) +} + +#[tokio::test] +async fn current_version_completes_handshake() { + let mut session = open_session().await; + let reply = session + .call( + "hello", + hello_payload(json!({ "client": "test", "version": PROTOCOL_VERSION })), + ) + .await + .expect("current-version hello should be welcomed"); + let welcome: serde_json::Value = serde_json::from_slice(&reply).unwrap(); + assert_eq!(welcome["version"], PROTOCOL_VERSION); +} + +#[tokio::test] +async fn legacy_version_is_refused() { + let mut session = open_session().await; + let err = session + .call( + "hello", + hello_payload(json!({ "client": "test", "version": "2.0" })), + ) + .await + .expect_err("a pre-3.x client must be refused at the handshake"); + assert_eq!(err, RpcError::VersionMismatch); +} + +#[tokio::test] +async fn missing_version_is_refused() { + let mut session = open_session().await; + let err = session + .call("hello", hello_payload(json!({ "client": "test" }))) + .await + .expect_err("a version-less hello must be refused (fail closed)"); + assert_eq!(err, RpcError::VersionMismatch); +} From 9da13a919146a3a2cf9d6a61ee73a1ee7ce959a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sun, 19 Jul 2026 12:25:08 +0000 Subject: [PATCH 12/45] feat: update topic separator from '/' to '.' for wildcard subscriptions and related components --- CHANGELOG.md | 15 +++ aimdb-client/tests/aimx_session.rs | 16 ++-- aimdb-core/src/session/aimx/codec.rs | 4 +- aimdb-core/src/session/topic_match.rs | 91 +++++++++++++------ aimdb-wasm-adapter/README.md | 4 +- aimdb-wasm-adapter/src/ws_bridge.rs | 2 +- .../src/client/builder.rs | 4 +- .../src/server/builder.rs | 6 +- .../src/server/client_manager.rs | 10 +- aimdb-websocket-connector/tests/e2e.rs | 40 ++++---- 10 files changed, 119 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b57b204..464c1b74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,21 @@ exported from `aimdb_core::remote`); a missing/malformed version fails closed. - Schema-level record migration over AimX (the `Migratable` trait) is **out of scope** here and tracked as a follow-up (`with_migration`). +### Changed — Design 048 WI3a: dot is the one topic separator (breaking) + +`topic_matches` (wildcard subscribe / WS fan-out) now splits topic patterns on +**`.` only** — RabbitMQ topic-exchange semantics (dot segments, `*` +single-level, `#` multi-level). Previously it split on `/`, so a dot-separated +key like `temp.vienna` (the codebase's dominant key style) silently failed to +match `temp.*` — a correctness bug. `/` is no longer a segment separator; it +stays only in external broker addresses (`mqtt://sensors/temp/x`), which are +unaffected. + +- **Breaking:** WebSocket / browser clients must subscribe with dot patterns + (`sensors.#`, not `sensors/#`). The WS connector, its e2e suite, ACL topics, + and the wasm-adapter README were migrated accordingly. +- `topic_leaf` (entity extraction) remains tolerant of both `.` and `/`. + ### Changed — Design 045: one wire protocol (AimX) for every transport (breaking) Implementation of [design 045](docs/design/045-retire-ws-protocol-converge-on-aimx.md) diff --git a/aimdb-client/tests/aimx_session.rs b/aimdb-client/tests/aimx_session.rs index 80269960..f60f13b7 100644 --- a/aimdb-client/tests/aimx_session.rs +++ b/aimdb-client/tests/aimx_session.rs @@ -154,7 +154,7 @@ async fn wildcard_subscribe_fans_in_matching_records() { let mut builder = AimDbBuilder::new() .runtime(Arc::new(TokioAdapter)) .with_connector(UdsServer::from_config(config)); - for key in ["temp/vienna", "temp/berlin", "humidity/london"] { + for key in ["temp.vienna", "temp.berlin", "humidity.london"] { builder.configure::(key, |reg| { reg.buffer(BufferCfg::SingleLatest).with_remote_access(); }); @@ -165,14 +165,14 @@ async fn wildcard_subscribe_fans_in_matching_records() { // Seed one matched record before subscribing — it must arrive as a // late-join snapshot on the wildcard stream. - db.set_record_from_json("temp/vienna", json!({ "level": 1 })) + db.set_record_from_json("temp.vienna", json!({ "level": 1 })) .expect("seed vienna"); let conn = AimxConnection::connect(sock.to_str().unwrap()) .await .expect("connect"); let mut stream = conn - .subscribe_with_topics("temp/#") + .subscribe_with_topics("temp.#") .expect("wildcard subscribe"); // The snapshot for the seeded record arrives first, tagged with its topic. @@ -180,7 +180,7 @@ async fn wildcard_subscribe_fans_in_matching_records() { .await .expect("snapshot within timeout") .expect("snapshot"); - assert_eq!(topic.as_deref(), Some("temp/vienna")); + assert_eq!(topic.as_deref(), Some("temp.vienna")); assert_eq!(value, json!({ "level": 1 })); // Live updates from both matched records ride the one subscription, each @@ -188,8 +188,8 @@ async fn wildcard_subscribe_fans_in_matching_records() { let db2 = db.clone(); tokio::spawn(async move { for n in 1..=50u64 { - let _ = db2.set_record_from_json("temp/berlin", json!({ "level": n })); - let _ = db2.set_record_from_json("humidity/london", json!({ "level": n })); + let _ = db2.set_record_from_json("temp.berlin", json!({ "level": n })); + let _ = db2.set_record_from_json("humidity.london", json!({ "level": n })); tokio::time::sleep(Duration::from_millis(10)).await; } }); @@ -202,11 +202,11 @@ async fn wildcard_subscribe_fans_in_matching_records() { Ok(Some((topic, value))) => { let topic = topic.expect("wildcard events are topic-tagged"); assert!( - topic.starts_with("temp/"), + topic.starts_with("temp."), "event from outside the pattern: {topic}" ); assert!(value.get("level").is_some()); - if topic == "temp/berlin" { + if topic == "temp.berlin" { saw_berlin = true; break; } diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index 74bb6247..88e2f1c8 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -313,11 +313,11 @@ mod tests { fn subscribe_and_unsubscribe_roundtrip() { match roundtrip_inbound(Inbound::Subscribe { id: 3, - topic: "sensors/#".to_string(), + topic: "sensors.#".to_string(), }) { Inbound::Subscribe { id, topic } => { assert_eq!(id, 3); - assert_eq!(topic, "sensors/#"); + assert_eq!(topic, "sensors.#"); } _ => panic!("expected Subscribe"), } diff --git a/aimdb-core/src/session/topic_match.rs b/aimdb-core/src/session/topic_match.rs index 922af7fa..75dac76d 100644 --- a/aimdb-core/src/session/topic_match.rs +++ b/aimdb-core/src/session/topic_match.rs @@ -1,20 +1,27 @@ -//! MQTT-style topic pattern matching (pure `&str` ops, no_std-safe). +//! Topic pattern matching over **dot-separated** record keys (pure `&str` ops, +//! no_std-safe). //! //! Shared by every transport that supports wildcard subscriptions: the AimX //! wildcard subscribe ([`crate::session::aimx`]) matches a pattern against the //! registry once at subscribe time, and the WebSocket connector's fan-out bus //! matches per broadcast. +//! +//! The one segment separator is `.` — AimDB record keys are dot-delimited +//! (`temp.vienna`, `app.config`), so wildcards split on `.`. The grammar +//! (dot segments, `*` single-level, `#` multi-level) is RabbitMQ topic-exchange +//! semantics. `/` is an ordinary character here — it belongs to external broker +//! addresses (`mqtt://sensors/temp/x`), not to AimDB's subscription grammar. /// Returns `true` if `topic` matches `pattern`. /// -/// Follows MQTT wildcard conventions: +/// Wildcard conventions over **dot-separated** segments: /// /// | Pattern | Semantics | /// |----------|-----------------------------------| /// | `#` | Multi-level wildcard (all topics) | -/// | `a/#` | Everything under `a/` | -/// | `a/*/c` | Single-level wildcard in segment | -/// | `a/b/c` | Exact match | +/// | `a.#` | Everything under `a.` | +/// | `a.*.c` | Single-level wildcard in segment | +/// | `a.b.c` | Exact match | pub fn topic_matches(pattern: &str, topic: &str) -> bool { // Fast path: exact match if pattern == topic { @@ -26,20 +33,20 @@ pub fn topic_matches(pattern: &str, topic: &str) -> bool { return true; } - // `prefix/#` matches everything under prefix — only when prefix is literal + // `prefix.#` matches everything under prefix — only when prefix is literal // (no wildcards in the prefix). When wildcards are present, fall through to // the segment loop which handles `#` at any position. - if let Some(prefix) = pattern.strip_suffix("/#") { + if let Some(prefix) = pattern.strip_suffix(".#") { if !prefix.contains('*') && !prefix.contains('#') { return topic.starts_with(prefix) && (topic.len() == prefix.len() - || topic.as_bytes().get(prefix.len()) == Some(&b'/')); + || topic.as_bytes().get(prefix.len()) == Some(&b'.')); } } // Segment-by-segment matching with `*` single-level wildcard - let mut pattern_parts = pattern.split('/'); - let mut topic_parts = topic.split('/'); + let mut pattern_parts = pattern.split('.'); + let mut topic_parts = topic.split('.'); loop { match (pattern_parts.next(), topic_parts.next()) { @@ -64,57 +71,81 @@ mod tests { #[test] fn exact_match() { - assert!(topic_matches("a/b/c", "a/b/c")); - assert!(!topic_matches("a/b/c", "a/b/d")); + assert!(topic_matches("a.b.c", "a.b.c")); + assert!(!topic_matches("a.b.c", "a.b.d")); } #[test] fn hash_wildcard() { - assert!(topic_matches("#", "anything/goes/here")); + assert!(topic_matches("#", "anything.goes.here")); assert!(topic_matches("#", "a")); } #[test] fn prefix_hash_wildcard() { - assert!(topic_matches("sensors/#", "sensors/temperature/vienna")); - assert!(topic_matches("sensors/#", "sensors/humidity/berlin")); - assert!(!topic_matches("sensors/#", "commands/setpoint")); + assert!(topic_matches("sensors.#", "sensors.temperature.vienna")); + assert!(topic_matches("sensors.#", "sensors.humidity.berlin")); + assert!(!topic_matches("sensors.#", "commands.setpoint")); // Edge: prefix itself - assert!(topic_matches("sensors/#", "sensors")); + assert!(topic_matches("sensors.#", "sensors")); + // A literal prefix is a whole segment — `sensors.#` must not swallow a + // key that merely *starts with* the string "sensors". + assert!(!topic_matches("sensors.#", "sensors_extra.temp")); } #[test] fn star_wildcard() { assert!(topic_matches( - "sensors/temperature/*", - "sensors/temperature/vienna" + "sensors.temperature.*", + "sensors.temperature.vienna" )); assert!(topic_matches( - "sensors/temperature/*", - "sensors/temperature/berlin" + "sensors.temperature.*", + "sensors.temperature.berlin" )); assert!(!topic_matches( - "sensors/temperature/*", - "sensors/humidity/vienna" + "sensors.temperature.*", + "sensors.humidity.vienna" )); assert!(!topic_matches( - "sensors/temperature/*", - "sensors/temperature/a/b" + "sensors.temperature.*", + "sensors.temperature.a.b" )); } + #[test] + fn star_matches_dotted_key_below_top_level() { + // The WI3 3a regression: a single-segment-below wildcard must match a + // dot-separated key. `temp.*` matches `temp.vienna`, not the old bug + // where `/`-splitting compared the literals `"temp.*"` and `"temp.vienna"`. + assert!(topic_matches("temp.*", "temp.vienna")); + assert!(topic_matches("temp.*", "temp.berlin")); + assert!(!topic_matches("temp.*", "temp")); + assert!(!topic_matches("temp.*", "temp.vienna.indoor")); + assert!(!topic_matches("temp.*", "humidity.vienna")); + } + #[test] fn mixed_wildcards() { - assert!(topic_matches("a/*/c/#", "a/b/c/d/e/f")); - assert!(!topic_matches("a/*/c/#", "a/b/x/d")); + assert!(topic_matches("a.*.c.#", "a.b.c.d.e.f")); + assert!(!topic_matches("a.*.c.#", "a.b.x.d")); + } + + #[test] + fn slash_is_not_a_separator() { + // `/` is an ordinary character — a slash key is one literal segment, so a + // dot wildcard doesn't reach into it and a slash "wildcard" isn't one. + assert!(!topic_matches("sensors.#", "sensors/temp/vienna")); + assert!(topic_matches("sensors/temp", "sensors/temp")); // exact still works + assert!(!is_wildcard("sensors/temp")); // no `#`/`*` → literal } #[test] fn wildcard_detection() { assert!(is_wildcard("#")); - assert!(is_wildcard("sensors/#")); - assert!(is_wildcard("a/*/c")); - // Dot-separated keys are single segments — literal unless `#`/`*` appears. + assert!(is_wildcard("sensors.#")); + assert!(is_wildcard("a.*.c")); + // Literal dotted keys are wildcards only when `#`/`*` appears. assert!(!is_wildcard("temp.vienna")); assert!(is_wildcard("temp.*")); } diff --git a/aimdb-wasm-adapter/README.md b/aimdb-wasm-adapter/README.md index a5d09d05..66cd23e1 100644 --- a/aimdb-wasm-adapter/README.md +++ b/aimdb-wasm-adapter/README.md @@ -72,7 +72,7 @@ Connect the browser-local AimDB to a remote server: import { WsBridge } from '@aimdb/wasm'; const bridge = WsBridge.connect(db, 'wss://api.example.com/ws', { - subscribeTopics: ['sensors/#'], + subscribeTopics: ['sensors.#'], autoReconnect: true, lateJoin: true, }); @@ -96,7 +96,7 @@ function App() { records: [ { key: 'sensors.temperature.vienna', schemaType: 'temperature', buffer: 'SingleLatest' }, ], - bridge: { url: 'wss://api.example.com/ws', subscribeTopics: ['sensors/#'] }, + bridge: { url: 'wss://api.example.com/ws', subscribeTopics: ['sensors.#'] }, }}> diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index 26accdef..84333f6b 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -67,7 +67,7 @@ impl ConnectionStatus { #[derive(Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct BridgeOptions { - /// MQTT-style topic patterns to subscribe to (e.g. `["sensors/#"]`). + /// Dot-separated topic patterns to subscribe to (e.g. `["sensors.#"]`). #[serde(default)] pub subscribe_topics: Vec, /// Re-connect automatically on close (default: true). diff --git a/aimdb-websocket-connector/src/client/builder.rs b/aimdb-websocket-connector/src/client/builder.rs index df984a83..a3cd9d79 100644 --- a/aimdb-websocket-connector/src/client/builder.rs +++ b/aimdb-websocket-connector/src/client/builder.rs @@ -54,7 +54,7 @@ pub struct WsClientConnectorBuilder { /// Maximum queued writes while disconnected (default: 256). max_offline_queue: usize, /// Topics to subscribe to on the remote server immediately after connect. - /// Wildcards supported (e.g., `["sensors/#"]`). + /// Wildcards supported (e.g., `["sensors.#"]`). subscribe_topics: Vec, } @@ -110,7 +110,7 @@ impl WsClientConnectorBuilder { /// ```no_run /// # use aimdb_websocket_connector::WsClientConnector; /// WsClientConnector::new("wss://cloud/ws") - /// .with_subscribe_topics(["sensors/#", "config/#"]); + /// .with_subscribe_topics(["sensors.#", "config.#"]); /// ``` pub fn with_subscribe_topics( mut self, diff --git a/aimdb-websocket-connector/src/server/builder.rs b/aimdb-websocket-connector/src/server/builder.rs index 9ca7f907..bc24069d 100644 --- a/aimdb-websocket-connector/src/server/builder.rs +++ b/aimdb-websocket-connector/src/server/builder.rs @@ -202,7 +202,7 @@ impl WebSocketConnectorBuilder { /// ```no_run /// # use aimdb_websocket_connector::WebSocketConnector; /// WebSocketConnector::new() - /// .with_auto_subscribe(["sensors/#"]); // or ["#"] to push everything + /// .with_auto_subscribe(["sensors.#"]); // or ["#"] to push everything /// ``` pub fn with_auto_subscribe( mut self, @@ -290,8 +290,8 @@ impl ConnectorBuilder for WebSocketConnectorBuilder { .streamable_registry .resolve_name(&type_id) .map(|s| s.to_string()); - // Leaf segment of the topic ("temp.vienna" or - // "sensors/temp/vienna" → "vienna"). The server owns the + // Leaf segment of the topic ("sensors.temp.vienna" → + // "vienna"). The server owns the // naming convention — clients receive the entity as a // first-class field and never parse topics. let entity = Some(topic_leaf(&topic).to_string()); diff --git a/aimdb-websocket-connector/src/server/client_manager.rs b/aimdb-websocket-connector/src/server/client_manager.rs index ff7c06d6..9739b67a 100644 --- a/aimdb-websocket-connector/src/server/client_manager.rs +++ b/aimdb-websocket-connector/src/server/client_manager.rs @@ -153,14 +153,14 @@ mod tests { #[tokio::test] async fn broadcast_reaches_matching_subscriptions() { let mgr = ClientManager::new(256); - let (_id, mut stream) = mgr.subscribe("sensors/#"); + let (_id, mut stream) = mgr.subscribe("sensors.#"); - mgr.broadcast("sensors/temp/vienna", b"22.5").await; + mgr.broadcast("sensors.temp.vienna", b"22.5").await; // Delivery is the raw payload tagged with the real topic — even for the // wildcard sub; the envelope is the per-connection codec's job. let update = stream.next().await.expect("should receive"); - assert_eq!(update.topic.as_deref(), Some("sensors/temp/vienna")); + assert_eq!(update.topic.as_deref(), Some("sensors.temp.vienna")); assert_eq!(&update.data[..], b"22.5"); } @@ -168,8 +168,8 @@ mod tests { async fn non_matching_topic_is_not_delivered() { use futures_util::FutureExt; let mgr = ClientManager::new(256); - let (_id, mut stream) = mgr.subscribe("commands/#"); - mgr.broadcast("sensors/temp", b"22.5").await; + let (_id, mut stream) = mgr.subscribe("commands.#"); + mgr.broadcast("sensors.temp", b"22.5").await; // Nothing queued: the next() future is not ready. assert!(stream.next().now_or_never().is_none()); } diff --git a/aimdb-websocket-connector/tests/e2e.rs b/aimdb-websocket-connector/tests/e2e.rs index dd900d83..220a4358 100644 --- a/aimdb-websocket-connector/tests/e2e.rs +++ b/aimdb-websocket-connector/tests/e2e.rs @@ -231,7 +231,7 @@ async fn server_subscribe_ack_and_wildcard_fanout() { let (addr, db) = spawn_default().await; let mut c = ws_connect(addr).await; - ws_send(&mut c, json!({"t":"sub","id":1,"topic":"sensors/#"})).await; + ws_send(&mut c, json!({"t":"sub","id":1,"topic":"sensors.#"})).await; // Explicit ack (acks_subscribe:true): the sub id echoes the request id. assert_eq!( ws_recv(&mut c).await, @@ -241,10 +241,10 @@ async fn server_subscribe_ack_and_wildcard_fanout() { // The ack means the bus subscription is registered, so a fan-out reaches us // — tagged with the concrete topic the wildcard matched. - inject(&db, "sensors/temp/vienna", json!(22.5)); + inject(&db, "sensors.temp.vienna", json!(22.5)); let ev = ws_recv_tag(&mut c, "event").await; assert_eq!(ev["sub"], "1"); - assert_eq!(ev["topic"], "sensors/temp/vienna"); + assert_eq!(ev["topic"], "sensors.temp.vienna"); assert_eq!(ev["data"], json!(22.5)); } @@ -288,29 +288,29 @@ async fn server_two_subscriptions_and_unsubscribe() { async fn server_late_join_snapshot() { let (addr, db) = spawn_default().await; // Produce the value first so the late-join cache holds it, then subscribe. - inject(&db, "sensors/temp", json!(99)); + inject(&db, "sensors.temp", json!(99)); tokio::time::sleep(Duration::from_millis(100)).await; let mut c = ws_connect(addr).await; - ws_send(&mut c, json!({"t":"sub","id":4,"topic":"sensors/temp"})).await; + ws_send(&mut c, json!({"t":"sub","id":4,"topic":"sensors.temp"})).await; assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"4"})); // The snapshot rides between the ack and the first event, tagged with the // subscription that triggered it. assert_eq!( ws_recv(&mut c).await, - json!({"t":"snap","sub":"4","topic":"sensors/temp","data":99}) + json!({"t":"snap","sub":"4","topic":"sensors.temp","data":99}) ); } #[tokio::test] async fn server_wildcard_late_join_snapshots_per_match() { let (addr, db) = spawn_default().await; - inject(&db, "sensors/temp", json!(1)); - inject(&db, "sensors/humidity", json!(2)); + inject(&db, "sensors.temp", json!(1)); + inject(&db, "sensors.humidity", json!(2)); tokio::time::sleep(Duration::from_millis(100)).await; let mut c = ws_connect(addr).await; - ws_send(&mut c, json!({"t":"sub","id":9,"topic":"sensors/#"})).await; + ws_send(&mut c, json!({"t":"sub","id":9,"topic":"sensors.#"})).await; assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"9"})); // One snapshot per cached record under the pattern (order is map order). let mut snaps = Vec::new(); @@ -320,7 +320,7 @@ async fn server_wildcard_late_join_snapshots_per_match() { snaps.push(s["topic"].as_str().unwrap().to_string()); } snaps.sort(); - assert_eq!(snaps, vec!["sensors/humidity", "sensors/temp"]); + assert_eq!(snaps, vec!["sensors.humidity", "sensors.temp"]); } #[tokio::test] @@ -474,12 +474,12 @@ async fn client_engine_receives_broadcast_over_real_socket() { ); let driver = tokio::spawn(engine); - let mut stream = handle.subscribe("sensors/temp").unwrap(); + let mut stream = handle.subscribe("sensors.temp").unwrap(); // Subscription registration is async; re-inject until the value arrives. let mut got = None; for _ in 0..100 { - inject(&db, "sensors/temp", json!(42)); + inject(&db, "sensors.temp", json!(42)); if let Ok(Some(Ok(item))) = timeout(Duration::from_millis(20), stream.next()).await { got = Some(item); break; @@ -488,7 +488,7 @@ async fn client_engine_receives_broadcast_over_real_socket() { // The record value round-trips; the bus tags every event with its topic. let update = got.expect("a value"); assert_eq!(&update.data[..], b"42"); - assert_eq!(update.topic.as_deref(), Some("sensors/temp")); + assert_eq!(update.topic.as_deref(), Some("sensors.temp")); drop(handle); drop(stream); @@ -504,16 +504,16 @@ async fn many_clients_fanout() { let mut clients = Vec::new(); for _ in 0..20 { let mut c = ws_connect(addr).await; - ws_send(&mut c, json!({"t":"sub","id":1,"topic":"evt/#"})).await; + ws_send(&mut c, json!({"t":"sub","id":1,"topic":"evt.#"})).await; assert_eq!(ws_recv(&mut c).await["t"], "subscribed"); clients.push(c); } // One broadcast reaches all 20. - inject(&db, "evt/x", json!(1)); + inject(&db, "evt.x", json!(1)); for c in &mut clients { let ev = ws_recv_tag(c, "event").await; - assert_eq!(ev["topic"], "evt/x"); + assert_eq!(ev["topic"], "evt.x"); } } @@ -581,16 +581,16 @@ async fn async_authorize_subscribe_gates_despite_allow_all_permissions() { // Denied topic: permissions are allow-all, but the *async* hook says no. // The refusal is a `reply` carrying the subscribe id + the 3-code error. - ws_send(&mut c, json!({"t":"sub","id":1,"topic":"secret/x"})).await; + ws_send(&mut c, json!({"t":"sub","id":1,"topic":"secret.x"})).await; assert_eq!( ws_recv(&mut c).await, json!({"t":"reply","id":1,"err":"denied"}) ); // An allowed topic still works end-to-end. - ws_send(&mut c, json!({"t":"sub","id":2,"topic":"public/x"})).await; + ws_send(&mut c, json!({"t":"sub","id":2,"topic":"public.x"})).await; assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"2"})); - inject(&db, "public/x", json!(1)); + inject(&db, "public.x", json!(1)); let ev = ws_recv_tag(&mut c, "event").await; - assert_eq!(ev["topic"], "public/x"); + assert_eq!(ev["topic"], "public.x"); } From c70263f3cd383b249f8bf07c00ed82deb7bdd848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sun, 19 Jul 2026 17:33:45 +0000 Subject: [PATCH 13/45] feat: enhance subscription updates to include skipped record count --- aimdb-core/src/remote/stream.rs | 46 +++++++++++++------- aimdb-core/src/session/aimx/dispatch.rs | 11 ++--- aimdb-core/src/session/client.rs | 58 +++++++++++++++++++------ aimdb-core/src/session/mod.rs | 17 +++++++- aimdb-core/src/session/server.rs | 2 +- 5 files changed, 98 insertions(+), 36 deletions(-) diff --git a/aimdb-core/src/remote/stream.rs b/aimdb-core/src/remote/stream.rs index 6adddcc8..786dfbea 100644 --- a/aimdb-core/src/remote/stream.rs +++ b/aimdb-core/src/remote/stream.rs @@ -1,8 +1,10 @@ //! Record-update streaming helper for AimX remote access. //! //! [`stream_record_updates`] adapts a record's [`JsonBufferReader`] into a -//! plain [`Stream`] of JSON values. Cancellation is by `drop`; backpressure -//! is the underlying buffer's responsibility. The handler holds the +//! [`Stream`] of `(JSON value, skipped)` pairs, where `skipped` is the number +//! of updates the buffer dropped immediately before this one. +//! Cancellation is by `drop`; backpressure is the underlying buffer's +//! responsibility. The handler holds the //! returned stream inside its per-subscription future so that the //! connection's `FuturesUnordered` is the sole owner of the subscription's //! lifecycle. @@ -14,12 +16,13 @@ use crate::{AimDb, DbError, DbResult}; use futures_core::Stream; use futures_util::stream::unfold; -/// Subscribe to a record and yield each update as a JSON value. +/// Subscribe to a record and yield each update as a `(JSON value, skipped)` +/// pair. /// /// The returned stream owns the underlying [`JsonBufferReader`]; dropping -/// it cancels the subscription. Lag (`BufferLagged`) is logged via -/// `tracing` and skipped; `BufferClosed` and other errors terminate the -/// stream cleanly (next `poll_next` returns `None`). +/// it cancels the subscription. Lag (`BufferLagged`) is logged via `tracing` +/// and its `lag_count` is accumulated into the `skipped` count carried on the +/// *next* yielded value; `BufferClosed` and other errors terminate the stream cleanly (next `poll_next` returns `None`). /// /// # Errors /// - [`DbError::RecordKeyNotFound`] if no record matches `record_key`. @@ -29,7 +32,7 @@ use futures_util::stream::unfold; pub(crate) fn stream_record_updates( db: &AimDb, record_key: &str, -) -> DbResult + Send + 'static> { +) -> DbResult + Send + 'static> { let inner = db.inner(); let id = inner .resolve_str(record_key) @@ -52,18 +55,22 @@ pub(crate) fn stream_record_updates( // logs identify which record fell behind — the previous mpsc-based // path carried this via `record_metadata.name`, and dropping it // hides which subscription is misbehaving in mixed-record traces. - let state = (reader, record_key.to_string()); + // `skipped` accumulates `BufferLagged.lag_count` across consecutive lag + // events so it can ride the next successfully-read value; it resets to 0 + // once a value is yielded. + let state = (reader, record_key.to_string(), 0_u64); - Ok(unfold(state, |(mut reader, key)| async move { + Ok(unfold(state, |(mut reader, key, mut skipped)| async move { loop { match reader.recv_json().await { - Ok(value) => return Some((value, (reader, key))), + Ok(value) => return Some(((value, skipped), (reader, key, 0))), Err(DbError::BufferLagged { lag_count, .. }) => { log_warn!( "stream_record_updates: record '{}' subscription lagged by {} messages", key, lag_count ); + skipped = skipped.saturating_add(lag_count); continue; } Err(DbError::BufferClosed { .. }) => return None, @@ -121,16 +128,21 @@ mod tests { } #[tokio::test] - async fn unfold_skips_lag_and_terminates_on_closed() { + async fn unfold_carries_lag_count_and_terminates_on_closed() { let reader = JsonReader::new(Box::new(FakeReader { step: Arc::new(AtomicUsize::new(0)), })); - let stream = unfold(reader, |mut reader| async move { + // Mirrors `stream_record_updates`'s unfold: accumulate `lag_count` into + // `skipped` and let it ride the next yielded value. + let stream = unfold((reader, 0_u64), |(mut reader, mut skipped)| async move { loop { match reader.recv_json().await { - Ok(v) => return Some((v, reader)), - Err(DbError::BufferLagged { .. }) => continue, + Ok(v) => return Some(((v, skipped), (reader, 0))), + Err(DbError::BufferLagged { lag_count, .. }) => { + skipped = skipped.saturating_add(lag_count); + continue; + } Err(DbError::BufferClosed { .. }) => return None, Err(_) => return None, } @@ -139,7 +151,9 @@ mod tests { let values: Vec<_> = stream.collect().await; assert_eq!(values.len(), 2); - assert_eq!(values[0], serde_json::json!({"v": 1})); - assert_eq!(values[1], serde_json::json!({"v": 2})); + // First value delivered cleanly; the Lagged(7) between the two folds + // into the second value's `skipped`, not swallowed. + assert_eq!(values[0], (serde_json::json!({"v": 1}), 0)); + assert_eq!(values[1], (serde_json::json!({"v": 2}), 7)); } } diff --git a/aimdb-core/src/session/aimx/dispatch.rs b/aimdb-core/src/session/aimx/dispatch.rs index d6c75ddb..5d7e3a47 100644 --- a/aimdb-core/src/session/aimx/dispatch.rs +++ b/aimdb-core/src/session/aimx/dispatch.rs @@ -111,8 +111,9 @@ impl Session for AimxSession { // Exact-topic fast path: resolve one key, untagged updates. let stream = crate::remote::stream::stream_record_updates(&self.db, topic) .map_err(map_db_err)?; - Ok(Box::pin(stream.map(|v| SubUpdate::new(to_payload(&v)))) - as BoxStream<'static, SubUpdate>) + Ok(Box::pin( + stream.map(|(v, skipped)| SubUpdate::new(to_payload(&v)).with_skipped(skipped)), + ) as BoxStream<'static, SubUpdate>) }) } @@ -174,9 +175,9 @@ impl AimxSession { match crate::remote::stream::stream_record_updates(&self.db, &key) { Ok(stream) => { let tag: Arc = Arc::from(key.as_str()); - streams.push(Box::pin( - stream.map(move |v| SubUpdate::tagged(tag.clone(), to_payload(&v))), - )); + streams.push(Box::pin(stream.map(move |(v, skipped)| { + SubUpdate::tagged(tag.clone(), to_payload(&v)).with_skipped(skipped) + }))); } Err(_e) => { log_warn!( diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index 5b1c2cf9..098123b0 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -29,6 +29,13 @@ use super::{ use crate::router::RouterBuilder; use crate::AimDb; +/// Capacity of a subscription's client-side event sink. Bounded (was +/// `unbounded`) so a slow consumer can't grow memory without limit under a fast +/// wildcard set; matches the server's per-connection `EVENT_BUFFER`. On overflow +/// the run loop drops the event and lets the loss surface as a `seq` gap +/// (`SubUpdate::skipped`) on the next delivered update. +const SUBSCRIBE_CHANNEL_CAP: usize = 256; + /// Client engine knobs. Durations are in **milliseconds** so the engine stays /// `no_std`-clean; plain milliseconds turned into `core::time::Duration` for the clock. #[derive(Debug, Clone)] @@ -150,7 +157,8 @@ impl ClientHandle { &self, topic: impl Into, ) -> Result>, RpcError> { - let (events, rx) = async_channel::unbounded::>(); + let (events, rx) = + async_channel::bounded::>(SUBSCRIBE_CHANNEL_CAP); self.enqueue(ClientCmd::Subscribe { topic: topic.into(), events, @@ -326,6 +334,11 @@ where // sub-id → event sink. The sub-id is `id.to_string()` of the opening // request, matching the server's derivation so `Event.sub` routes back. let mut subs: HashMap>> = HashMap::new(); + // sub-id → last *delivered* wire `seq`. Baseline is 0, so the server's first + // seq (`1`) is the expected first delivery; any shortfall is loss. Not + // advanced on a locally-dropped event, so a full-channel drop also surfaces + // as a gap on the next delivery. + let mut last_seq: HashMap = HashMap::new(); let mut out = Vec::new(); let keepalive_ms = config.keepalive_interval; // Keepalive is deadline-based: activity only records a timestamp (one dyn @@ -401,26 +414,45 @@ where // stream end instead of re-subscribing forever. if let Some(tx) = subs.remove(&id.to_string()) { let _ = tx.try_send(Err(err)); + last_seq.remove(&id.to_string()); } } } Ok(Outbound::Event { sub, - seq: _, + seq, topic, data, }) => { - let dead = match subs.get(sub) { - Some(tx) => tx - .try_send(Ok(SubUpdate { - topic: topic.map(Arc::from), - data, - })) - .is_err(), - None => false, // late event for a dropped sub — ignore - }; - if dead { - subs.remove(sub); + // Loss = the shortfall between the last delivered seq and + // this one. Server `seq` counts true production (buffer lag + // folded in via `+= skipped + 1`, funnel drops bump then + // drop), so one delta captures every loss point, plus any + // prior local full-channel drop that left `last_seq` behind. + let prev = last_seq.get(sub).copied().unwrap_or(0); + let skipped = seq.saturating_sub(prev + 1); + // `None` here is a late event for a dropped sub — ignore. + if let Some(tx) = subs.get(sub) { + let update = SubUpdate { + topic: topic.map(Arc::from), + data, + skipped, + }; + match tx.try_send(Ok(update)) { + // Delivered — advance the per-sub cursor. + Ok(()) => { + last_seq.insert(sub.to_string(), seq); + } + // Slow consumer: drop this event but leave + // `last_seq` so the shortfall folds into the next + // delivered update's `skipped`. + Err(e) if e.is_full() => {} + // Receiver gone — the sub was dropped; prune it. + Err(_) => { + subs.remove(sub); + last_seq.remove(sub); + } + } } } Ok(Outbound::Snapshot { sub, topic, data }) => { diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index bfea43b8..e780cb23 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -78,12 +78,18 @@ pub struct SubUpdate { pub topic: Option>, /// The serialized record value. pub data: Payload, + /// Count of skipped records + pub skipped: u64, } impl SubUpdate { /// An untagged update (exact-topic subscription). pub fn new(data: Payload) -> Self { - Self { topic: None, data } + Self { + topic: None, + data, + skipped: 0, + } } /// A topic-tagged update (wildcard / bus subscription). @@ -91,8 +97,17 @@ impl SubUpdate { Self { topic: Some(topic), data, + skipped: 0, } } + + /// Mark that `n` updates were lost on this subscription immediately before + /// this one (`0` leaves the update lossless). Builder form so the server's + /// subscribe-stream fold can attach a buffer's `BufferLagged` count. + pub fn with_skipped(mut self, n: u64) -> Self { + self.skipped = n; + self + } } /// Result of a transport-layer operation. diff --git a/aimdb-core/src/session/server.rs b/aimdb-core/src/session/server.rs index 76f7dde8..bbaec450 100644 --- a/aimdb-core/src/session/server.rs +++ b/aimdb-core/src/session/server.rs @@ -337,7 +337,7 @@ async fn pump_subscription( None => break, // stream exhausted }, }; - seq += 1; + seq += update.skipped + 1; // Non-blocking: drop on a full funnel (slow-client protection); only a // disconnected funnel ends the pump. match tx.try_send(SubEvent { From 2ef058f74b1ebe0079ba7e6d2393f7a922562d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sun, 19 Jul 2026 18:15:57 +0000 Subject: [PATCH 14/45] Refactor documentation and comments across multiple modules - Updated comments in `Cargo.toml` files to clarify transport features. - Revised comments in `engine.rs`, `aimx_session.rs`, and various other files to remove design references and improve clarity. - Removed outdated design references in `protocol.rs`, `query.rs`, and `codec.rs`. - Enhanced clarity in `dispatch.rs`, `builder_ext.rs`, and `http.rs` by simplifying comments. - Adjusted test descriptions in `query_handler_shape.rs` and `e2e.rs` for better understanding. - Cleaned up `ws_bridge.rs`, `registry.rs`, and `session.rs` to focus on current functionality without legacy design notes. - Updated `CHANGELOG.md` to reflect changes in `record.list` response structure and removed references to obsolete types. - Improved overall consistency and readability of comments throughout the codebase. --- aimdb-client/Cargo.toml | 4 +- aimdb-client/src/engine.rs | 4 +- aimdb-client/tests/aimx_session.rs | 2 +- aimdb-core/Cargo.toml | 9 ++-- aimdb-core/src/buffer/test_support.rs | 12 ++--- aimdb-core/src/remote/protocol.rs | 2 +- aimdb-core/src/remote/query.rs | 2 +- aimdb-core/src/session/aimx/codec.rs | 2 +- aimdb-core/src/session/aimx/dispatch.rs | 10 ++-- aimdb-core/src/session/topic_match.rs | 6 +-- aimdb-persistence/src/builder_ext.rs | 4 +- .../tests/query_handler_shape.rs | 2 +- .../tests/handshake_version.rs | 2 +- aimdb-wasm-adapter/src/bindings.rs | 13 ++--- aimdb-wasm-adapter/src/buffer.rs | 11 ++--- aimdb-wasm-adapter/src/ws_bridge.rs | 7 +-- aimdb-websocket-connector/CHANGELOG.md | 8 +++- .../examples/ws_server.rs | 4 +- .../src/client/builder.rs | 2 +- aimdb-websocket-connector/src/lib.rs | 10 ++-- .../src/server/builder.rs | 33 ++++--------- .../src/server/dispatch.rs | 34 +++++++++---- aimdb-websocket-connector/src/server/http.rs | 10 ++-- .../src/server/registry.rs | 48 ++++++++++--------- .../src/server/session.rs | 29 ++--------- aimdb-websocket-connector/src/transport.rs | 2 +- aimdb-websocket-connector/tests/e2e.rs | 26 +++++++--- .../tests/ws_roundtrip.rs | 2 +- 28 files changed, 150 insertions(+), 150 deletions(-) diff --git a/aimdb-client/Cargo.toml b/aimdb-client/Cargo.toml index 2b3b687f..20524073 100644 --- a/aimdb-client/Cargo.toml +++ b/aimdb-client/Cargo.toml @@ -29,8 +29,8 @@ aimdb-core = { version = "1.1.0", path = "../aimdb-core", features = [ "connector-session", ] } -# Transports the resolver dials over (relocated out of core in Phase 6), each -# behind its `transport-*` feature so a binary links only what it needs. +# Transports the resolver dials over, each behind its `transport-*` feature so a +# binary links only what it needs. aimdb-uds-connector = { version = "0.1.0", path = "../aimdb-uds-connector", optional = true } aimdb-serial-connector = { version = "0.1.0", path = "../aimdb-serial-connector", default-features = false, features = [ "tokio-runtime", diff --git a/aimdb-client/src/engine.rs b/aimdb-client/src/engine.rs index 49d27fdb..df013ecb 100644 --- a/aimdb-client/src/engine.rs +++ b/aimdb-client/src/engine.rs @@ -119,8 +119,8 @@ impl AimxConnection { .await .map_err(|_| ClientError::connection_failed(label, "handshake timed out"))? .map_err(|e| match e { - // The server refused our declared version (design 048 WI1) — - // report it as such, not as an unreachable-engine failure. + // The server refused our declared version — report it as + // such, not as an unreachable-engine failure. RpcError::VersionMismatch => ClientError::connection_failed( label, format!( diff --git a/aimdb-client/tests/aimx_session.rs b/aimdb-client/tests/aimx_session.rs index f60f13b7..f0e790b0 100644 --- a/aimdb-client/tests/aimx_session.rs +++ b/aimdb-client/tests/aimx_session.rs @@ -144,7 +144,7 @@ async fn aimx_roundtrip_over_uds_production_server() { /// One wildcard subscription fans in every matching record: events arrive /// tagged with the concrete record topic, and matched records with a current -/// value are delivered up front as snapshots (design 045 §3.1/§3.3). +/// value are delivered up front as snapshots. #[tokio::test] async fn wildcard_subscribe_fans_in_matching_records() { let dir = tempfile::tempdir().unwrap(); diff --git a/aimdb-core/Cargo.toml b/aimdb-core/Cargo.toml index 45353a9f..0c8927cc 100644 --- a/aimdb-core/Cargo.toml +++ b/aimdb-core/Cargo.toml @@ -47,8 +47,7 @@ remote = ["alloc", "serde_json"] # `run_session`), the proactive client (`run_client`/`pump_client`), the # `pump_sink`/`pump_source` data plane, and the generic session connectors. Engine # logic included; all compiles on `no_std + alloc`. The AimX protocol port -# (`session::aimx`) additionally needs `remote-access`. See the design doc: -# docs/design/remote-access-via-connectors.md. +# (`session::aimx`) additionally needs `remote-access`. connector-session = ["alloc"] # Observability features (available on both std/no_std) @@ -73,7 +72,7 @@ aimdb-derive = { version = "0.3.0", path = "../aimdb-derive", optional = true } # Stream trait for bidirectional connectors (minimal, no_std compatible) futures-core = { version = "0.3", default-features = false } # `async-await-macro` enables the `select_biased!` macro the runtime-neutral -# session engines use (Phase 5; it pulls in `futures-macro` + `async-await`); +# session engines use (it pulls in `futures-macro` + `async-await`); # `alloc` covers the combinators (`fuse`, `select_next_some`). All no_std- # compatible — no `std`/`tokio` enters the engine compile path. futures-util = { version = "0.3", default-features = false, features = [ @@ -97,8 +96,8 @@ serde = { workspace = true, optional = true } thiserror = { version = "2.0.16", default-features = false } anyhow = { workspace = true, optional = true } # `raw_value` lets the connector-session `EnvelopeCodec` splice an already- -# serialized record-value `Payload` into a JSON envelope without re-escaping -# (037 Decision 1). Only compiled when `serde_json` is pulled in (std / +# serialized record-value `Payload` into a JSON envelope without re-escaping. +# Only compiled when `serde_json` is pulled in (std / # remote); the no_std `connector-session` contracts build never sees it. serde_json = { workspace = true, optional = true, features = ["raw_value"] } diff --git a/aimdb-core/src/buffer/test_support.rs b/aimdb-core/src/buffer/test_support.rs index 795a4d01..15ae3923 100644 --- a/aimdb-core/src/buffer/test_support.rs +++ b/aimdb-core/src/buffer/test_support.rs @@ -3,9 +3,9 @@ //! Each adapter crate calls these from a test running under its own executor //! (`#[tokio::test]`, `block_on`, or a host `#[test]` + `block_on`), so the one //! contract is exercised against every runtime buffer implementation — the -//! buffer analogue of [`crate::executor::test_support`]. This is the structural -//! fix behind design 040: cross-runtime buffer parity moves from per-adapter -//! discipline to a single suite every adapter must pass. +//! buffer analogue of [`crate::executor::test_support`]. It moves cross-runtime +//! buffer parity from per-adapter discipline to a single suite every adapter +//! must pass. //! //! # Executor-agnostic by construction //! @@ -35,9 +35,9 @@ fn reader>(buffer: &B) -> Reader { /// Asserts the behavioral contract every `SingleLatest` buffer must hold. /// -/// Covers the design-040 fresh-subscriber rule (the divergence this suite -/// exists to prevent from recurring), plus overwrite collapsing and non- -/// destructive [`peek`](DynBuffer::peek). +/// Covers the fresh-subscriber rule (the divergence this suite exists to +/// prevent from recurring), plus overwrite collapsing and non-destructive +/// [`peek`](DynBuffer::peek). pub async fn assert_single_latest_contract(mk: F) where B: Buffer + DynBuffer, diff --git a/aimdb-core/src/remote/protocol.rs b/aimdb-core/src/remote/protocol.rs index 264abaa6..02c01bab 100644 --- a/aimdb-core/src/remote/protocol.rs +++ b/aimdb-core/src/remote/protocol.rs @@ -30,7 +30,7 @@ fn protocol_major(version: &str) -> Option<&str> { /// major (2.x → 3.0), a compatible additive change bumps only the minor. A /// missing or malformed version string is treated as **incompatible** (fail /// closed): a peer that cannot state a version it speaks is exactly the legacy -/// client this gate exists to turn away (design 048 WI1). +/// client this gate exists to turn away. pub fn version_compatible(their_version: &str) -> bool { match ( protocol_major(their_version), diff --git a/aimdb-core/src/remote/query.rs b/aimdb-core/src/remote/query.rs index f51690ea..5b757933 100644 --- a/aimdb-core/src/remote/query.rs +++ b/aimdb-core/src/remote/query.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; /// /// A boxed async function that accepts query parameters (record pattern, limit, /// start/end timestamps) and returns a JSON value with the results, shaped -/// `{"records": [QueryRecord…], "total": N}` (design 045 §3.4). +/// `{"records": [QueryRecord…], "total": N}`. pub type QueryHandlerFn = Box< dyn Fn( QueryHandlerParams, diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index 88e2f1c8..a37fd23d 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -14,7 +14,7 @@ //! concrete record on wildcard/bus subscriptions (no server-side //! `timestamp`/`dropped`). //! - snapshots carry the opening subscription's `sub` so the client demux -//! routes them like events (design 045 §3.3). +//! routes them like events. //! - the Hello/Welcome handshake is a normal `call("hello", …)`, so //! `authenticate` stays peer-only. //! diff --git a/aimdb-core/src/session/aimx/dispatch.rs b/aimdb-core/src/session/aimx/dispatch.rs index 5d7e3a47..e99ec793 100644 --- a/aimdb-core/src/session/aimx/dispatch.rs +++ b/aimdb-core/src/session/aimx/dispatch.rs @@ -155,8 +155,8 @@ impl Session for AimxSession { impl AimxSession { /// Record keys matching a wildcard pattern. The record set is frozen at - /// builder time, so matching once at subscribe time is complete (design - /// 045 §3.1 — dynamic membership waits for runtime registration). + /// builder time, so matching once at subscribe time is complete — dynamic + /// membership would require runtime registration. fn matched_keys(&self, pattern: &str) -> Vec { self.db .list_records() @@ -368,7 +368,7 @@ impl AimxSession { /// A pre-3.x — or version-less — client is refused **here** with /// [`RpcError::VersionMismatch`], rather than completing the handshake and /// letting the client trip over the new `reply`/`event` shapes on its first - /// `record.query` (design 048 WI1, decision 2: hard refusal at `hello`). + /// `record.query` — a hard refusal at `hello`. fn hello(&self, params: Value) -> Result { let version = str_field(¶ms, "version").unwrap_or_default(); if !crate::remote::version_compatible(&version) { @@ -435,8 +435,8 @@ impl AimxSession { } } -/// Serialize a JSON value into an owned record-value [`Payload`] (one serde pass -/// at the reply boundary, per doc 037 Decision 1). +/// Serialize a JSON value into an owned record-value [`Payload`] — one serde +/// pass at the reply boundary. fn to_payload(v: &Value) -> Payload { Payload::from(serde_json::to_vec(v).unwrap_or_default().as_slice()) } diff --git a/aimdb-core/src/session/topic_match.rs b/aimdb-core/src/session/topic_match.rs index 75dac76d..dbfc9cec 100644 --- a/aimdb-core/src/session/topic_match.rs +++ b/aimdb-core/src/session/topic_match.rs @@ -115,9 +115,9 @@ mod tests { #[test] fn star_matches_dotted_key_below_top_level() { - // The WI3 3a regression: a single-segment-below wildcard must match a - // dot-separated key. `temp.*` matches `temp.vienna`, not the old bug - // where `/`-splitting compared the literals `"temp.*"` and `"temp.vienna"`. + // A single-segment-below wildcard must match a dot-separated key: + // `temp.*` matches `temp.vienna`, not the old bug where `/`-splitting + // compared the literals `"temp.*"` and `"temp.vienna"`. assert!(topic_matches("temp.*", "temp.vienna")); assert!(topic_matches("temp.*", "temp.berlin")); assert!(!topic_matches("temp.*", "temp")); diff --git a/aimdb-persistence/src/builder_ext.rs b/aimdb-persistence/src/builder_ext.rs index 6f394d23..dfb2eb03 100644 --- a/aimdb-persistence/src/builder_ext.rs +++ b/aimdb-persistence/src/builder_ext.rs @@ -55,8 +55,8 @@ impl AimDbBuilderPersistExt for AimDbBuilder { }); // Register a QueryHandlerFn so AimX record.query can delegate to us. - // Result shape is the shared `{records, total}` vocabulary (design 045 - // §3.4) — one row type (`QueryRecord`) for every transport. + // Result shape is the shared `{records, total}` vocabulary — one row type + // (`QueryRecord`) for every transport. let query_backend = backend.clone(); let handler: QueryHandlerFn = Box::new(move |params: QueryHandlerParams| { let backend = query_backend.clone(); diff --git a/aimdb-persistence/tests/query_handler_shape.rs b/aimdb-persistence/tests/query_handler_shape.rs index 4b1f7458..5119a1e0 100644 --- a/aimdb-persistence/tests/query_handler_shape.rs +++ b/aimdb-persistence/tests/query_handler_shape.rs @@ -1,6 +1,6 @@ //! The `QueryHandlerFn` registered by `with_persistence` produces the shared //! `record.query` result shape — `{"records": [{topic, payload, ts}, …], -//! "total": N}`, rows sorted by `ts` ascending (design 045 §3.4). +//! "total": N}`, rows sorted by `ts` ascending. use std::sync::Arc; use std::time::Duration; diff --git a/aimdb-uds-connector/tests/handshake_version.rs b/aimdb-uds-connector/tests/handshake_version.rs index 337ae4de..f15c99d1 100644 --- a/aimdb-uds-connector/tests/handshake_version.rs +++ b/aimdb-uds-connector/tests/handshake_version.rs @@ -1,4 +1,4 @@ -//! End-to-end handshake version gate (design 048 WI1). +//! End-to-end handshake version gate. //! //! The AimX server must refuse a client whose declared protocol version is //! major-incompatible — or absent — rather than completing `hello` into a diff --git a/aimdb-wasm-adapter/src/bindings.rs b/aimdb-wasm-adapter/src/bindings.rs index 5d98363f..fc85645d 100644 --- a/aimdb-wasm-adapter/src/bindings.rs +++ b/aimdb-wasm-adapter/src/bindings.rs @@ -248,9 +248,10 @@ impl WasmDb { /// Discover topics served at `url` without building a full database. /// - /// Opens a one-shot WebSocket, sends an AimX `record.list` request, and resolves with - /// `TopicInfo[]` once the server responds. Rejects after 30 s if no - /// response arrives, or immediately on connection error. + /// Opens a one-shot WebSocket, sends an AimX `record.list` request, and + /// resolves with one record-metadata object per record once the server + /// responds. Rejects after 30 s if no response arrives, or immediately on + /// connection error. /// /// # Example (TypeScript) /// ```ts @@ -342,11 +343,11 @@ impl WasmDb { // ─── discover_impl ──────────────────────────────────────────────────────── -/// Build a one-shot WebSocket promise that resolves with `TopicInfo[]`. +/// Build a one-shot WebSocket promise that resolves with the `record.list` rows. /// /// Speaks one AimX exchange via the shared [`AimxCodec`]: a `record.list` -/// request (`id` 1) whose reply carries `[{name, schema_type, entity}, …]` -/// (design 045 §2.1 — discovery is a plain `record.list` request). +/// request (`id` 1) whose reply carries the record-metadata rows, forwarded to +/// JS unchanged. /// /// Each callback pair (resolve, reject) is stored in an `Rc>` /// so that whichever event fires first wins and subsequent events are no-ops. diff --git a/aimdb-wasm-adapter/src/buffer.rs b/aimdb-wasm-adapter/src/buffer.rs index 79a53aa0..9b3d86ac 100644 --- a/aimdb-wasm-adapter/src/buffer.rs +++ b/aimdb-wasm-adapter/src/buffer.rs @@ -481,10 +481,9 @@ mod tests { } // ── SingleLatest fresh-subscriber regression tests ────────────────────── - // Ported from the tokio adapter (`test_watch_fresh_subscriber_*`). Before - // the design-040 fix these all failed on WASM: a fresh subscriber snapshotted - // the current version and so never observed a value published before it - // subscribed. + // Ported from the tokio adapter (`test_watch_fresh_subscriber_*`). These + // used to fail on WASM: a fresh subscriber snapshotted the current version + // and so never observed a value published before it subscribed. #[test] fn test_single_latest_fresh_subscriber_sees_current_value() { @@ -558,8 +557,8 @@ mod tests { // ======================================================================== // peek() Tests — non-destructive buffer-native reads // - // Mirrors the tokio adapter's `peek_tests`; before design 040 WASM had no - // `peek()` override at all, so AimX `record.get` returned `None` on WASM. + // Mirrors the tokio adapter's `peek_tests`; WASM previously had no `peek()` + // override at all, so AimX `record.get` returned `None` on WASM. // ======================================================================== mod peek_tests { diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index 84333f6b..e00e7985 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -4,7 +4,7 @@ //! [`Connection`]/[`Dialer`] pair drives [`run_client`] with the AimX codec //! ([`AimxCodec`]), so reply/subscription correlation, reconnect backoff, //! keepalive, and the offline queue all live in -//! `aimdb-core/src/session/client.rs` — exactly once (design 045 Phase 3). +//! `aimdb-core/src/session/client.rs` — exactly once. //! This module only adapts the JS-event-driven socket to the engine's async //! `recv`/`send` interface and maps incoming updates to local buffer pushes. //! @@ -427,9 +427,10 @@ impl WsBridge { self.call_as_promise("record.query", params) } - /// List all topics served by the endpoint (AimX `record.list`). + /// List all records served by the endpoint (AimX `record.list`). /// - /// Returns a `Promise` that resolves with `TopicInfo[]`. + /// Returns a `Promise` that resolves with one record-metadata object + /// per record. /// /// ```ts /// const topics = await bridge.listTopics(); diff --git a/aimdb-websocket-connector/CHANGELOG.md b/aimdb-websocket-connector/CHANGELOG.md index 7dedb374..64b5ad73 100644 --- a/aimdb-websocket-connector/CHANGELOG.md +++ b/aimdb-websocket-connector/CHANGELOG.md @@ -21,8 +21,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `QueryHandler` returns the shared `aimdb_core::remote::QueryRecord` rows; without a plugged-in handler the dispatch now falls back to the `QueryHandlerFn` registered by `aimdb-persistence::with_persistence` - (`NoQuery` is gone). `record.list` replies with `{name, schema_type, - entity}` rows (`TopicInfo` now lives in this crate and is serialize-only). + (`NoQuery` is gone). `record.list` replies with core's shared + `aimdb_core::remote::RecordMetadata` rows — the same shape every transport + serves, keyed by `record_key` — with the data-contract `schema_type` the + connector resolves stamped in. The connector-only `TopicInfo` row type and + its topic-scoped `{name, schema_type, entity}` shape are gone; `record.list` + now enumerates every record, not only WS-outbound topics. - **`with_raw_payload` removed** — its purpose was bypassing the ws `Data` envelope; under AimX the envelope is the protocol. - **`SnapshotProvider::snapshot(topic)` became `snapshots(pattern)`**, diff --git a/aimdb-websocket-connector/examples/ws_server.rs b/aimdb-websocket-connector/examples/ws_server.rs index 988ed139..ebd1bd65 100644 --- a/aimdb-websocket-connector/examples/ws_server.rs +++ b/aimdb-websocket-connector/examples/ws_server.rs @@ -1,5 +1,5 @@ -//! Minimal runnable WebSocket **server** demo (doc 039-validation Layer 5) — the -//! first real consumer of the connector and a manual-smoke vehicle. +//! Minimal runnable WebSocket **server** demo — the first real consumer of the +//! connector and a manual-smoke vehicle. //! //! Run: //! ```text diff --git a/aimdb-websocket-connector/src/client/builder.rs b/aimdb-websocket-connector/src/client/builder.rs index a3cd9d79..0a4dc7a7 100644 --- a/aimdb-websocket-connector/src/client/builder.rs +++ b/aimdb-websocket-connector/src/client/builder.rs @@ -138,7 +138,7 @@ impl ConnectorBuilder for WsClientConnectorBuilder { ) -> Pin>> + Send + 'a>> { Box::pin(async move { - // ── Engine config from the WS-specific knobs (doc 039 § 5) ── + // ── Engine config from the WS-specific knobs ── // Reconnect/keepalive/offline-queue are `ClientConfig`/engine // concerns; subscriptions are id-routed like every AimX transport. let config = ClientConfig { diff --git a/aimdb-websocket-connector/src/lib.rs b/aimdb-websocket-connector/src/lib.rs index c2f8534a..eddb997a 100644 --- a/aimdb-websocket-connector/src/lib.rs +++ b/aimdb-websocket-connector/src/lib.rs @@ -15,7 +15,7 @@ //! //! Both modes speak **AimX** ([`aimdb_core::session::aimx`]) — the same NDJSON //! tagged frames as the UDS/serial/TCP connectors, one frame per WS text -//! message (design 045 retired the separate ws wire protocol). +//! message. There is no separate WS wire protocol. //! //! ## Server Quick Start //! @@ -81,9 +81,9 @@ //! //! ## Wire Protocol //! -//! AimX-v2 tagged frames — see [`aimdb_core::session::aimx`] and design doc -//! 045 for the frame set (`req`/`reply`/`sub`/`subscribed`/`unsub`/`event`/ -//! `snap`/`write`/`ping`/`pong`). +//! AimX-v2 tagged frames — see [`aimdb_core::session::aimx`] for the frame set +//! (`req`/`reply`/`sub`/`subscribed`/`unsub`/`event`/`snap`/`write`/`ping`/ +//! `pong`). //! //! ## Authentication (server only) //! @@ -140,4 +140,4 @@ pub type WsClientConnector = client::WsClientConnectorBuilder; pub use aimdb_core::remote::QueryRecord; #[cfg(feature = "server")] -pub use server::session::{QueryFuture, QueryHandler, TopicInfo}; +pub use server::session::{QueryFuture, QueryHandler}; diff --git a/aimdb-websocket-connector/src/server/builder.rs b/aimdb-websocket-connector/src/server/builder.rs index bc24069d..3a42a95b 100644 --- a/aimdb-websocket-connector/src/server/builder.rs +++ b/aimdb-websocket-connector/src/server/builder.rs @@ -27,7 +27,7 @@ use aimdb_data_contracts::Streamable; use aimdb_core::{pump_sink, router::RouterBuilder, ConnectorBuilder, Dispatch}; use axum::Router as AxumRouter; -use aimdb_core::{topic_leaf, topic_matches}; +use aimdb_core::topic_matches; use super::{ auth::{AuthHandler, DynAuthHandler, NoAuth}, @@ -36,7 +36,7 @@ use super::{ dispatch::WsDispatch, http::{build_server_future, ServerState}, registry::StreamableRegistry, - session::{NoSnapshot, QueryHandler, SnapshotProvider, TopicInfo}, + session::{NoSnapshot, QueryHandler, SnapshotProvider}, }; // ════════════════════════════════════════════════════════════════════ @@ -280,28 +280,11 @@ impl ConnectorBuilder for WebSocketConnectorBuilder { None => Arc::new(NoSnapshot), }; - // ── Known topics (for list_topics responses) ────────── - // Use the registered streamable types to resolve TypeId → schema name. - let topic_type_ids = db.collect_outbound_topic_type_ids("ws"); - let known_topics: Vec = topic_type_ids - .into_iter() - .map(|(topic, type_id)| { - let schema_type = self - .streamable_registry - .resolve_name(&type_id) - .map(|s| s.to_string()); - // Leaf segment of the topic ("sensors.temp.vienna" → - // "vienna"). The server owns the - // naming convention — clients receive the entity as a - // first-class field and never parse topics. - let entity = Some(topic_leaf(&topic).to_string()); - TopicInfo { - name: topic, - schema_type, - entity, - } - }) - .collect(); + // ── Schema names for record.list enrichment ─────────── + // Core builds each `record.list` row but can't map a `TypeId` to a + // data-contract schema name; the connector's registry can, so hand + // the dispatch the name lookup to stamp onto core's rows. + let schema_by_type = Arc::new(self.streamable_registry.schema_by_type_id()); // ── Shared dispatch (one Arc per server) ─── let dispatch: Arc = Arc::new(WsDispatch { @@ -310,7 +293,7 @@ impl ConnectorBuilder for WebSocketConnectorBuilder { snapshot_provider, query_handler: self.query_handler.clone(), router: router.clone(), - known_topics: Arc::new(known_topics), + schema_by_type, auth: self.auth.clone(), late_join: self.late_join, runtime_ctx: db.runtime_ctx(), diff --git a/aimdb-websocket-connector/src/server/dispatch.rs b/aimdb-websocket-connector/src/server/dispatch.rs index 1af5cd40..0c62598d 100644 --- a/aimdb-websocket-connector/src/server/dispatch.rs +++ b/aimdb-websocket-connector/src/server/dispatch.rs @@ -9,10 +9,11 @@ //! The wire is AimX ([`aimdb_core::session::aimx::AimxCodec`]); this dispatch //! only supplies WS-specific semantics on top of the shared vocabulary: //! HTTP-upgrade auth, the bus-backed subscribe, and the `record.query` / -//! `record.list` calls (design 045 §3.4). The `Subscribed` ack and late-join -//! `Snapshot`s are engine emissions; this session only supplies the snapshot -//! bytes and the subscription stream. +//! `record.list` calls. The `Subscribed` ack and late-join `Snapshot`s are +//! engine emissions; this session only supplies the snapshot bytes and the +//! subscription stream. +use std::collections::HashMap; use std::sync::Arc; use aimdb_core::remote::{QueryHandlerFn, QueryHandlerParams}; @@ -25,7 +26,7 @@ use serde_json::Value; use super::{ auth::{AuthHandler, ClientId, ClientInfo, Permissions}, client_manager::{ClientManager, ConnectionGuard}, - session::{QueryHandler, Router, SnapshotProvider, TopicInfo}, + session::{QueryHandler, Router, SnapshotProvider}, }; /// The shared WS dispatch — one `Arc` per server. @@ -37,7 +38,9 @@ pub struct WsDispatch { pub(crate) snapshot_provider: Arc, pub(crate) query_handler: Option>, pub(crate) router: Arc, - pub(crate) known_topics: Arc>, + /// Record `type_id` string → data-contract schema name, used to stamp + /// `schema_type` onto the `record.list` rows core hands back. + pub(crate) schema_by_type: Arc>, pub(crate) auth: Arc, pub(crate) late_join: bool, pub(crate) runtime_ctx: aimdb_core::RuntimeContext, @@ -74,7 +77,7 @@ impl Dispatch for WsDispatch { snapshot_provider: self.snapshot_provider.clone(), query_handler: self.query_handler.clone(), router: self.router.clone(), - known_topics: self.known_topics.clone(), + schema_by_type: self.schema_by_type.clone(), auth: self.auth.clone(), late_join: self.late_join, runtime_ctx: self.runtime_ctx.clone(), @@ -91,7 +94,7 @@ struct WsSession { snapshot_provider: Arc, query_handler: Option>, router: Arc, - known_topics: Arc>, + schema_by_type: Arc>, auth: Arc, late_join: bool, runtime_ctx: aimdb_core::RuntimeContext, @@ -108,7 +111,20 @@ impl Session for WsSession { ) -> BoxFut<'a, Result> { Box::pin(async move { let value = match method { - "record.list" => serde_json::json!(*self.known_topics), + "record.list" => { + // Same `RecordMetadata` rows core serves over every other + // transport; the connector only fills in the schema name core + // can't resolve. + let mut records = self.db.list_records(); + for record in &mut records { + if record.schema_type.is_none() { + if let Some(name) = self.schema_by_type.get(&record.type_id) { + record.schema_type = Some(name.clone()); + } + } + } + serde_json::json!(records) + } "record.query" => { let params: Value = serde_json::from_slice(¶ms).unwrap_or(Value::Null); self.record_query(params).await? @@ -164,7 +180,7 @@ impl Session for WsSession { impl WsSession { /// `record.query` with the shared `{name, limit, start, end}` params and - /// `{records, total}` result (design 045 §3.4): a plugged-in + /// `{records, total}` result: a plugged-in /// [`QueryHandler`] wins; otherwise delegate to the Extensions-registered /// `QueryHandlerFn` (`with_persistence`); neither → `NotFound`. async fn record_query(&self, params: Value) -> Result { diff --git a/aimdb-websocket-connector/src/server/http.rs b/aimdb-websocket-connector/src/server/http.rs index b40e53c4..93fa3ffe 100644 --- a/aimdb-websocket-connector/src/server/http.rs +++ b/aimdb-websocket-connector/src/server/http.rs @@ -42,7 +42,7 @@ use super::{ /// State shared across upgrade/health handlers. The per-connection session engine /// (`run_session`) is driven from [`ws_upgrade_handler`]; only the *accept* loop -/// stays axum's (Option A, doc 039 § 6). +/// stays axum's. #[derive(Clone)] pub(crate) struct ServerState { /// Shared application dispatch (one `Arc` per server). @@ -149,7 +149,7 @@ async fn ws_upgrade_handler( }; // Resolve identity synchronously, before the upgrade, and carry it into the - // engine via `PeerInfo::ext` (WS-style `reads_hello:false`, doc 039 § 4). + // engine via `PeerInfo::ext` (WS-style `reads_hello:false`). let id = state.client_mgr.next_client_id(); let info = ClientInfo { id, @@ -168,7 +168,7 @@ async fn ws_upgrade_handler( let auto_subscribe = state.auto_subscribe.clone(); let config = SessionConfig { limits: SessionLimits { - max_connections: usize::MAX, // axum owns the accept loop (Option A) + max_connections: usize::MAX, // axum owns the accept loop max_subs_per_connection: state.max_subs_per_connection, }, reads_hello: false, @@ -179,8 +179,8 @@ async fn ws_upgrade_handler( let peer = PeerInfo::default().with_ext(Arc::new(info)); let conn: Box = Box::new(WsServerConnection::new(socket, peer, &auto_subscribe)); - // The shared AimX codec + run_session drive this socket (doc 045); - // each codec blob rides as one WS text frame. + // The shared AimX codec + run_session drive this socket; each codec blob + // rides as one WS text frame. run_session(conn, &AimxCodec, dispatch.as_ref(), &config).await; }) .into_response() diff --git a/aimdb-websocket-connector/src/server/registry.rs b/aimdb-websocket-connector/src/server/registry.rs index 165918cd..d9743c0f 100644 --- a/aimdb-websocket-connector/src/server/registry.rs +++ b/aimdb-websocket-connector/src/server/registry.rs @@ -1,10 +1,10 @@ //! Schema-name registry for [`Streamable`] types. //! //! Built incrementally via [`StreamableRegistry::register::()`] at connector -//! construction time. It exists only to answer `list_topics` (resolve an -//! outbound topic's `TypeId` → schema name) and to reject schema-name -//! collisions. The actual record (de)serialization lives in the link routes' -//! serializer/deserializer, not here. +//! construction time. It resolves a record's `TypeId` → schema name so the +//! dispatch can stamp schema names onto `record.list` rows, and rejects +//! schema-name collisions. The actual record (de)serialization lives in the link +//! routes' serializer/deserializer, not here. use std::any::TypeId; use std::collections::HashMap; @@ -15,7 +15,7 @@ use aimdb_data_contracts::Streamable; pub(crate) struct StreamableRegistry { /// Schema name → `TypeId` (collision detection on `register`). name_to_type_id: HashMap<&'static str, TypeId>, - /// `TypeId` → schema name (outbound topic → schema for `list_topics`). + /// `TypeId` → schema name (resolved onto `record.list` rows). type_id_to_name: HashMap, } @@ -47,9 +47,14 @@ impl StreamableRegistry { Ok(()) } - /// Resolve a `TypeId` to its registered schema name. - pub fn resolve_name(&self, type_id: &TypeId) -> Option<&'static str> { - self.type_id_to_name.get(type_id).copied() + /// Every registered schema keyed by the record's `type_id` string (matching + /// `RecordMetadata::type_id`), so the dispatch can stamp schema names onto + /// the `record.list` rows it gets back from core. + pub fn schema_by_type_id(&self) -> HashMap { + self.type_id_to_name + .iter() + .map(|(type_id, name)| (format!("{type_id:?}"), name.to_string())) + .collect() } } @@ -59,6 +64,14 @@ mod tests { use aimdb_data_contracts::SchemaType; use serde::{Deserialize, Serialize}; + /// Look up a type's resolved schema name via the public map, keyed the same + /// way `RecordMetadata::type_id` is rendered. + fn resolved(reg: &StreamableRegistry) -> Option { + reg.schema_by_type_id() + .get(&format!("{:?}", TypeId::of::())) + .cloned() + } + #[derive(Clone, Debug, Serialize, Deserialize)] struct TestSensor { value: f32, @@ -86,16 +99,13 @@ mod tests { fn register_and_resolve_name() { let mut reg = StreamableRegistry::new(); reg.register::().unwrap(); - assert_eq!( - reg.resolve_name(&TypeId::of::()), - Some("test_sensor") - ); + assert_eq!(resolved::(®).as_deref(), Some("test_sensor")); } #[test] fn unknown_type_resolves_to_none() { let reg = StreamableRegistry::new(); - assert_eq!(reg.resolve_name(&TypeId::of::()), None); + assert_eq!(resolved::(®), None); } #[test] @@ -103,10 +113,7 @@ mod tests { let mut reg = StreamableRegistry::new(); reg.register::().unwrap(); reg.register::().unwrap(); - assert_eq!( - reg.resolve_name(&TypeId::of::()), - Some("test_sensor") - ); + assert_eq!(resolved::(®).as_deref(), Some("test_sensor")); } #[test] @@ -135,12 +142,9 @@ mod tests { let mut reg = StreamableRegistry::new(); reg.register::().unwrap(); reg.register::().unwrap(); + assert_eq!(resolved::(®).as_deref(), Some("test_sensor")); assert_eq!( - reg.resolve_name(&TypeId::of::()), - Some("test_sensor") - ); - assert_eq!( - reg.resolve_name(&TypeId::of::()), + resolved::(®).as_deref(), Some("test_actuator") ); } diff --git a/aimdb-websocket-connector/src/server/session.rs b/aimdb-websocket-connector/src/server/session.rs index ccf574ff..1f23a0b5 100644 --- a/aimdb-websocket-connector/src/server/session.rs +++ b/aimdb-websocket-connector/src/server/session.rs @@ -9,38 +9,19 @@ //! handler the dispatch falls back to the `QueryHandlerFn` that //! `aimdb-persistence::with_persistence` registers in Extensions); //! - [`SnapshotProvider`] — supplies the late-join current values for a -//! subscription pattern; -//! - [`TopicInfo`] — one `record.list` result row (topic name + schema/entity). +//! subscription pattern. +//! +//! `record.list` rows are core's [`RecordMetadata`](aimdb_core::remote::RecordMetadata), +//! shared with every other transport; the dispatch only stamps in the schema +//! name core can't resolve. use core::future::Future; use core::pin::Pin; -use serde::Serialize; - pub use aimdb_core::remote::QueryRecord; // Re-export so the builder/dispatch can use it easily. pub use aimdb_core::router::Router; -// ════════════════════════════════════════════════════════════════════ -// Topic metadata (record.list result rows) -// ════════════════════════════════════════════════════════════════════ - -/// Metadata for a single outbound topic served by a WebSocket endpoint — one -/// row of the `record.list` reply. -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct TopicInfo { - /// Record key / topic name (e.g. `"temp.vienna"`). - pub name: String, - /// Schema type name (e.g. `"temperature"`), if known by the server. - #[serde(skip_serializing_if = "Option::is_none")] - pub schema_type: Option, - /// Entity / node identifier (e.g. `"vienna"`), extracted server-side from the - /// topic name. The server is the authority on naming conventions — clients - /// should use this field directly rather than parsing the topic name. - #[serde(skip_serializing_if = "Option::is_none")] - pub entity: Option, -} - // ════════════════════════════════════════════════════════════════════ // Query handler // ════════════════════════════════════════════════════════════════════ diff --git a/aimdb-websocket-connector/src/transport.rs b/aimdb-websocket-connector/src/transport.rs index 6dcb67d0..14fc3a19 100644 --- a/aimdb-websocket-connector/src/transport.rs +++ b/aimdb-websocket-connector/src/transport.rs @@ -32,7 +32,7 @@ impl WsServerConnection { /// message. Their ids count down from `u64::MAX` so they cannot collide /// with client-chosen ids (engine clients allocate from 1 upward); the /// resulting events carry these server-side sub ids, so engine-demuxed - /// clients should subscribe explicitly instead (design 045 §3.6). + /// clients should subscribe explicitly instead. pub fn new(ws: WebSocket, peer: PeerInfo, auto_subscribe: &[String]) -> Self { let pending = auto_subscribe .iter() diff --git a/aimdb-websocket-connector/tests/e2e.rs b/aimdb-websocket-connector/tests/e2e.rs index 220a4358..2b55528a 100644 --- a/aimdb-websocket-connector/tests/e2e.rs +++ b/aimdb-websocket-connector/tests/e2e.rs @@ -10,7 +10,7 @@ //! //! The parity block at the bottom locks the AimX WS wire to the semantics the //! retired ws-protocol offered (subscribe ack, wildcard fan-out, late-join -//! snapshot, query, list — design 045 §2.1). +//! snapshot, query, list). //! //! Needs both halves (`server` + `client`); compiles away otherwise. @@ -338,7 +338,7 @@ async fn server_query_and_record_list() { .with_serializer(|_ctx, t: &Temp| Ok(serde_json::to_vec(t).unwrap())) .finish(); }); - let (_db, runner) = sb.build().await.expect("build db"); + let (db, runner) = sb.build().await.expect("build db"); tokio::spawn(runner.run()); wait_for_listen(addr).await; @@ -359,7 +359,9 @@ async fn server_query_and_record_list() { json!([{"topic":"temp","payload":21.0,"ts":7}]) ); - // record.list returns `{name, schema_type, entity}` rows. + // record.list returns core's shared `RecordMetadata` rows — the same shape + // every transport serves — with the schema name the connector resolved + // stamped in. ws_send( &mut c, json!({"t":"req","id":11,"method":"record.list","params":null}), @@ -367,10 +369,20 @@ async fn server_query_and_record_list() { .await; let reply = ws_recv_tag(&mut c, "reply").await; assert_eq!(reply["id"], 11); - assert_eq!( - reply["ok"], - json!([{"name":"temp","schema_type":"temperature","entity":"temp"}]) - ); + let rows = reply["ok"].as_array().expect("record.list array"); + assert_eq!(rows.len(), 1); + let row = &rows[0]; + assert_eq!(row["record_key"], "temp"); + assert_eq!(row["entity"], "temp"); + assert_eq!(row["buffer_type"], "single_latest"); + assert_eq!(row["schema_type"], "temperature"); + + // The row is byte-for-byte what core produces for the same record, aside + // from the schema name only the connector can resolve. + let core_rows = serde_json::to_value(db.list_records()).unwrap(); + let mut ws_row = row.clone(); + ws_row.as_object_mut().unwrap().remove("schema_type"); + assert_eq!(ws_row, core_rows[0]); } #[tokio::test] diff --git a/aimdb-websocket-connector/tests/ws_roundtrip.rs b/aimdb-websocket-connector/tests/ws_roundtrip.rs index c2ce13ee..4de1a89a 100644 --- a/aimdb-websocket-connector/tests/ws_roundtrip.rs +++ b/aimdb-websocket-connector/tests/ws_roundtrip.rs @@ -1,4 +1,4 @@ -//! Layer 1.3 (doc 039-validation) — full **AimDB ↔ AimDB over a real WebSocket**. +//! Full **AimDB ↔ AimDB over a real WebSocket**. //! //! A server `AimDb` (served by `WebSocketConnector`) and a client `AimDb` (whose //! records carry `ws-client://` links via `WsClientConnector`). This exercises the From c06b07e53e1746734f7d4937ebc3034499007508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sun, 19 Jul 2026 18:24:05 +0000 Subject: [PATCH 15/45] feat: remove collect_outbound_topic_type_ids method from AimDb --- aimdb-core/src/builder.rs | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/aimdb-core/src/builder.rs b/aimdb-core/src/builder.rs index bde5e49a..80c38817 100644 --- a/aimdb-core/src/builder.rs +++ b/aimdb-core/src/builder.rs @@ -1132,30 +1132,6 @@ impl AimDb { routes } - /// Collect `(topic, TypeId)` pairs for all outbound routes matching `scheme`. - /// - /// Complements [`collect_outbound_routes`](Self::collect_outbound_routes) when - /// callers need to know the concrete record type behind each outbound topic - /// (e.g. to resolve a schema name for discovery responses). - /// - /// The returned TypeId is the `TypeId::of::()` for the record type `T` - /// that was used in the corresponding `configure::()` call. - pub fn collect_outbound_topic_type_ids(&self, scheme: &str) -> Vec<(String, TypeId)> { - let mut result = Vec::new(); - - for entry in self.inner.storages.iter() { - let type_id = entry.type_id; - - for link in entry.record.outbound_connectors() { - if link.url.scheme() != scheme { - continue; - } - result.push((link.url.resource_id().to_string(), type_id)); - } - } - - result - } /// Collects outbound routes for a specific protocol scheme /// From 1714e62c2373e67c7ed85a23e2c4676347a94994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 20 Jul 2026 19:36:18 +0000 Subject: [PATCH 16/45] feat: add fan-out encoding benchmarks for allocation and latency comparisons --- aimdb-bench/Cargo.toml | 10 + aimdb-bench/benches/b0_alloc_fanout_encode.rs | 209 ++++++++++++ aimdb-bench/benches/b1_b2_fanout_encode.rs | 113 ++++++ aimdb-bench/src/fanout_encode.rs | 321 ++++++++++++++++++ aimdb-bench/src/lib.rs | 4 + 5 files changed, 657 insertions(+) create mode 100644 aimdb-bench/benches/b0_alloc_fanout_encode.rs create mode 100644 aimdb-bench/benches/b1_b2_fanout_encode.rs create mode 100644 aimdb-bench/src/fanout_encode.rs diff --git a/aimdb-bench/Cargo.toml b/aimdb-bench/Cargo.toml index 26079d4e..df1fd3c3 100644 --- a/aimdb-bench/Cargo.toml +++ b/aimdb-bench/Cargo.toml @@ -44,6 +44,16 @@ harness = false name = "b0_alloc_linkable" harness = false +# Fan-out event-encoding microbench (design 048 WI4): naive per-subscriber +# codec vs the shared-suffix fast path, N = 1/10/100/500. +[[bench]] +name = "b0_alloc_fanout_encode" +harness = false + +[[bench]] +name = "b1_b2_fanout_encode" +harness = false + [features] default = ["std"] # Gates `profiles`/`reports`/`harness` and their criterion/serde_json/ diff --git a/aimdb-bench/benches/b0_alloc_fanout_encode.rs b/aimdb-bench/benches/b0_alloc_fanout_encode.rs new file mode 100644 index 00000000..efa1914e --- /dev/null +++ b/aimdb-bench/benches/b0_alloc_fanout_encode.rs @@ -0,0 +1,209 @@ +//! B0 fan-out encoding allocation comparison — design 048 WI4 (H-C). +//! +//! Allocation counts are deterministic, so this is the most reproducible +//! signal for whether the shared-suffix fast path is worth its special-case +//! code. One measured "op" is one broadcast to N subscribers (N frames). +//! +//! Expectation: the naive path allocates per subscriber (a fresh output buffer +//! plus serde intermediates each `encode`); the fast path escapes `topic` once +//! per broadcast, then allocates one output buffer per subscriber. + +use std::hint::black_box; + +use aimdb_bench::alloc::{reset, snapshot}; +use aimdb_bench::fanout_encode::{ + encode_naive_into, subscribers, verify_byte_identity, PayloadShape, SharedSuffix, ValidateOnce, +}; +use aimdb_core::session::aimx::AimxCodec; + +const WARMUP_BROADCASTS: usize = 50; +const MEASURE_BROADCASTS: usize = 500; +const SUBSCRIBER_COUNTS: [usize; 4] = [1, 10, 100, 500]; +const ALLOCATOR_PROBE_BYTES: usize = 64; + +#[global_allocator] +static GLOBAL: aimdb_bench::alloc::CountingAllocator = + aimdb_bench::alloc::CountingAllocator(std::alloc::System); + +#[derive(Debug)] +struct Row { + shape: &'static str, + strategy: &'static str, + subscribers: usize, + frame_bytes: usize, + allocs_per_broadcast: f64, + allocs_per_frame: f64, + bytes_per_broadcast: f64, +} + +fn run_naive( + codec: &AimxCodec, + topic: &str, + data: &aimdb_core::session::Payload, + subs: &[aimdb_bench::fanout_encode::Subscriber], +) { + for subscriber in subs { + let mut frame = Vec::new(); + encode_naive_into( + codec, + topic, + data, + &subscriber.sub, + subscriber.seq, + &mut frame, + ); + black_box(&frame); + } +} + +fn run_validate_once( + topic: &str, + data: &aimdb_core::session::Payload, + subs: &[aimdb_bench::fanout_encode::Subscriber], +) { + let validate_once = ValidateOnce::new(data); + for subscriber in subs { + let frame = validate_once.encode_one_owned(topic, &subscriber.sub, subscriber.seq); + black_box(&frame); + } +} + +fn run_shared( + topic: &str, + data: &aimdb_core::session::Payload, + subs: &[aimdb_bench::fanout_encode::Subscriber], +) { + let shared = SharedSuffix::new(topic, data.clone()); + for subscriber in subs { + let frame = shared.encode_one_owned(&subscriber.sub, subscriber.seq); + black_box(&frame); + } +} + +fn measure(shape: PayloadShape, rows: &mut Vec) { + verify_byte_identity(shape) + .unwrap_or_else(|error| panic!("fast path must match the codec: {error}")); + + let codec = AimxCodec; + let topic = shape.topic(); + let data = shape.payload(); + let frame_bytes = SharedSuffix::new(topic, data.clone()) + .encode_one_owned("s0", 1_000_000) + .len(); + + for count in SUBSCRIBER_COUNTS { + let subs = subscribers(count); + + for _ in 0..WARMUP_BROADCASTS { + run_naive(&codec, topic, &data, &subs); + } + reset(); + for _ in 0..MEASURE_BROADCASTS { + run_naive(&codec, topic, &data, &subs); + } + let (allocs, bytes) = snapshot(); + rows.push(row(shape, "naive", count, frame_bytes, allocs, bytes)); + + for _ in 0..WARMUP_BROADCASTS { + run_validate_once(topic, &data, &subs); + } + reset(); + for _ in 0..MEASURE_BROADCASTS { + run_validate_once(topic, &data, &subs); + } + let (allocs, bytes) = snapshot(); + rows.push(row( + shape, + "validate_once", + count, + frame_bytes, + allocs, + bytes, + )); + + for _ in 0..WARMUP_BROADCASTS { + run_shared(topic, &data, &subs); + } + reset(); + for _ in 0..MEASURE_BROADCASTS { + run_shared(topic, &data, &subs); + } + let (allocs, bytes) = snapshot(); + rows.push(row( + shape, + "shared_suffix", + count, + frame_bytes, + allocs, + bytes, + )); + } +} + +fn row( + shape: PayloadShape, + strategy: &'static str, + subscribers: usize, + frame_bytes: usize, + allocs: u64, + bytes: u64, +) -> Row { + let broadcasts = MEASURE_BROADCASTS as f64; + Row { + shape: shape.name(), + strategy, + subscribers, + frame_bytes, + allocs_per_broadcast: allocs as f64 / broadcasts, + allocs_per_frame: allocs as f64 / broadcasts / subscribers as f64, + bytes_per_broadcast: bytes as f64 / broadcasts, + } +} + +/// Prove this bench binary's global allocator reaches the shared counters. +fn allocation_counter_positive_control() -> (u64, u64) { + reset(); + let mut probe = Vec::::with_capacity(black_box(ALLOCATOR_PROBE_BYTES)); + probe.push(black_box(0xA5)); + black_box(probe.as_slice()); + let measured = snapshot(); + assert!( + measured.0 >= 1, + "allocation counter missed the deliberate Vec" + ); + drop(probe); + measured +} + +fn main() { + let mut rows = Vec::new(); + measure(PayloadShape::SmallStructured, &mut rows); + measure(PayloadShape::ByteHeavy, &mut rows); + let (probe_allocs, probe_bytes) = allocation_counter_positive_control(); + + println!("=== B0 fan-out encoding allocation comparison ==="); + println!("Broadcasts per row: {MEASURE_BROADCASTS}"); + println!("Allocator probe: {probe_allocs} calls / {probe_bytes} bytes"); + println!( + "{:<18} {:<14} {:>5} {:>11} {:>16} {:>14} {:>16}", + "shape", + "strategy", + "subs", + "frame_B", + "allocs/broadcast", + "allocs/frame", + "bytes/broadcast" + ); + for row in rows { + println!( + "{:<18} {:<14} {:>5} {:>11} {:>16.1} {:>14.3} {:>16.1}", + row.shape, + row.strategy, + row.subscribers, + row.frame_bytes, + row.allocs_per_broadcast, + row.allocs_per_frame, + row.bytes_per_broadcast, + ); + } +} diff --git a/aimdb-bench/benches/b1_b2_fanout_encode.rs b/aimdb-bench/benches/b1_b2_fanout_encode.rs new file mode 100644 index 00000000..dce0590f --- /dev/null +++ b/aimdb-bench/benches/b1_b2_fanout_encode.rs @@ -0,0 +1,113 @@ +//! B1/B2 fan-out encoding — design 048 WI4 (H-B scaling, H-C payoff). +//! +//! One timed iteration is **one broadcast to N subscribers**: turning a single +//! shared record value into N per-subscriber AimX `event` frames. Compares the +//! production per-subscriber codec (`naive`) against the shared-suffix fast +//! path (`shared_suffix`) at N = 1/10/100/500 for a small and a byte-heavy +//! payload. Pure CPU/allocation — no sockets, scheduling, or backpressure. + +use std::hint::black_box; +use std::time::Duration; + +use aimdb_bench::fanout_encode::{ + encode_naive_into, subscribers, verify_byte_identity, PayloadShape, SharedSuffix, ValidateOnce, +}; +use aimdb_core::session::aimx::AimxCodec; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; + +const SUBSCRIBER_COUNTS: [usize; 4] = [1, 10, 100, 500]; + +fn bench_shape(criterion: &mut Criterion, shape: PayloadShape) { + verify_byte_identity(shape) + .unwrap_or_else(|error| panic!("fast path must match the codec: {error}")); + + let codec = AimxCodec; + let topic = shape.topic(); + let data = shape.payload(); + + // Per-frame size is identical across strategies (byte-identical frames); + // report it once, outside the timed windows. + let sample = SharedSuffix::new(topic, data.clone()).encode_one_owned("s0", 1_000_000); + eprintln!( + "=== fanout_encode {}: payload {} B, event frame {} B ===", + shape.name(), + data.len(), + sample.len(), + ); + + let mut group = criterion.benchmark_group(format!("fanout_encode/{}", shape.name())); + for count in SUBSCRIBER_COUNTS { + let subs = subscribers(count); + group.throughput(Throughput::Elements(count as u64)); + + group.bench_with_input(BenchmarkId::new("naive", count), &count, |bencher, _| { + bencher.iter(|| { + for subscriber in &subs { + let mut frame = Vec::new(); + encode_naive_into( + &codec, + topic, + &data, + &subscriber.sub, + subscriber.seq, + &mut frame, + ); + black_box(frame); + } + }); + }); + + // Improvement A alone: hoist the record-value validation out of the + // per-subscriber loop, but keep the serde frame serialization. + group.bench_with_input( + BenchmarkId::new("validate_once", count), + &count, + |bencher, _| { + bencher.iter(|| { + let validate_once = ValidateOnce::new(&data); + for subscriber in &subs { + let frame = + validate_once.encode_one_owned(topic, &subscriber.sub, subscriber.seq); + black_box(frame); + } + }); + }, + ); + + // Improvements A + B: shared-suffix fast path (no per-subscriber serde). + group.bench_with_input( + BenchmarkId::new("shared_suffix", count), + &count, + |bencher, _| { + bencher.iter(|| { + // The per-broadcast precompute (escape `topic` once) is part + // of the measured cost, so it lives inside the timed loop. + let shared = SharedSuffix::new(topic, data.clone()); + for subscriber in &subs { + let frame = shared.encode_one_owned(&subscriber.sub, subscriber.seq); + black_box(frame); + } + }); + }, + ); + } + group.finish(); +} + +fn bench_small(criterion: &mut Criterion) { + bench_shape(criterion, PayloadShape::SmallStructured); +} + +fn bench_byte_heavy(criterion: &mut Criterion) { + bench_shape(criterion, PayloadShape::ByteHeavy); +} + +criterion_group! { + name = benches; + config = Criterion::default() + .sample_size(30) + .warm_up_time(Duration::from_secs(1)) + .measurement_time(Duration::from_secs(3)); + targets = bench_small, bench_byte_heavy +} +criterion_main!(benches); diff --git a/aimdb-bench/src/fanout_encode.rs b/aimdb-bench/src/fanout_encode.rs new file mode 100644 index 00000000..0ad2a89b --- /dev/null +++ b/aimdb-bench/src/fanout_encode.rs @@ -0,0 +1,321 @@ +//! Fan-out event-encoding microbenchmark for design 048 WI4. +//! +//! Measures the per-broadcast cost of turning one shared record value into +//! `N` per-subscriber AimX `event` frames — the only work that differs +//! between the converged AimX-over-WS path and the retired `aimdb-ws-protocol` +//! path (047 §2.2, §3.1). It is a pure CPU/allocation microbenchmark: no +//! sockets, no scheduling, no `ClientManager`, no backpressure. +//! +//! Two strategies produce **byte-identical** frames: +//! +//! - [`encode_naive`] — the production `AimxCodec::encode(Outbound::Event)`, +//! re-run once per subscriber (re-escapes `topic`/`sub`, re-validates the +//! record `RawValue`, allocates serde intermediates every time). +//! - [`SharedSuffix`] — the "encode once, patch the per-subscriber fields" +//! fast path: the shared `topic` is escaped once per broadcast; per +//! subscriber only the two varying scalars (`sub`, `seq`) are formatted and +//! the shared record bytes are spliced in. +//! +//! The record payload is copied into each frame by **both** strategies +//! (a contiguous WS text frame embeds it), so this isolates the serialization +//! delta, not a copy the fast path avoids — see design 048 WI4's caveat. + +use std::io::Write as _; + +use aimdb_core::session::aimx::AimxCodec; +use aimdb_core::session::{EnvelopeCodec, Outbound, Payload}; +use serde::Serialize; +use serde_json::value::RawValue; + +/// Constant frame head up to (and including) the `seq` key. The AimX event +/// frame serializes its fields in `Frame` declaration order — `t, seq, topic, +/// sub, data` — so `seq` leads and everything after it is either shared per +/// broadcast (`topic`, `data`) or a small per-subscriber scalar (`sub`). +const EVENT_HEAD: &[u8] = br#"{"t":"event","seq":"#; + +/// A deterministic record-payload shape used by the fan-out benchmark. +/// +/// Payloads are already-serialized JSON (what a record codec hands the +/// session layer): the small shape is a compact structured object; the +/// byte-heavy shape is the JSON integer-array a byte record expands into +/// (JSON has no byte type — see design 046 §6.1), the case where per-subscriber +/// work is most likely to matter. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PayloadShape { + /// Small structured telemetry object. + SmallStructured, + /// ~1 KiB byte record expanded to a JSON integer array. + ByteHeavy, +} + +impl PayloadShape { + /// Stable order shared by benchmark tables and Criterion identifiers. + pub const ALL: [Self; 2] = [Self::SmallStructured, Self::ByteHeavy]; + + /// Stable benchmark identifier. + pub const fn name(self) -> &'static str { + match self { + Self::SmallStructured => "small_structured", + Self::ByteHeavy => "byte_heavy", + } + } + + /// The concrete record topic the broadcast fires on (shared by every + /// subscriber of one exact-topic broadcast). + pub const fn topic(self) -> &'static str { + "plant/line-7/temperature" + } + + /// Deterministic already-serialized JSON record value. + pub fn payload(self) -> Payload { + match self { + Self::SmallStructured => Payload::from( + &br#"{"sensor_id":7,"sequence":1000042,"temperature_c":23.625,"pressure_hpa":1013.25,"rssi_dbm":-67,"healthy":true}"#[..], + ), + Self::ByteHeavy => { + let mut bytes = Vec::with_capacity(4096); + bytes.push(b'['); + for index in 0_u16..1_024 { + if index != 0 { + bytes.push(b','); + } + write!(bytes, "{}", (index.wrapping_mul(31).wrapping_add(7)) % 256) + .expect("write to Vec is infallible"); + } + bytes.push(b']'); + Payload::from(bytes.as_slice()) + } + } + } +} + +/// One subscriber's per-event varying fields: its subscription id and the +/// monotonic sequence number the server would stamp for that subscription. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Subscriber { + /// Subscription routing id (serialized as a JSON string on the wire). + pub sub: String, + /// Per-subscription sequence number. + pub seq: u64, +} + +/// Build `count` deterministic subscribers. Ids are engine-style JSON-safe +/// tokens (no characters that would need escaping — asserted byte-identical to +/// the production codec by [`verify_byte_identity`]). +pub fn subscribers(count: usize) -> Vec { + (0..count) + .map(|index| Subscriber { + sub: format!("s{index}"), + seq: 1_000_000 + index as u64, + }) + .collect() +} + +/// Encode one subscriber's event frame with the production codec (the naive +/// per-subscriber baseline). Appends into `out`, which the caller supplies +/// fresh per frame (each frame is an independent buffer bound for a socket). +pub fn encode_naive(codec: &AimxCodec, topic: &str, data: &Payload, sub: &str, seq: u64) { + let mut out = Vec::new(); + encode_naive_into(codec, topic, data, sub, seq, &mut out); + // Discourage the optimizer from eliding the buffer entirely. + std::hint::black_box(out); +} + +/// As [`encode_naive`] but writes into a caller-owned buffer (used by the +/// allocation bench, which owns the buffer lifecycle). +pub fn encode_naive_into( + codec: &AimxCodec, + topic: &str, + data: &Payload, + sub: &str, + seq: u64, + out: &mut Vec, +) { + codec + .encode( + Outbound::Event { + sub, + seq, + topic: Some(topic), + data: data.clone(), + }, + out, + ) + .expect("event encode should succeed"); +} + +/// The shared-suffix fast path: precompute the per-broadcast constant parts +/// once (escaping `topic` a single time), then stamp cheap per-subscriber +/// frames. +pub struct SharedSuffix { + /// `,"topic":,"sub":` — built once, reused for every frame. + mid: Vec, + /// The shared already-serialized record value (spliced into every frame, + /// exactly as the naive path copies it per subscriber). + data: Payload, +} + +impl SharedSuffix { + /// Prepare the shared segments for one broadcast of `data` on `topic`. + /// This is the fast path's per-broadcast fixed cost — it escapes `topic` + /// exactly once, versus the naive path re-escaping it per subscriber. + pub fn new(topic: &str, data: Payload) -> Self { + let mut mid = Vec::with_capacity(topic.len() + 24); + mid.extend_from_slice(br#","topic":"#); + serde_json::to_writer(&mut mid, topic).expect("topic escapes to valid JSON"); + mid.extend_from_slice(br#","sub":"#); + Self { mid, data } + } + + /// Stamp one subscriber's frame. Byte-identical to [`encode_naive`] for + /// JSON-safe `sub` tokens (verified). Only the two varying scalars are + /// formatted per call; `topic` is not re-escaped and the record bytes are + /// spliced verbatim. + pub fn encode_one(&self, sub: &str, seq: u64, out: &mut Vec) { + out.clear(); + out.reserve(EVENT_HEAD.len() + self.mid.len() + self.data.len() + sub.len() + 24); + out.extend_from_slice(EVENT_HEAD); + write!(out, "{seq}").expect("write to Vec is infallible"); + out.extend_from_slice(&self.mid); + // `sub` is an engine-generated JSON-safe token: quote it directly + // rather than paying serde's escape machinery per subscriber. + out.push(b'"'); + out.extend_from_slice(sub.as_bytes()); + out.push(b'"'); + out.extend_from_slice(br#","data":"#); + out.extend_from_slice(&self.data); + out.push(b'}'); + } + + /// Stamp one subscriber's frame into a fresh buffer. + pub fn encode_one_owned(&self, sub: &str, seq: u64) -> Vec { + let mut out = Vec::new(); + self.encode_one(sub, seq, &mut out); + out + } +} + +/// Event frame subset mirroring `AimxCodec`'s private `Frame` in serialized +/// field order (`t, seq, topic, sub, data`). Used by [`ValidateOnce`] to run +/// the same serde frame serialization as the codec while borrowing a record +/// value that was validated once, rather than re-validating it per subscriber. +#[derive(Serialize)] +struct EventFrameRef<'a> { + t: &'a str, + seq: u64, + #[serde(skip_serializing_if = "Option::is_none")] + topic: Option<&'a str>, + sub: &'a str, + data: &'a RawValue, +} + +/// Improvement A in isolation: the production per-subscriber serde frame +/// serialization (topic re-escape + scaffolding + verbatim record splice), but +/// with the record value validated **once per broadcast** instead of once per +/// subscriber. The delta from [`encode_naive`] is exactly the cost of the +/// per-subscriber `as_raw` re-validation; the delta to [`SharedSuffix`] is the +/// serde scaffolding the fast path additionally removes. +pub struct ValidateOnce { + raw: Box, +} + +impl ValidateOnce { + /// Validate the record value a single time (the hoisted `as_raw`). + pub fn new(data: &Payload) -> Self { + let raw = serde_json::from_slice::>(data) + .expect("record payload must be one valid JSON value"); + Self { raw } + } + + /// Serialize one subscriber's frame via serde, reusing the pre-validated + /// record value. Byte-identical to [`encode_naive`]. + pub fn encode_one_owned(&self, topic: &str, sub: &str, seq: u64) -> Vec { + let frame = EventFrameRef { + t: "event", + seq, + topic: Some(topic), + sub, + data: &self.raw, + }; + serde_json::to_vec(&frame).expect("event frame serializes") + } +} + +/// Assert all three strategies reproduce the production codec's bytes for +/// `shape` across a spread of subscribers. This is what licenses treating them +/// as measuring the same output rather than comparing different frames. +pub fn verify_byte_identity(shape: PayloadShape) -> Result<(), String> { + let codec = AimxCodec; + let topic = shape.topic(); + let data = shape.payload(); + let shared = SharedSuffix::new(topic, data.clone()); + let validate_once = ValidateOnce::new(&data); + + for subscriber in subscribers(512) { + let mut expected = Vec::new(); + encode_naive_into( + &codec, + topic, + &data, + &subscriber.sub, + subscriber.seq, + &mut expected, + ); + for (strategy, actual) in [ + ( + "shared_suffix", + shared.encode_one_owned(&subscriber.sub, subscriber.seq), + ), + ( + "validate_once", + validate_once.encode_one_owned(topic, &subscriber.sub, subscriber.seq), + ), + ] { + if actual != expected { + return Err(format!( + "shape {} strategy {} sub {} seq {}: diverged from codec\n codec: {}\n {}: {}", + shape.name(), + strategy, + subscriber.sub, + subscriber.seq, + String::from_utf8_lossy(&expected), + strategy, + String::from_utf8_lossy(&actual), + )); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fast_path_is_byte_identical_to_codec() { + for shape in PayloadShape::ALL { + verify_byte_identity(shape).unwrap_or_else(|error| panic!("{error}")); + } + } + + #[test] + fn payloads_are_valid_json_and_non_empty() { + for shape in PayloadShape::ALL { + let payload = shape.payload(); + assert!(!payload.is_empty(), "{} payload empty", shape.name()); + serde_json::from_slice::(&payload) + .unwrap_or_else(|error| panic!("{} payload invalid JSON: {error}", shape.name())); + } + } + + #[test] + fn byte_heavy_is_much_larger_than_small() { + let small = PayloadShape::SmallStructured.payload().len(); + let heavy = PayloadShape::ByteHeavy.payload().len(); + assert!( + heavy > small * 8, + "expected byte-heavy ≫ small ({heavy} vs {small})" + ); + } +} diff --git a/aimdb-bench/src/lib.rs b/aimdb-bench/src/lib.rs index 46ccbd6e..d0526d1b 100644 --- a/aimdb-bench/src/lib.rs +++ b/aimdb-bench/src/lib.rs @@ -26,6 +26,8 @@ //! | `benches/b_alloc_pipeline.rs` | info | Per-message allocation (runner pipeline) | //! | `benches/b_runner_pipeline.rs` | info | Runner pipeline throughput (Criterion) | //! | `benches/b0_alloc_migration.rs` | B0 | Per-call allocation (migrate_from_bytes) | +//! | `benches/b0_alloc_fanout_encode.rs`| B0 | Fan-out event-encode allocation (048 WI4)| +//! | `benches/b1_b2_fanout_encode.rs` | B1+B2 | Fan-out event-encode latency (048 WI4) | //! //! On-target cycle profiling (B3) lives in the hardware-only //! `examples/embassy-bench-stm32h5` crate, since DWT cycle counting cannot run @@ -35,6 +37,8 @@ pub mod alloc; #[cfg(feature = "std")] +pub mod fanout_encode; +#[cfg(feature = "std")] pub mod harness; pub mod payloads; #[cfg(feature = "std")] From 548de31af027eb08f57271e8572f32593c2984f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 20 Jul 2026 19:52:47 +0000 Subject: [PATCH 17/45] feat: add fan-out socket-driven benchmark for performance measurement --- aimdb-websocket-connector/Cargo.toml | 7 + .../benches/fanout_socket.rs | 294 ++++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 aimdb-websocket-connector/benches/fanout_socket.rs diff --git a/aimdb-websocket-connector/Cargo.toml b/aimdb-websocket-connector/Cargo.toml index 733d5462..f1601ba8 100644 --- a/aimdb-websocket-connector/Cargo.toml +++ b/aimdb-websocket-connector/Cargo.toml @@ -77,3 +77,10 @@ required-features = ["server"] [[example]] name = "ws_client" required-features = ["client"] + +# H-A socket-driven fan-out benchmark (design 048 WI4). Custom timing harness; +# raw tokio-tungstenite clients (a dev-dependency) against the real server. +[[bench]] +name = "fanout_socket" +harness = false +required-features = ["server"] diff --git a/aimdb-websocket-connector/benches/fanout_socket.rs b/aimdb-websocket-connector/benches/fanout_socket.rs new file mode 100644 index 00000000..26d8b21a --- /dev/null +++ b/aimdb-websocket-connector/benches/fanout_socket.rs @@ -0,0 +1,294 @@ +//! H-A socket-driven fan-out benchmark (design 048 WI4). +//! +//! Unlike the `aimdb-bench` encoding microbench (which isolates the codec), +//! this drives the **real** converged path end-to-end: a live WebSocket server +//! (`WebSocketConnector` + `ClientManager` + `AimxCodec` + Axum/Tokio sockets) +//! fanning one record update out to N concurrently-subscribed clients. +//! +//! The clients are **raw** `tokio-tungstenite` sockets, not aimdb client +//! connectors — 500 aimdb instances would swamp the box and put the cost on the +//! client side, whereas raw sockets keep the measured work on the server's +//! fan-out path (broadcast → per-connection encode → N socket writes). +//! +//! One measured round: set the record to a fresh value, then wait until every +//! client has observed a value ≥ that round. The elapsed time is the +//! end-to-end fan-out latency of one broadcast to N subscribers. This latency +//! **includes** the client receive/parse and the driver's poll granularity — +//! it is a system measurement, not a codec measurement. +//! +//! Run (raise the fd limit — 500 clients means ~1000 sockets in-process): +//! +//! ```bash +//! bash -c 'ulimit -n 8192; cargo bench -p aimdb-websocket-connector --bench fanout_socket' +//! ``` + +use std::net::SocketAddr; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use aimdb_core::buffer::BufferCfg; +use aimdb_core::{AimDb, AimDbBuilder}; +use aimdb_tokio_adapter::{TokioAdapter, TokioRecordRegistrarExt}; +use aimdb_websocket_connector::WebSocketConnector; +use futures_util::{SinkExt, StreamExt}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use tokio::net::TcpStream; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; +use tokio::time::timeout; +use tokio_tungstenite::tungstenite::Message; + +const SUBSCRIBER_COUNTS: [usize; 4] = [1, 10, 100, 500]; +const ROUNDS: usize = 100; +const WARMUP_ROUNDS: usize = 10; +const ROUND_TIMEOUT: Duration = Duration::from_secs(15); + +/// Record whose `n` field carries the round marker; `blob` sizes the payload +/// (empty for the small shape, ~1 KiB for byte-heavy — serialized as a JSON +/// integer array, the same shape the encoding microbench used). +#[derive(Debug, Serialize, Deserialize, Clone)] +struct Rec { + n: u64, + blob: Vec, +} + +struct Shape { + name: &'static str, + blob_len: usize, +} + +const SHAPES: [Shape; 2] = [ + Shape { + name: "small", + blob_len: 0, + }, + Shape { + name: "byte_heavy", + blob_len: 1024, + }, +]; + +fn free_addr() -> SocketAddr { + std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() +} + +async fn wait_for_listen(addr: SocketAddr) { + for _ in 0..200 { + if TcpStream::connect(addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("server never bound at {addr}"); +} + +async fn spawn_server() -> (SocketAddr, Arc, JoinHandle<()>) { + let addr = free_addr(); + let mut sb = AimDbBuilder::new() + .runtime(Arc::new(TokioAdapter)) + .with_connector( + WebSocketConnector::new() + .bind(addr) + .path("/ws") + .with_late_join(true), + ); + sb.configure::("counter", |reg| { + reg.buffer(BufferCfg::SingleLatest) + .with_remote_access() + .link_to("ws://counter") + .with_serializer(|_ctx, r: &Rec| Ok(serde_json::to_vec(r).unwrap())) + .finish(); + }); + let (db, runner) = sb.build().await.expect("build server db"); + let db = Arc::new(db); + let handle = tokio::spawn(async move { + runner.run().await; + }); + wait_for_listen(addr).await; + (addr, db, handle) +} + +/// Cheap scan for the record's `"n":` marker without a full JSON parse. +fn parse_n(bytes: &[u8]) -> Option { + let needle = b"\"n\":"; + let start = bytes.windows(needle.len()).position(|w| w == needle)? + needle.len(); + let mut value: u64 = 0; + let mut any = false; + for &byte in &bytes[start..] { + if byte.is_ascii_digit() { + value = value * 10 + (byte - b'0') as u64; + any = true; + } else { + break; + } + } + any.then_some(value) +} + +fn note_seen(seen: &AtomicU64, bytes: &[u8]) { + if let Some(value) = parse_n(bytes) { + seen.fetch_max(value, Ordering::Relaxed); + } +} + +/// Spawn one raw WS client: connect, subscribe, signal ready once the +/// `subscribed` ack lands, then update `seen` with every observed marker. +fn spawn_client( + addr: SocketAddr, + id: u64, + seen: Arc, + ready: mpsc::Sender<()>, +) -> JoinHandle<()> { + tokio::spawn(async move { + let (mut ws, _) = tokio_tungstenite::connect_async(format!("ws://{addr}/ws")) + .await + .expect("client connect"); + ws.send(Message::Text( + json!({"t":"sub","id":id,"topic":"counter"}) + .to_string() + .into(), + )) + .await + .expect("send subscribe"); + + // Drain until the subscribe ack, then announce readiness. + loop { + match ws.next().await { + Some(Ok(Message::Text(text))) => { + if text.contains("\"subscribed\"") { + break; + } + note_seen(&seen, text.as_bytes()); + } + Some(Ok(Message::Binary(bin))) => note_seen(&seen, &bin), + Some(Ok(_)) => {} + _ => return, + } + } + if ready.send(()).await.is_err() { + return; + } + + // Event loop: record the highest marker seen. + loop { + match ws.next().await { + Some(Ok(Message::Text(text))) => note_seen(&seen, text.as_bytes()), + Some(Ok(Message::Binary(bin))) => note_seen(&seen, &bin), + Some(Ok(_)) => {} + _ => return, + } + } + }) +} + +async fn set_and_wait(db: &AimDb, seen: &[Arc], round: u64, blob: &[u8]) -> Duration { + let start = Instant::now(); + db.set_record_from_json("counter", json!({"n": round, "blob": blob})) + .expect("set record"); + loop { + if seen.iter().all(|s| s.load(Ordering::Relaxed) >= round) { + return start.elapsed(); + } + if start.elapsed() > ROUND_TIMEOUT { + let reached = seen + .iter() + .filter(|s| s.load(Ordering::Relaxed) >= round) + .count(); + panic!( + "round {round} timed out: {reached}/{} clients delivered", + seen.len() + ); + } + tokio::task::yield_now().await; + } +} + +struct Stats { + median_us: f64, + p99_us: f64, + per_delivery_ns: f64, + throughput_msgs_per_s: f64, +} + +async fn measure(blob_len: usize, subscribers: usize) -> Stats { + let (addr, db, server) = spawn_server().await; + + let seen: Vec> = (0..subscribers) + .map(|_| Arc::new(AtomicU64::new(0))) + .collect(); + let (ready_tx, mut ready_rx) = mpsc::channel(subscribers.max(1)); + let mut clients = Vec::with_capacity(subscribers); + for (id, seen_slot) in seen.iter().enumerate() { + clients.push(spawn_client( + addr, + id as u64, + seen_slot.clone(), + ready_tx.clone(), + )); + } + drop(ready_tx); + for _ in 0..subscribers { + timeout(Duration::from_secs(30), ready_rx.recv()) + .await + .expect("clients failed to subscribe in time") + .expect("client dropped before ready"); + } + + let blob = vec![0u8; blob_len]; + let mut round = 0u64; + for _ in 0..WARMUP_ROUNDS { + round += 1; + set_and_wait(&db, &seen, round, &blob).await; + } + + let mut latencies = Vec::with_capacity(ROUNDS); + for _ in 0..ROUNDS { + round += 1; + latencies.push(set_and_wait(&db, &seen, round, &blob).await); + } + + for client in clients { + client.abort(); + } + server.abort(); + + latencies.sort_unstable(); + let median = latencies[latencies.len() / 2]; + let p99 = latencies[(latencies.len() * 99 / 100).min(latencies.len() - 1)]; + let total: Duration = latencies.iter().sum(); + let deliveries = (latencies.len() * subscribers) as f64; + Stats { + median_us: median.as_secs_f64() * 1e6, + p99_us: p99.as_secs_f64() * 1e6, + per_delivery_ns: median.as_secs_f64() * 1e9 / subscribers as f64, + throughput_msgs_per_s: deliveries / total.as_secs_f64(), + } +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + println!("=== H-A socket-driven fan-out ({ROUNDS} rounds/config) ==="); + println!( + "{:<12} {:>5} {:>14} {:>14} {:>16} {:>16}", + "shape", "subs", "median (µs)", "p99 (µs)", "per-delivery ns", "msgs/s" + ); + for shape in SHAPES { + for subscribers in SUBSCRIBER_COUNTS { + let stats = measure(shape.blob_len, subscribers).await; + println!( + "{:<12} {:>5} {:>14.1} {:>14.1} {:>16.0} {:>16.0}", + shape.name, + subscribers, + stats.median_us, + stats.p99_us, + stats.per_delivery_ns, + stats.throughput_msgs_per_s, + ); + } + } +} From 266e34b16811db850e6d49c6289baa2a0a1665d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 20 Jul 2026 20:15:13 +0000 Subject: [PATCH 18/45] feat: add splicing method for outbound event data to optimize serialization --- aimdb-core/src/session/aimx/codec.rs | 63 ++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index a37fd23d..73a27156 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -122,6 +122,34 @@ fn write_frame(out: &mut alloc::vec::Vec, frame: &Frame<'_>) -> Result<(), C Ok(()) } +/// Encode a frame whose `data` field is already-serialized JSON (a record +/// serializer's output), splicing `data` verbatim instead of re-validating it +/// into a [`RawValue`]. Outbound event payloads are trusted-valid, so `as_raw`'s +/// O(payload) scan is redundant — and on a fan-out subscription it would run +/// once *per subscriber* over the shared payload (design 048 WI4, Improvement A). +/// +/// `frame` must carry `data = None` so serde emits every other field; the +/// spliced `data` is appended as the final member. Byte-identical to serializing +/// the same frame with `data = Some(as_raw(payload))`, since `data` is the last +/// field in declaration order. +fn write_frame_splicing_data( + out: &mut alloc::vec::Vec, + frame: &Frame<'_>, + data: &[u8], +) -> Result<(), CodecError> { + debug_assert!(frame.data.is_none(), "data must be spliced, not serialized"); + let scaffold = serde_json::to_vec(frame).map_err(|_| CodecError::Malformed)?; + // A `Frame` always serializes to a JSON object closed by `}`. + let Some((&b'}', head)) = scaffold.split_last() else { + return Err(CodecError::Malformed); + }; + out.extend_from_slice(head); + out.extend_from_slice(br#","data":"#); + out.extend_from_slice(data); + out.push(b'}'); + Ok(()) +} + impl EnvelopeCodec for AimxCodec { // --- server direction: read a request, write a reply/event ------------- fn decode(&self, frame: &[u8]) -> Result { @@ -171,13 +199,13 @@ impl EnvelopeCodec for AimxCodec { topic, data, } => { - let raw = as_raw(&data)?; let mut frame = Frame::tagged("event"); frame.sub = Some(sub); frame.seq = Some(seq); frame.topic = topic; - frame.data = Some(raw); - write_frame(out, &frame) + // `data` is spliced verbatim (Improvement A) — see + // `write_frame_splicing_data`. + write_frame_splicing_data(out, &frame, &data) } Outbound::Snapshot { sub, topic, data } => { let raw = as_raw(&data)?; @@ -428,6 +456,35 @@ mod tests { } } + /// The verbatim-splice event encode (Improvement A) must be byte-identical + /// to the previous `data = Some(as_raw(payload))` serde path — the record + /// payload is spliced unchanged as the final `data` member, in both the + /// with-topic and without-topic layouts. + #[test] + fn event_encode_splices_data_byte_for_byte() { + let with_topic = encode_outbound(Outbound::Event { + sub: "5", + seq: 9, + topic: Some("temp.vienna"), + data: payload(r#"{"c":21.5,"a":[1,2,3]}"#), + }); + assert_eq!( + with_topic, + br#"{"t":"event","seq":9,"topic":"temp.vienna","sub":"5","data":{"c":21.5,"a":[1,2,3]}}"# + ); + + let without_topic = encode_outbound(Outbound::Event { + sub: "5", + seq: 2, + topic: None, + data: payload("42"), + }); + assert_eq!( + without_topic, + br#"{"t":"event","seq":2,"sub":"5","data":42}"# + ); + } + #[test] fn snapshot_roundtrip_carries_sub_and_topic() { let frame = encode_outbound(Outbound::Snapshot { From 0fa1346ade893fef9fe69a379042573a8160a3bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Wed, 22 Jul 2026 17:23:01 +0000 Subject: [PATCH 19/45] format --- aimdb-core/src/builder.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/aimdb-core/src/builder.rs b/aimdb-core/src/builder.rs index 80c38817..1e6706f8 100644 --- a/aimdb-core/src/builder.rs +++ b/aimdb-core/src/builder.rs @@ -1132,7 +1132,6 @@ impl AimDb { routes } - /// Collects outbound routes for a specific protocol scheme /// /// Mirrors `collect_inbound_routes()` for symmetry. Iterates all records, From aba0fef3e18e2e3986b5f87d5394e4d0b6bdea1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Wed, 22 Jul 2026 17:47:33 +0000 Subject: [PATCH 20/45] feat: refine SendFuture implementation and simplify recv method in WasmWsConnection --- aimdb-wasm-adapter/src/time.rs | 22 +++++++++++++--------- aimdb-wasm-adapter/src/ws_bridge.rs | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/aimdb-wasm-adapter/src/time.rs b/aimdb-wasm-adapter/src/time.rs index ab14e78a..6396cd69 100644 --- a/aimdb-wasm-adapter/src/time.rs +++ b/aimdb-wasm-adapter/src/time.rs @@ -21,13 +21,17 @@ use core::task::{Context, Poll}; /// /// # Safety /// -/// Only ever polled on `wasm32-unknown-unknown` without the `atomics` / -/// shared-memory proposal — single-threaded by construction (see the compile -/// guard below), so the inner future is never actually sent between threads. -/// The `Send` impl is unconditional (not gated to `target_arch = "wasm32"`) so -/// this type is also `Send` in the native test lane, where the WASM bridge -/// compiles — to satisfy the same `dyn Future + Send` coercions — but never -/// runs. +/// `wasm-runtime` is a wasm32-only feature in practice: the JS-driven bridge it +/// gates (`ws_bridge`) holds `Rc`/`Closure` state that is genuinely `!Send`, so +/// the crate only compiles for `wasm32-unknown-unknown` once the feature is on +/// (the native `cargo test` lane runs `--no-default-features`, which compiles +/// this type and the bridge out entirely). That leaves +/// `wasm32-unknown-unknown` without the `atomics` / shared-memory proposal as +/// the only configuration where the `Send` impl is live — single-threaded by +/// construction, so the inner future is never actually sent between threads. +/// The `compile_error!` guard below is the backstop: enabling wasm threads +/// invalidates that reasoning and fails the build rather than silently +/// producing UB. #[cfg(feature = "wasm-runtime")] pub(crate) struct SendFuture(pub(crate) F); @@ -39,8 +43,8 @@ compile_error!( Disable the `atomics` target feature or provide a thread-safe implementation." ); -// SAFETY: see the type's docs — wasm32 (without atomics) is single-threaded, and -// the native lane only compiles (never polls) this type. +// SAFETY: see the type's docs — with `wasm-runtime` on, the only target this +// compiles for is wasm32 (without atomics), which is single-threaded. #[cfg(feature = "wasm-runtime")] unsafe impl Send for SendFuture {} diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index e00e7985..928c93d4 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -173,7 +173,7 @@ unsafe impl Send for WasmWsConnection {} impl Connection for WasmWsConnection { fn recv(&mut self) -> BoxFut<'_, TransportResult>>> { // `Ok(None)` when the frame funnel closes (socket closed/errored). - Box::pin(SendFuture(async move { Ok(self.frames.next().await) })) + Box::pin(async move { Ok(self.frames.next().await) }) } fn send<'a>(&'a mut self, frame: &'a [u8]) -> BoxFut<'a, TransportResult<()>> { From e8b2cb9bc9ecb206c884afc01e9698bc9ffa187a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Fri, 24 Jul 2026 12:37:39 +0000 Subject: [PATCH 21/45] perf(remote): carry subscribe updates as JSON bytes, not a Value tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The subscribe path never inspects a record update between reading it and encoding it onto the wire, yet it parsed the buffer's JSON bytes into a `serde_json::Value` (`recv_json`) only to re-serialize the tree back to bytes (`to_payload`) at the dispatch boundary — a pure parse+re-serialize round-trip per update. This is the direct-JSON-bytes follow-up design 048 flagged. `stream_record_updates` now reads owned JSON bytes via `recv_json_bytes` and yields `(Payload, u64)`; the two `stream.map` sites in the AimX dispatch build `SubUpdate` directly from those bytes. The design-048 `skipped` loss signal is untouched — it still rides each `SubUpdate`. No wire or public API change. Verified with the `remote_json` subscription-event benchmark (Tree vs Direct): the direct-bytes path sustains ~2× the throughput of the former tree path, matching the 2.06–2.49× encode figure in design 048. Co-Authored-By: Claude Opus 4.8 --- aimdb-core/CHANGELOG.md | 11 ++++++ aimdb-core/src/remote/stream.rs | 45 +++++++++++++++++-------- aimdb-core/src/session/aimx/dispatch.rs | 6 ++-- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index 97da4a07..5fd093ed 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -35,6 +35,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `{records, total}`); `RecordMetadata` gains optional `schema_type` / `entity` fields (`entity` derived from the record key's final `.` segment). +### Performance + +- **Subscribe path carries JSON bytes, not a `serde_json::Value` tree.** + `stream_record_updates` now reads owned JSON bytes (`recv_json_bytes`) and + yields `(Payload, u64)`, so a record update flows buffer → wire without the + parse-into-`Value`-then-re-serialize round-trip the dispatch layer used to + pay per update (design 048: "removing the intermediate `serde_json::Value`"). + The `remote_json` subscription-event benchmark measures the direct-bytes path + at ~2× the throughput of the former tree path. No wire or API change — the + `skipped` loss signal still rides each `SubUpdate`. + ### Added - **Issue #196 — direct JSON bytes for type-erased remote reads.** The public diff --git a/aimdb-core/src/remote/stream.rs b/aimdb-core/src/remote/stream.rs index 786dfbea..935996c4 100644 --- a/aimdb-core/src/remote/stream.rs +++ b/aimdb-core/src/remote/stream.rs @@ -1,8 +1,12 @@ //! Record-update streaming helper for AimX remote access. //! //! [`stream_record_updates`] adapts a record's [`JsonBufferReader`] into a -//! [`Stream`] of `(JSON value, skipped)` pairs, where `skipped` is the number -//! of updates the buffer dropped immediately before this one. +//! [`Stream`] of `(JSON bytes, skipped)` pairs, where `skipped` is the number +//! of updates the buffer dropped immediately before this one. The value is +//! carried as already-serialized bytes ([`Payload`](crate::session::Payload)), +//! not a `serde_json::Value` tree — the subscribe path never inspects it, so +//! parsing it only to re-serialize at the dispatch boundary is pure overhead +//! (design 048: "removing the intermediate `serde_json::Value`"). //! Cancellation is by `drop`; backpressure is the underlying buffer's //! responsibility. The handler holds the //! returned stream inside its per-subscription future so that the @@ -16,7 +20,7 @@ use crate::{AimDb, DbError, DbResult}; use futures_core::Stream; use futures_util::stream::unfold; -/// Subscribe to a record and yield each update as a `(JSON value, skipped)` +/// Subscribe to a record and yield each update as a `(JSON bytes, skipped)` /// pair. /// /// The returned stream owns the underlying [`JsonBufferReader`]; dropping @@ -32,7 +36,7 @@ use futures_util::stream::unfold; pub(crate) fn stream_record_updates( db: &AimDb, record_key: &str, -) -> DbResult + Send + 'static> { +) -> DbResult + Send + 'static> { let inner = db.inner(); let id = inner .resolve_str(record_key) @@ -62,8 +66,11 @@ pub(crate) fn stream_record_updates( Ok(unfold(state, |(mut reader, key, mut skipped)| async move { loop { - match reader.recv_json().await { - Ok(value) => return Some(((value, skipped), (reader, key, 0))), + match reader.recv_json_bytes().await { + Ok(bytes) => { + let data = crate::session::Payload::from(bytes); + return Some(((data, skipped), (reader, key, 0))); + } Err(DbError::BufferLagged { lag_count, .. }) => { log_warn!( "stream_record_updates: record '{}' subscription lagged by {} messages", @@ -133,12 +140,13 @@ mod tests { step: Arc::new(AtomicUsize::new(0)), })); - // Mirrors `stream_record_updates`'s unfold: accumulate `lag_count` into - // `skipped` and let it ride the next yielded value. + // Mirrors `stream_record_updates`'s unfold: read owned JSON *bytes* + // (not a `Value` tree), accumulate `lag_count` into `skipped`, and let + // it ride the next yielded value. let stream = unfold((reader, 0_u64), |(mut reader, mut skipped)| async move { loop { - match reader.recv_json().await { - Ok(v) => return Some(((v, skipped), (reader, 0))), + match reader.recv_json_bytes().await { + Ok(bytes) => return Some(((bytes, skipped), (reader, 0))), Err(DbError::BufferLagged { lag_count, .. }) => { skipped = skipped.saturating_add(lag_count); continue; @@ -149,11 +157,20 @@ mod tests { } }); - let values: Vec<_> = stream.collect().await; + let values: Vec<(Vec, u64)> = stream.collect().await; assert_eq!(values.len(), 2); // First value delivered cleanly; the Lagged(7) between the two folds - // into the second value's `skipped`, not swallowed. - assert_eq!(values[0], (serde_json::json!({"v": 1}), 0)); - assert_eq!(values[1], (serde_json::json!({"v": 2}), 7)); + // into the second value's `skipped`, not swallowed. The value rides as + // raw JSON bytes, so decode before comparing. + assert_eq!( + serde_json::from_slice::(&values[0].0).unwrap(), + serde_json::json!({"v": 1}) + ); + assert_eq!(values[0].1, 0); + assert_eq!( + serde_json::from_slice::(&values[1].0).unwrap(), + serde_json::json!({"v": 2}) + ); + assert_eq!(values[1].1, 7); } } diff --git a/aimdb-core/src/session/aimx/dispatch.rs b/aimdb-core/src/session/aimx/dispatch.rs index 61dc9fd4..7f8f5b21 100644 --- a/aimdb-core/src/session/aimx/dispatch.rs +++ b/aimdb-core/src/session/aimx/dispatch.rs @@ -129,7 +129,7 @@ impl Session for AimxSession { let stream = crate::remote::stream::stream_record_updates(&self.db, topic) .map_err(map_db_err)?; Ok(Box::pin( - stream.map(|(v, skipped)| SubUpdate::new(to_payload(&v)).with_skipped(skipped)), + stream.map(|(data, skipped)| SubUpdate::new(data).with_skipped(skipped)), ) as BoxStream<'static, SubUpdate>) }) } @@ -192,8 +192,8 @@ impl AimxSession { match crate::remote::stream::stream_record_updates(&self.db, &key) { Ok(stream) => { let tag: Arc = Arc::from(key.as_str()); - streams.push(Box::pin(stream.map(move |(v, skipped)| { - SubUpdate::tagged(tag.clone(), to_payload(&v)).with_skipped(skipped) + streams.push(Box::pin(stream.map(move |(data, skipped)| { + SubUpdate::tagged(tag.clone(), data).with_skipped(skipped) }))); } Err(_e) => { From 9d0930de114a0d014da02f6b802fae1aba55f813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Fri, 24 Jul 2026 12:50:01 +0000 Subject: [PATCH 22/45] format --- aimdb-core/src/session/aimx/dispatch.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/aimdb-core/src/session/aimx/dispatch.rs b/aimdb-core/src/session/aimx/dispatch.rs index 7f8f5b21..d2c0ab4b 100644 --- a/aimdb-core/src/session/aimx/dispatch.rs +++ b/aimdb-core/src/session/aimx/dispatch.rs @@ -128,9 +128,10 @@ impl Session for AimxSession { // Exact-topic fast path: resolve one key, untagged updates. let stream = crate::remote::stream::stream_record_updates(&self.db, topic) .map_err(map_db_err)?; - Ok(Box::pin( - stream.map(|(data, skipped)| SubUpdate::new(data).with_skipped(skipped)), - ) as BoxStream<'static, SubUpdate>) + Ok( + Box::pin(stream.map(|(data, skipped)| SubUpdate::new(data).with_skipped(skipped))) + as BoxStream<'static, SubUpdate>, + ) }) } From a32b05ea767586af9f27c9146829e263db2de385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 20:02:16 +0000 Subject: [PATCH 23/45] chore: update changelogs and design documents to reflect Design 047 changes --- CHANGELOG.md | 20 +++++++++++++++++-- aimdb-client/CHANGELOG.md | 2 +- aimdb-core/CHANGELOG.md | 2 +- aimdb-persistence/CHANGELOG.md | 2 +- aimdb-wasm-adapter/CHANGELOG.md | 2 +- aimdb-websocket-connector/CHANGELOG.md | 4 ++-- ...echnical-debt-and-simplification-review.md | 6 +++--- ...047-retire-ws-protocol-converge-on-aimx.md | 2 +- 8 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dbed4cc..46e189d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,9 +67,9 @@ unaffected. and the wasm-adapter README were migrated accordingly. - `topic_leaf` (entity extraction) remains tolerant of both `.` and `/`. -### Changed — Design 045: one wire protocol (AimX) for every transport (breaking) +### Changed — Design 047: one wire protocol (AimX) for every transport (breaking) -Implementation of [design 045](docs/design/045-retire-ws-protocol-converge-on-aimx.md) +Implementation of [design 047](docs/design/047-retire-ws-protocol-converge-on-aimx.md) (the 038 §3.9 / 036 A2 protocol unification). The `aimdb-ws-protocol` crate is **deleted**; the WebSocket connector and the browser `WsBridge` now speak the same AimX tagged frames as UDS/serial/TCP, over the same session engines. @@ -83,6 +83,22 @@ but the wire is new); breaking Rust API changes are listed per crate (`aimdb-core`, `aimdb-websocket-connector`, `aimdb-wasm-adapter`, `aimdb-client`, `aimdb-persistence`). +### Changed — knx-pico submodule + +- **Submodule:** bump `_external/knx-pico` to upstream `0.3` (commit 158325bd4) + +### Added — per-link record codecs + +- **Issue #178:** one `Linkable` record type can now select JSON, bounded + Postcard, or a custom codec independently on each inbound/outbound connector + route through `linked_*_with(url, codec)` or builder-level + `with_link_codec(codec)`. Selection is fused at registration time; bounded + Postcard routes retain the reusable scratch/owned-overflow behavior from + issue #177 without a per-message registry, lock, or lookup. See + [Design 045](docs/design/045-per-link-codec-selection.md). Re-selecting a + codec replaces both the owned and scratch serializers, so changing a route + from bounded postcard to owned JSON cannot retain stale postcard output. + ### Added — direct JSON remote-access payloads - **Issue #196:** state `record.get` and AimX subscription events now serialize diff --git a/aimdb-client/CHANGELOG.md b/aimdb-client/CHANGELOG.md index 2677f3bf..d55ea060 100644 --- a/aimdb-client/CHANGELOG.md +++ b/aimdb-client/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Changed (breaking) — Design 045 +### Changed (breaking) — Design 047 - **Wildcard subscriptions:** new `AimxConnection::subscribe_with_topics` yields `(Option, Value)` pairs so one pattern subscription (`#`, diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index 5fd093ed..ca037d32 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Changed (breaking) — Design 045: one protocol (AimX) for every transport +### Changed (breaking) — Design 047: one protocol (AimX) for every transport - **Wildcard / multi-record subscribe.** `Inbound::Subscribe` topics may carry MQTT-style wildcards (`#`, `*`): `AimxDispatch` matches the pattern against diff --git a/aimdb-persistence/CHANGELOG.md b/aimdb-persistence/CHANGELOG.md index dd1b19e5..6d0f0b9d 100644 --- a/aimdb-persistence/CHANGELOG.md +++ b/aimdb-persistence/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Changed (breaking) — Design 045 +### Changed (breaking) — Design 047 - **`with_persistence`'s registered `QueryHandlerFn` returns the shared `record.query` shape** `{"records": [{topic, payload, ts}, …], "total": N}` diff --git a/aimdb-wasm-adapter/CHANGELOG.md b/aimdb-wasm-adapter/CHANGELOG.md index b82b0090..a5d52076 100644 --- a/aimdb-wasm-adapter/CHANGELOG.md +++ b/aimdb-wasm-adapter/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Changed (breaking) — Design 045: `WsBridge` is an AimX engine client +### Changed (breaking) — Design 047: `WsBridge` is an AimX engine client - **`WsBridge` rewritten on `run_client` + `ClientHandle`** over a `web_sys::WebSocket`-backed `Connection`/`Dialer`: reply/subscription diff --git a/aimdb-websocket-connector/CHANGELOG.md b/aimdb-websocket-connector/CHANGELOG.md index 64b5ad73..65b747b9 100644 --- a/aimdb-websocket-connector/CHANGELOG.md +++ b/aimdb-websocket-connector/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Changed (breaking) — Design 045: the WS wire is now AimX +### Changed (breaking) — Design 047: the WS wire is now AimX - **The wire protocol is AimX** (`aimdb-core::session::aimx`), one tagged JSON frame per WS text message — the same envelope as UDS/serial/TCP. The @@ -34,7 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 subscriptions late-join every covered record (previously wildcard patterns never hit the exact-key cache). - **Auto-subscribe ids are server-chosen** (counting down from `u64::MAX`); - engine-demuxed clients should subscribe explicitly (design 045 §3.6). + engine-demuxed clients should subscribe explicitly (design 047 §3.6). ### Internal refactors diff --git a/docs/design/038-technical-debt-and-simplification-review.md b/docs/design/038-technical-debt-and-simplification-review.md index a5c40933..a77bb4ee 100644 --- a/docs/design/038-technical-debt-and-simplification-review.md +++ b/docs/design/038-technical-debt-and-simplification-review.md @@ -89,7 +89,7 @@ The knock-on structure this forces: ### 2.5 The remote-access stack: one excellent engine, two protocols, three clients -> **Status update:** resolved by [design 045](./045-retire-ws-protocol-converge-on-aimx.md) — the table below is the *pre-045* picture; every transport (including the browser) now speaks AimX over the shared engines. +> **Status update:** resolved by [design 047](./047-retire-ws-protocol-converge-on-aimx.md) — the table below is the *pre-047* picture; every transport (including the browser) now speaks AimX over the shared engines. The session substrate (`session/mod.rs` + server/client engines + pumps, ~2,300 lines) is genuinely good work: dyn-safe layering (Connection/Listener/Dialer → EnvelopeCodec → Dispatch/Session), runtime-neutral (`futures` channels + `RuntimeOps` clock), and it made `aimdb-uds-connector` a 391-line crate. That is the target picture. @@ -203,7 +203,7 @@ Validate "at most one of {source, transform, link_from}" in exactly one place: ` ### 3.9 One remote-access protocol — **~1,200–1,800 lines, breaking for browser clients, real work** -> **Status update: implemented via [design 045](./045-retire-ws-protocol-converge-on-aimx.md)** (the design-doc gate 036 A2 asked for). `aimdb-ws-protocol`, the WS codec, and the hand-rolled `WsBridge` demux are gone; AimX gained wildcard subscribe (optional `Event.topic`, `Snapshot.sub`, the `subscribed` ack frame) and the shared `record.query`/`record.list` result shapes. Two scope corrections vs. the paragraph below: the `aimdb-client` demux fold had **already** happened (PR #124 — only dead re-exports remained), and query passthrough already existed (`record.query` + `QueryHandlerFn`); what was missing was the result shape, not the passthrough. +> **Status update: implemented via [design 047](./047-retire-ws-protocol-converge-on-aimx.md)** (the design-doc gate 036 A2 asked for). `aimdb-ws-protocol`, the WS codec, and the hand-rolled `WsBridge` demux are gone; AimX gained wildcard subscribe (optional `Event.topic`, `Snapshot.sub`, the `subscribed` ack frame) and the shared `record.query`/`record.list` result shapes. Two scope corrections vs. the paragraph below: the `aimdb-client` demux fold had **already** happened (PR #124 — only dead re-exports remained), and query passthrough already existed (`record.query` + `QueryHandlerFn`); what was missing was the result shape, not the passthrough. Port the WebSocket connector to speak AimX over its existing WS transport (it already runs on the session engine; the delta is the envelope), extend AimX with the two features that justified the fork (wildcard subscribe, query passthrough — both fit the `method`/`topic` model), and rewrite `WsBridge` as an AimX client. Then delete `aimdb-ws-protocol` (353), the WS codec (507), and the protocol-specific halves of `server/` and `WsBridge`. End state: **one protocol, one snapshot/ack/auth semantics, every transport including the browser**. This also collapses the documentation story ("AimX is how you talk to AimDB — over UDS, serial, TCP, or WebSocket"). @@ -252,7 +252,7 @@ Respecting the 034 Phase-4 decision to stay monolithic: at minimum move the 5-cr | §3.3 DbError diet | ~450 | match sites | low | **implemented** | | §3.4–§3.6 registry/traits/validation | ~520 | no | low | **implemented** | | §3.7–§3.8 API pruning | ~450 | mechanical | low | **implemented** | -| §3.9 one protocol | ~1,500 | browser clients | medium–high | **implemented (design 045)** | +| §3.9 one protocol | ~1,500 | browser clients | medium–high | **implemented (design 047)** | | §3.10 extract agent | ~4,000 (relocated) | no | organizational | **deferred** — CI drift check implemented instead | | §3.11 features | (gates, not lines) | feature selectors | low | **implemented** as amended | | §3.12 examples/workspace | (build time) | no | low | **discarded for now** | diff --git a/docs/design/047-retire-ws-protocol-converge-on-aimx.md b/docs/design/047-retire-ws-protocol-converge-on-aimx.md index d0212922..686a6229 100644 --- a/docs/design/047-retire-ws-protocol-converge-on-aimx.md +++ b/docs/design/047-retire-ws-protocol-converge-on-aimx.md @@ -1,4 +1,4 @@ -# 045 — Retire `aimdb-ws-protocol`: every transport speaks AimX +# 047 — Retire `aimdb-ws-protocol`: every transport speaks AimX **Status:** ✅ Implemented (mapping-table gate from 036 A2; shipped on `feat/retire-aimdb-ws-protocol`) From ce23c1a25a82f6599214fcd97ec41307c0668bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 20:21:34 +0000 Subject: [PATCH 24/45] doc: add contract clarification for write_frame_splicing_data function --- aimdb-core/src/session/aimx/codec.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index 73a27156..581c72e9 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -132,6 +132,21 @@ fn write_frame(out: &mut alloc::vec::Vec, frame: &Frame<'_>) -> Result<(), C /// spliced `data` is appended as the final member. Byte-identical to serializing /// the same frame with `data = Some(as_raw(payload))`, since `data` is the last /// field in declaration order. +/// +/// # Contract (deliberately unchecked) +/// +/// `data` **must** be exactly one well-formed JSON value. The splice does not +/// validate it, that is the entire point of Improvement A and re-checking +/// here (or per-subscriber) is the O(payload) cost this path exists to avoid. +/// The invariant is upheld upstream, not defended here: `data` is a record +/// serializer's output and every record reachable over an AimX transport +/// serializes to JSON (the wire envelope *is* JSON). A caller that feeds +/// non-JSON bytes — e.g. a non-JSON per-link codec (`linked_*_with`) on a +/// `ws://` route, which is a JSON-envelope transport — violates the contract +/// and produces a corrupt frame (empty `data` yields `…,"data":}`); such a +/// codec/transport pairing is a misconfiguration, not an input to guard against. +/// The `debug_assert` below is a dev-time tripwire for the `frame.data` half of +/// the contract; there is intentionally no release-mode check on `data` itself. fn write_frame_splicing_data( out: &mut alloc::vec::Vec, frame: &Frame<'_>, From c9a803e93e8ca341ca537910c0c84db77d60368f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 20:32:05 +0000 Subject: [PATCH 25/45] feat: implement pattern containment check for subscription ACLs --- aimdb-core/src/lib.rs | 7 +- aimdb-core/src/session/mod.rs | 2 +- aimdb-core/src/session/topic_match.rs | 117 +++++++++++++++++++ aimdb-websocket-connector/CHANGELOG.md | 11 ++ aimdb-websocket-connector/src/server/auth.rs | 48 +++++++- 5 files changed, 180 insertions(+), 5 deletions(-) diff --git a/aimdb-core/src/lib.rs b/aimdb-core/src/lib.rs index 4f6c11cb..1bd06226 100644 --- a/aimdb-core/src/lib.rs +++ b/aimdb-core/src/lib.rs @@ -84,9 +84,10 @@ pub use remote::topic_leaf; // compatible). See docs/design/remote-access-via-connectors.md. #[cfg(feature = "connector-session")] pub use session::{ - is_wildcard, pump_sink, pump_source, topic_matches, AuthError, BoxFut, BoxStream, CodecError, - Connection, Dialer, Dispatch, EnvelopeCodec, Inbound, Listener, Outbound, Payload, PeerInfo, - RpcError, SessionCtx, SessionLimits, Source, SubUpdate, TransportError, TransportResult, + is_wildcard, pattern_contains, pump_sink, pump_source, topic_matches, AuthError, BoxFut, + BoxStream, CodecError, Connection, Dialer, Dispatch, EnvelopeCodec, Inbound, Listener, + Outbound, Payload, PeerInfo, RpcError, SessionCtx, SessionLimits, Source, SubUpdate, + TransportError, TransportResult, }; // Signal gauge handle (always available; inert without `observability`) diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index e780cb23..ff911598 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -36,7 +36,7 @@ mod server; pub mod aimx; mod topic_match; -pub use topic_match::{is_wildcard, topic_matches}; +pub use topic_match::{is_wildcard, pattern_contains, topic_matches}; #[cfg(feature = "connector-session")] pub use client::{pump_client, run_client, ClientConfig, ClientHandle}; diff --git a/aimdb-core/src/session/topic_match.rs b/aimdb-core/src/session/topic_match.rs index dbfc9cec..50f80c92 100644 --- a/aimdb-core/src/session/topic_match.rs +++ b/aimdb-core/src/session/topic_match.rs @@ -65,6 +65,59 @@ pub fn is_wildcard(pattern: &str) -> bool { pattern.contains('#') || pattern.contains('*') } +/// Returns `true` if every topic matched by `requested` is also matched by +/// `grant` — i.e. `grant`'s match set ⊇ `requested`'s match set (`grant` +/// *contains* `requested`). +/// +/// This is **pattern containment**, not topic matching, and the two diverge +/// once `requested` is itself a wildcard. An access check must ask containment: +/// "does my grant cover the whole pattern this client asked to subscribe to?" +/// [`topic_matches`] answers a different question ("does this pattern match +/// this string?") and would treat the requested pattern as an opaque topic — +/// so a one-level grant `a.*` "matches" the all-levels request `a.#` (the `*` +/// swallows the `#`) and silently widens the grant. Containment denies that: +/// `a.#` reaches deeper than `a.*` covers. +/// +/// When `requested` is a concrete topic (no wildcard) this collapses to exactly +/// [`topic_matches`]`(grant, requested)`, so it is a safe drop-in for an ACL +/// that must accept both concrete and wildcard subscription requests. +/// +/// Grammar is the same dot-separated `*` (single-level) / `#` (multi-level, and +/// as `grant`'s tail, zero-or-more) set as [`topic_matches`]. +pub fn pattern_contains(grant: &str, requested: &str) -> bool { + let mut g = grant.split('.'); + let mut r = requested.split('.'); + loop { + match (g.next(), r.next()) { + // A `#` in the grant absorbs the entire remaining request tail + // (including an already-exhausted one, e.g. `a.#` contains `a`). + (Some("#"), _) => return true, + // Both exhausted together: an exact structural cover. + (None, None) => return true, + // Request reaches past where the grant stops covering. + (None, Some(_)) => return false, + // Grant still requires ≥1 concrete segment the request never yields. + (Some(_), None) => return false, + // A grant `*` covers exactly one segment. A request `#` here could + // expand to zero or many segments, so `*` cannot contain it; a + // request `*` or literal is exactly one segment and is covered. + (Some("*"), Some(rs)) => { + if rs == "#" { + return false; + } + } + // A grant literal covers only itself: the request segment must be + // that same literal (a request `*`/`#` here would reach topics the + // literal doesn't, so it is not contained). + (Some(gs), Some(rs)) => { + if gs != rs { + return false; + } + } + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -149,4 +202,68 @@ mod tests { assert!(!is_wildcard("temp.vienna")); assert!(is_wildcard("temp.*")); } + + #[test] + fn containment_denies_wildcard_escalation() { + // The bug this function exists to prevent: a one-level grant must not + // cover an all-levels request just because `topic_matches` lets the + // grant's `*` swallow the request's `#`. + assert!(!pattern_contains("sensors.*", "sensors.#")); + assert!(topic_matches("sensors.*", "sensors.#")); // the trap it avoids + // `*` also can't cover a deeper concrete request. + assert!(!pattern_contains("sensors.*", "sensors.temp.vienna")); + } + + #[test] + fn containment_allows_when_grant_is_broader_or_equal() { + // `#` covers everything. + assert!(pattern_contains("#", "sensors.#")); + assert!(pattern_contains("#", "anything.deep.here")); + // A grant `#` tail covers deeper requests and the prefix itself. + assert!(pattern_contains("sensors.#", "sensors.temp.#")); + assert!(pattern_contains("sensors.#", "sensors.*")); + assert!(pattern_contains("sensors.#", "sensors.temp")); + assert!(pattern_contains("sensors.#", "sensors")); // `a.#` contains `a` + // Equal patterns contain each other. + assert!(pattern_contains("sensors.#", "sensors.#")); + assert!(pattern_contains("a.*.c", "a.*.c")); + // One-level grant covers one-level requests (literal or `*`). + assert!(pattern_contains("sensors.*", "sensors.temp")); + assert!(pattern_contains("sensors.*", "sensors.*")); + } + + #[test] + fn containment_denies_out_of_scope_or_shallower() { + // Different subtree. + assert!(!pattern_contains("sensors.#", "commands.#")); + assert!(!pattern_contains("sensors.temp", "sensors.humidity")); + // A deeper grant does not cover a shallower request. + assert!(!pattern_contains("a.b.c", "a.b")); + // A literal grant is not widened by a `*`/`#` request. + assert!(!pattern_contains("sensors.temp", "sensors.*")); + assert!(!pattern_contains("sensors.temp", "sensors.#")); + } + + #[test] + fn containment_matches_topic_matches_on_concrete_requests() { + // For a concrete (wildcard-free) request, containment must be exactly + // `topic_matches` — the safe-drop-in property the ACL relies on. + let grants = ["#", "sensors.#", "sensors.*", "sensors.temp", "*.temp"]; + let topics = [ + "sensors.temp", + "sensors.temp.vienna", + "sensors", + "commands.on", + "a.temp", + ]; + for g in grants { + for t in topics { + assert_eq!( + pattern_contains(g, t), + topic_matches(g, t), + "grant={g:?} topic={t:?}" + ); + } + } + } } diff --git a/aimdb-websocket-connector/CHANGELOG.md b/aimdb-websocket-connector/CHANGELOG.md index 65b747b9..806a51fe 100644 --- a/aimdb-websocket-connector/CHANGELOG.md +++ b/aimdb-websocket-connector/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Security + +- **Subscribe ACL now checks pattern *containment*, not topic matching.** A + granted subscribe pattern is honored only if it covers the *whole* pattern the + client requests. Previously the check matched the requested pattern as if it + were a concrete topic, so a one-level grant (`sensors.*`) admitted an + all-levels request (`sensors.#`) — the grant's `*` swallowed the request's + `#` — silently widening the grant. Concrete (wildcard-free) subscribes are + unaffected. (Latent before Design 047's `/`→`.` separator fix, which is what + made dot-keyed grant patterns match at all.) + ### Changed (breaking) — Design 047: the WS wire is now AimX - **The wire protocol is AimX** (`aimdb-core::session::aimx`), one tagged JSON diff --git a/aimdb-websocket-connector/src/server/auth.rs b/aimdb-websocket-connector/src/server/auth.rs index d0727461..62c0ce43 100644 --- a/aimdb-websocket-connector/src/server/auth.rs +++ b/aimdb-websocket-connector/src/server/auth.rs @@ -65,13 +65,26 @@ impl Permissions { } /// Returns `true` if the client is allowed to subscribe to `topic`. + /// + /// `topic` is the client's requested subscription, which may itself be a + /// wildcard — so this asks **pattern containment** ([`pattern_contains`]): + /// does a granted pattern cover the *whole* requested pattern? Plain + /// [`topic_matches`](aimdb_core::topic_matches) would let a one-level grant + /// (`sensors.*`) admit an all-levels request (`sensors.#`) by having the + /// `*` swallow the `#`, silently widening the grant. For a concrete request + /// `pattern_contains` collapses to `topic_matches`, so exact subscribes are + /// unaffected. pub fn can_subscribe(&self, topic: &str) -> bool { self.subscribe_patterns .iter() - .any(|p| aimdb_core::topic_matches(p, topic)) + .any(|p| aimdb_core::pattern_contains(p, topic)) } /// Returns `true` if the client is allowed to write to `topic`. + /// + /// Writes target a single concrete record, so `topic` is never a wildcard + /// here and plain [`topic_matches`](aimdb_core::topic_matches) is the right + /// check (a wildcard write topic would resolve to no record downstream). pub fn can_write(&self, topic: &str) -> bool { self.write_patterns .iter() @@ -190,3 +203,36 @@ impl AuthHandler for NoAuth { /// Type-erased auth handler stored inside the connector. pub(crate) type DynAuthHandler = Arc; + +#[cfg(test)] +mod tests { + use super::Permissions; + + fn perms(subscribe: &[&str]) -> Permissions { + Permissions { + subscribe_patterns: subscribe.iter().map(|s| s.to_string()).collect(), + write_patterns: Vec::new(), + } + } + + #[test] + fn subscribe_grant_is_not_widened_by_a_wildcard_request() { + // A one-level grant must not admit an all-levels request (the `*`-eats-`#` + // escalation): `sensors.*` covers one level, `sensors.#` covers all. + let p = perms(&["sensors.*"]); + assert!(p.can_subscribe("sensors.temp")); // in scope + assert!(!p.can_subscribe("sensors.#")); // escalation — denied + assert!(!p.can_subscribe("sensors.temp.vienna")); // deeper — denied + } + + #[test] + fn subscribe_allows_requests_the_grant_actually_covers() { + assert!(perms(&["#"]).can_subscribe("sensors.#")); + let p = perms(&["sensors.#"]); + assert!(p.can_subscribe("sensors.#")); + assert!(p.can_subscribe("sensors.temp.#")); + assert!(p.can_subscribe("sensors.temp")); + // Out of the granted subtree stays denied. + assert!(!p.can_subscribe("commands.#")); + } +} From da0db91cc1b2dc85eaabe47663cce7221ef1f829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 20:43:35 +0000 Subject: [PATCH 26/45] feat: implement WebSocket protocol versioning via upgrade URL for AimX clients --- CHANGELOG.md | 9 ++++++ aimdb-core/CHANGELOG.md | 9 +++++- aimdb-core/src/remote/mod.rs | 4 +-- aimdb-core/src/remote/protocol.rs | 20 +++++++++++++ aimdb-wasm-adapter/CHANGELOG.md | 6 +++- aimdb-wasm-adapter/src/bindings.rs | 3 ++ aimdb-wasm-adapter/src/ws_bridge.rs | 5 +++- aimdb-websocket-connector/CHANGELOG.md | 6 ++++ aimdb-websocket-connector/src/server/http.rs | 28 ++++++++++++++++++ aimdb-websocket-connector/src/transport.rs | 6 +++- aimdb-websocket-connector/tests/e2e.rs | 31 ++++++++++++++++++-- 11 files changed, 118 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46e189d4..9ab98bec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,15 @@ exported from `aimdb_core::remote`); a missing/malformed version fails closed. - **Caveat carried from design 047 §3.6:** server-seeded auto-subscriptions are invisible to `run_client`-based consumers. Client authors upgrading to 3.0 must drive their own `subscribe` for records they want streamed. +- **WebSocket / browser upgrade gate.** The socket transports negotiate the + version inside `hello`, but the WS server runs `reads_hello:false` and a + browser cannot set WebSocket handshake headers — so a WS client declares its + version in the upgrade URL (`…/ws?v=3.0`) and the server refuses an + incompatible/absent one with **HTTP 426 Upgrade Required** before the socket + opens. New `aimdb_core::remote::{VERSION_PARAM, ws_url_with_version}` hold the + contract; the browser `WsBridge`/`WasmDb.discover` and the Rust + `WsClientConnector` append it automatically. A stale browser client now fails + at the upgrade rather than on its first frame. - Schema-level record migration over AimX (the `Migratable` trait) is **out of scope** here and tracked as a follow-up (`with_migration`). diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index ca037d32..2ac865fa 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -16,7 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and emits one late-join `Snapshot` per matched record. The matcher moved in from the retired `aimdb-ws-protocol` crate as `session::topic_match::{topic_matches, is_wildcard}` (re-exported at the - crate root). + crate root), alongside a new `pattern_contains(grant, requested)` — pattern + *containment* (grant's match set ⊇ requested's), the check an ACL needs when + the requested subscription is itself a wildcard (see the WS connector's + subscribe-ACL fix). +- **AimX version handshake.** New `remote::{VERSION_PARAM, ws_url_with_version}` + carry the WS upgrade-URL version contract (`?v=3.0`) that the WS server gates + on; `version_compatible` is now also re-exported from `remote` for reuse by + transports that check the version out-of-band. - **Subscription streams carry the firing record.** `Session::subscribe` and `ClientHandle::subscribe` now yield `SubUpdate { topic: Option>, data: Payload }` instead of bare `Payload`; `Outbound::Event` gains an diff --git a/aimdb-core/src/remote/mod.rs b/aimdb-core/src/remote/mod.rs index 6fdbf9f0..393de583 100644 --- a/aimdb-core/src/remote/mod.rs +++ b/aimdb-core/src/remote/mod.rs @@ -54,8 +54,8 @@ pub use config::{AimxConfig, SecurityPolicy}; pub use error::{RemoteError, RemoteResult}; pub use metadata::RecordMetadata; pub use protocol::{ - version_compatible, ErrorObject, Event, HelloMessage, Request, Response, WelcomeMessage, - PROTOCOL_VERSION, + version_compatible, ws_url_with_version, ErrorObject, Event, HelloMessage, Request, Response, + WelcomeMessage, PROTOCOL_VERSION, VERSION_PARAM, }; pub use query::{QueryHandlerFn, QueryHandlerParams, QueryRecord}; diff --git a/aimdb-core/src/remote/protocol.rs b/aimdb-core/src/remote/protocol.rs index 02c01bab..d5628e09 100644 --- a/aimdb-core/src/remote/protocol.rs +++ b/aimdb-core/src/remote/protocol.rs @@ -14,6 +14,26 @@ use serde_json::Value as JsonValue; /// Version of the AimX wire protocol spoken by this crate. pub const PROTOCOL_VERSION: &str = "3.0"; +/// Query-parameter name a WebSocket client uses to declare its +/// [`PROTOCOL_VERSION`] at the HTTP upgrade. The socket transports gate the +/// version inside the `hello` handshake, but a browser cannot set custom +/// headers on the WebSocket handshake **and** the WS server runs +/// `reads_hello:false`, so the WS upgrade carries the version in the URL +/// instead (`…/ws?v=3.0`) and the server refuses an incompatible/absent one +/// before the socket opens. See [`ws_url_with_version`]. +pub const VERSION_PARAM: &str = "v"; + +/// Append this crate's [`PROTOCOL_VERSION`] as the [`VERSION_PARAM`] query +/// parameter to a WebSocket upgrade `url`, joining with `&` if `url` already +/// carries a query string and `?` otherwise. Every first-party WS client +/// dialer routes its URL through this so the server's upgrade-time gate sees a +/// version; keeping the `NAME=VALUE` contract here means the reader +/// (the server) and the writers (the dialers) never drift. +pub fn ws_url_with_version(url: &str) -> String { + let sep = if url.contains('?') { '&' } else { '?' }; + alloc::format!("{url}{sep}{VERSION_PARAM}={PROTOCOL_VERSION}") +} + /// Major-version component of a `"MAJOR.MINOR"` protocol string, or `None` if it /// is empty or not in that shape. fn protocol_major(version: &str) -> Option<&str> { diff --git a/aimdb-wasm-adapter/CHANGELOG.md b/aimdb-wasm-adapter/CHANGELOG.md index a5d52076..5347a05a 100644 --- a/aimdb-wasm-adapter/CHANGELOG.md +++ b/aimdb-wasm-adapter/CHANGELOG.md @@ -15,11 +15,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `aimdb-core`'s client engine — the hand-rolled 835-line demux is gone. The `#[wasm_bindgen]` surface is preserved (`write`, `query`, `listTopics`, `onStatusChange`, `status`, `disconnect`, `connectBridge` options), but the - wire is AimX, so the bridge only talks to design-045 servers. + wire is AimX, so the bridge only talks to design-047 servers. `BridgeOptions.lateJoin` is retained for option-shape compatibility (snapshots are server-driven under AimX). - **`WasmDb.discover` / the raw discovery path speak `record.list`** and resolve with `{name, schema_type, entity}` rows. +- **The bridge and discovery declare their AimX version** on the WebSocket + upgrade URL (`?v=3.0`, via `aimdb_core::remote::ws_url_with_version`), which a + design-047 server gates on (HTTP 426 on mismatch) — a browser cannot set + handshake headers, so the version rides the URL. - Depends on `aimdb-core`'s `connector-session` + `remote` features (the engines cross-compile to wasm32); the `aimdb-ws-protocol` dependency is gone. diff --git a/aimdb-wasm-adapter/src/bindings.rs b/aimdb-wasm-adapter/src/bindings.rs index fc85645d..c9122208 100644 --- a/aimdb-wasm-adapter/src/bindings.rs +++ b/aimdb-wasm-adapter/src/bindings.rs @@ -353,6 +353,9 @@ impl WasmDb { /// so that whichever event fires first wins and subsequent events are no-ops. fn discover_impl(url: String) -> js_sys::Promise { js_sys::Promise::new(&mut move |resolve, reject| { + // Declare our AimX version in the URL — the server gates the WS upgrade + // on it, and a browser cannot set handshake headers (`?v=3.0`). + let url = aimdb_core::remote::ws_url_with_version(&url); let ws = match web_sys::WebSocket::new(&url) { Ok(ws) => ws, Err(e) => { diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index 928c93d4..a12adb3d 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -226,7 +226,10 @@ unsafe impl Send for WasmWsDialer {} impl Dialer for WasmWsDialer { fn connect(&self) -> BoxFut<'_, TransportResult>> { - let url = self.url.clone(); + // Declare our AimX version in the URL so the server's upgrade-time gate + // admits us — a browser cannot set custom WebSocket handshake headers, + // so the version rides the query string (`?v=3.0`). + let url = aimdb_core::remote::ws_url_with_version(&self.url); let shared = self.shared.clone(); Box::pin(SendFuture(async move { if shared.stopped.get() { diff --git a/aimdb-websocket-connector/CHANGELOG.md b/aimdb-websocket-connector/CHANGELOG.md index 806a51fe..ad8b4fd9 100644 --- a/aimdb-websocket-connector/CHANGELOG.md +++ b/aimdb-websocket-connector/CHANGELOG.md @@ -46,6 +46,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 never hit the exact-key cache). - **Auto-subscribe ids are server-chosen** (counting down from `u64::MAX`); engine-demuxed clients should subscribe explicitly (design 047 §3.6). +- **Protocol-version gate at the WS upgrade.** The client declares its AimX + version as `?v=` on the upgrade URL (browsers cannot set + handshake headers, and the server runs `reads_hello:false`); an + incompatible/absent version is refused with **HTTP 426 Upgrade Required** + before the socket opens, so a stale client fails at the handshake instead of + on its first frame. The bundled `WsClientConnector` appends it automatically. ### Internal refactors diff --git a/aimdb-websocket-connector/src/server/http.rs b/aimdb-websocket-connector/src/server/http.rs index 93fa3ffe..1b930ade 100644 --- a/aimdb-websocket-connector/src/server/http.rs +++ b/aimdb-websocket-connector/src/server/http.rs @@ -14,6 +14,7 @@ use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Instant}; use aimdb_core::{ + remote::{version_compatible, PROTOCOL_VERSION, VERSION_PARAM}, session::{aimx::AimxCodec, run_session, SessionConfig}, Connection, Dispatch, PeerInfo, SessionLimits, }; @@ -138,6 +139,33 @@ async fn ws_upgrade_handler( remote_addr, }; + // Protocol-version gate, before auth: the socket transports negotiate the + // version inside `hello`, but the WS server runs `reads_hello:false` and a + // browser cannot set handshake headers — so the client declares its version + // in the URL (`?v=3.0`, see `ws_url_with_version`). A missing or + // major-incompatible version is refused here with 426 so a stale client + // fails at the upgrade rather than on its first frame's new shape. Absent + // fails closed (`version_compatible("")` is false), matching the socket gate. + let client_version = auth_req + .query_params + .get(VERSION_PARAM) + .map(String::as_str) + .unwrap_or_default(); + if !version_compatible(client_version) { + #[cfg(feature = "tracing")] + tracing::warn!( + "WebSocket upgrade from {} refused: incompatible protocol version {:?} (server speaks {})", + remote_addr, + client_version, + PROTOCOL_VERSION + ); + return ( + StatusCode::UPGRADE_REQUIRED, + format!("incompatible AimX protocol version (server speaks {PROTOCOL_VERSION})"), + ) + .into_response(); + } + // Authenticate at the HTTP upgrade — returns permissions or rejects (401). let permissions = match state.auth.authenticate(&auth_req).await { Ok(p) => p, diff --git a/aimdb-websocket-connector/src/transport.rs b/aimdb-websocket-connector/src/transport.rs index 14fc3a19..e3c049fd 100644 --- a/aimdb-websocket-connector/src/transport.rs +++ b/aimdb-websocket-connector/src/transport.rs @@ -115,7 +115,11 @@ impl aimdb_core::Dialer for WsDialer { &self, ) -> aimdb_core::BoxFut<'_, aimdb_core::TransportResult>> { Box::pin(async move { - let (ws, _resp) = tokio_tungstenite::connect_async(&self.url) + // Declare our AimX version in the URL so the server's upgrade-time + // gate admits us (browsers can't set handshake headers; the Rust + // client follows the same contract for symmetry). + let url = aimdb_core::remote::ws_url_with_version(&self.url); + let (ws, _resp) = tokio_tungstenite::connect_async(&url) .await .map_err(|_| aimdb_core::TransportError::Io)?; Ok(Box::new(WsClientConnection { diff --git a/aimdb-websocket-connector/tests/e2e.rs b/aimdb-websocket-connector/tests/e2e.rs index 2b55528a..3e40bea2 100644 --- a/aimdb-websocket-connector/tests/e2e.rs +++ b/aimdb-websocket-connector/tests/e2e.rs @@ -185,7 +185,10 @@ type WsClient = tokio_tungstenite::WebSocketStream>; async fn ws_connect(addr: SocketAddr) -> WsClient { - tokio_tungstenite::connect_async(format!("ws://{addr}/ws")) + // Every real client declares its AimX version at the upgrade; go through the + // shared helper so the tests exercise the exact URL the dialers produce. + let url = aimdb_core::remote::ws_url_with_version(&format!("ws://{addr}/ws")); + tokio_tungstenite::connect_async(url) .await .expect("connect") .0 @@ -412,11 +415,33 @@ async fn server_ping_pong() { #[tokio::test] async fn server_rejects_unauthenticated_upgrade() { let (addr, _db) = spawn(WebSocketConnector::new().with_auth(DenyAuth)).await; - // The upgrade must be refused with HTTP 401 → the WS handshake fails. - let result = tokio_tungstenite::connect_async(format!("ws://{addr}/ws")).await; + // A compatible version so the request reaches auth; the upgrade must then be + // refused with HTTP 401 → the WS handshake fails. + let url = aimdb_core::remote::ws_url_with_version(&format!("ws://{addr}/ws")); + let result = tokio_tungstenite::connect_async(url).await; assert!(result.is_err(), "auth-rejected upgrade should not connect"); } +#[tokio::test] +async fn server_rejects_incompatible_protocol_version() { + let (addr, _db) = spawn_default().await; + // A pre-3.x client (or one omitting `?v`) is refused at the upgrade (426), + // before the socket opens — it never reaches the AimX frame loop. + let stale = tokio_tungstenite::connect_async(format!("ws://{addr}/ws?v=2.0")).await; + assert!(stale.is_err(), "incompatible version must not upgrade"); + let missing = tokio_tungstenite::connect_async(format!("ws://{addr}/ws")).await; + assert!( + missing.is_err(), + "absent version must not upgrade (fail closed)" + ); + // The current version still connects fine. + let ok = tokio_tungstenite::connect_async(aimdb_core::remote::ws_url_with_version(&format!( + "ws://{addr}/ws" + ))) + .await; + assert!(ok.is_ok(), "current version must upgrade"); +} + #[tokio::test] async fn server_survives_malformed_frame() { let (addr, db) = spawn_default().await; From eadfab14393dfe552aa89c510d5874a3e706a1bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 20:44:50 +0000 Subject: [PATCH 27/45] fix: clarify serialization behavior in write_frame and add assertions for data integrity --- aimdb-core/src/session/aimx/codec.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index 581c72e9..b3fd05ec 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -129,9 +129,14 @@ fn write_frame(out: &mut alloc::vec::Vec, frame: &Frame<'_>) -> Result<(), C /// once *per subscriber* over the shared payload (design 048 WI4, Improvement A). /// /// `frame` must carry `data = None` so serde emits every other field; the -/// spliced `data` is appended as the final member. Byte-identical to serializing -/// the same frame with `data = Some(as_raw(payload))`, since `data` is the last -/// field in declaration order. +/// spliced `data` is then appended as the final member. This is byte-identical +/// to serializing the same frame with `data = Some(as_raw(payload))` **only +/// while `data` is the last member serde emits** — i.e. every field declared +/// after `data` in [`Frame`] (`ok`, `err`) also skip-serializes. That holds for +/// the frames this path encodes (`Event` never sets `ok`/`err`), and the +/// `debug_assert` below makes the precondition explicit rather than incidental: +/// splicing a frame that set `ok`/`err` would emit `…,"ok":X,"data":Y}` with +/// `data` no longer last, breaking the byte-identical claim. /// /// # Contract (deliberately unchecked) /// @@ -153,6 +158,10 @@ fn write_frame_splicing_data( data: &[u8], ) -> Result<(), CodecError> { debug_assert!(frame.data.is_none(), "data must be spliced, not serialized"); + debug_assert!( + frame.ok.is_none() && frame.err.is_none(), + "fields declared after `data` must be None so the spliced `data` stays last" + ); let scaffold = serde_json::to_vec(frame).map_err(|_| CodecError::Malformed)?; // A `Frame` always serializes to a JSON object closed by `}`. let Some((&b'}', head)) = scaffold.split_last() else { From a61c02b330dd3a2fcd2c257d2dfeb6e518319bc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 20:55:05 +0000 Subject: [PATCH 28/45] chore: update changelogs and documentation to reflect design 047 changes --- CHANGELOG.md | 6 +- aimdb-bench/benches/b0_alloc_fanout_encode.rs | 3 +- aimdb-bench/benches/b1_b2_fanout_encode.rs | 3 +- aimdb-bench/src/fanout_encode.rs | 6 +- aimdb-bench/src/lib.rs | 4 +- aimdb-core/CHANGELOG.md | 2 +- aimdb-core/src/remote/stream.rs | 2 +- aimdb-core/src/session/aimx/codec.rs | 3 +- aimdb-core/src/session/aimx/dispatch.rs | 4 +- .../benches/fanout_socket.rs | 3 +- docs/design/048-fanout-event-encoding.md | 390 ++++++++++++++++++ 11 files changed, 411 insertions(+), 15 deletions(-) create mode 100644 docs/design/048-fanout-event-encoding.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ab98bec..bad891fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,9 +29,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Changed — Design 048 WI1: AimX protocol version handshake gate (breaking) +### Changed — Design 047: AimX protocol version handshake gate (breaking) -`PROTOCOL_VERSION` is bumped **`"2.0"` → `"3.0"`** to mark the design-047/048 +`PROTOCOL_VERSION` is bumped **`"2.0"` → `"3.0"`** to mark the design-047 convergence as a breaking wire change (`record.query` results moved from `{values, count}` to `{records, total}`; wildcard subscribe / auto-subscribe added). The `hello` handshake now **negotiates** the version instead of ignoring @@ -61,7 +61,7 @@ exported from `aimdb_core::remote`); a missing/malformed version fails closed. - Schema-level record migration over AimX (the `Migratable` trait) is **out of scope** here and tracked as a follow-up (`with_migration`). -### Changed — Design 048 WI3a: dot is the one topic separator (breaking) +### Changed — Design 047: dot is the one topic separator (breaking) `topic_matches` (wildcard subscribe / WS fan-out) now splits topic patterns on **`.` only** — RabbitMQ topic-exchange semantics (dot segments, `*` diff --git a/aimdb-bench/benches/b0_alloc_fanout_encode.rs b/aimdb-bench/benches/b0_alloc_fanout_encode.rs index efa1914e..7e39f415 100644 --- a/aimdb-bench/benches/b0_alloc_fanout_encode.rs +++ b/aimdb-bench/benches/b0_alloc_fanout_encode.rs @@ -1,4 +1,5 @@ -//! B0 fan-out encoding allocation comparison — design 048 WI4 (H-C). +//! B0 fan-out encoding allocation comparison (H-C). +//! See `docs/design/048-fanout-event-encoding.md`. //! //! Allocation counts are deterministic, so this is the most reproducible //! signal for whether the shared-suffix fast path is worth its special-case diff --git a/aimdb-bench/benches/b1_b2_fanout_encode.rs b/aimdb-bench/benches/b1_b2_fanout_encode.rs index dce0590f..e91a2197 100644 --- a/aimdb-bench/benches/b1_b2_fanout_encode.rs +++ b/aimdb-bench/benches/b1_b2_fanout_encode.rs @@ -1,4 +1,5 @@ -//! B1/B2 fan-out encoding — design 048 WI4 (H-B scaling, H-C payoff). +//! B1/B2 fan-out encoding (H-B scaling, H-C payoff). +//! See `docs/design/048-fanout-event-encoding.md`. //! //! One timed iteration is **one broadcast to N subscribers**: turning a single //! shared record value into N per-subscriber AimX `event` frames. Compares the diff --git a/aimdb-bench/src/fanout_encode.rs b/aimdb-bench/src/fanout_encode.rs index 0ad2a89b..a88cc5a6 100644 --- a/aimdb-bench/src/fanout_encode.rs +++ b/aimdb-bench/src/fanout_encode.rs @@ -1,4 +1,5 @@ -//! Fan-out event-encoding microbenchmark for design 048 WI4. +//! Fan-out event-encoding microbenchmark. +//! See `docs/design/048-fanout-event-encoding.md`. //! //! Measures the per-broadcast cost of turning one shared record value into //! `N` per-subscriber AimX `event` frames — the only work that differs @@ -18,7 +19,8 @@ //! //! The record payload is copied into each frame by **both** strategies //! (a contiguous WS text frame embeds it), so this isolates the serialization -//! delta, not a copy the fast path avoids — see design 048 WI4's caveat. +//! delta, not a copy the fast path avoids — see the benchmark doc's caveat +//! (`docs/design/048-fanout-event-encoding.md`). use std::io::Write as _; diff --git a/aimdb-bench/src/lib.rs b/aimdb-bench/src/lib.rs index 61ce6ce6..63109324 100644 --- a/aimdb-bench/src/lib.rs +++ b/aimdb-bench/src/lib.rs @@ -28,8 +28,8 @@ //! | `benches/b_alloc_pipeline.rs` | info | Per-message allocation (runner pipeline) | //! | `benches/b_runner_pipeline.rs` | info | Runner pipeline throughput (Criterion) | //! | `benches/b0_alloc_migration.rs` | B0 | Per-call allocation (migrate_from_bytes) | -//! | `benches/b0_alloc_fanout_encode.rs`| B0 | Fan-out event-encode allocation (048 WI4)| -//! | `benches/b1_b2_fanout_encode.rs` | B1+B2 | Fan-out event-encode latency (048 WI4) | +//! | `benches/b0_alloc_fanout_encode.rs`| B0 | Fan-out event-encode allocation | +//! | `benches/b1_b2_fanout_encode.rs` | B1+B2 | Fan-out event-encode latency | //! //! On-target cycle profiling (B3) lives in the hardware-only //! `examples/embassy-bench-stm32h5` crate, since DWT cycle counting cannot run diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index 2ac865fa..0aac6865 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -48,7 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `stream_record_updates` now reads owned JSON bytes (`recv_json_bytes`) and yields `(Payload, u64)`, so a record update flows buffer → wire without the parse-into-`Value`-then-re-serialize round-trip the dispatch layer used to - pay per update (design 048: "removing the intermediate `serde_json::Value`"). + pay per update (removing the intermediate `serde_json::Value`). The `remote_json` subscription-event benchmark measures the direct-bytes path at ~2× the throughput of the former tree path. No wire or API change — the `skipped` loss signal still rides each `SubUpdate`. diff --git a/aimdb-core/src/remote/stream.rs b/aimdb-core/src/remote/stream.rs index 935996c4..df9639fc 100644 --- a/aimdb-core/src/remote/stream.rs +++ b/aimdb-core/src/remote/stream.rs @@ -6,7 +6,7 @@ //! carried as already-serialized bytes ([`Payload`](crate::session::Payload)), //! not a `serde_json::Value` tree — the subscribe path never inspects it, so //! parsing it only to re-serialize at the dispatch boundary is pure overhead -//! (design 048: "removing the intermediate `serde_json::Value`"). +//! (removing the intermediate `serde_json::Value`). //! Cancellation is by `drop`; backpressure is the underlying buffer's //! responsibility. The handler holds the //! returned stream inside its per-subscription future so that the diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index b3fd05ec..fc9f2744 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -126,7 +126,8 @@ fn write_frame(out: &mut alloc::vec::Vec, frame: &Frame<'_>) -> Result<(), C /// serializer's output), splicing `data` verbatim instead of re-validating it /// into a [`RawValue`]. Outbound event payloads are trusted-valid, so `as_raw`'s /// O(payload) scan is redundant — and on a fan-out subscription it would run -/// once *per subscriber* over the shared payload (design 048 WI4, Improvement A). +/// once *per subscriber* over the shared payload (the fan-out encode fast path; +/// see `docs/design/048-fanout-event-encoding.md`). /// /// `frame` must carry `data = None` so serde emits every other field; the /// spliced `data` is then appended as the final member. This is byte-identical diff --git a/aimdb-core/src/session/aimx/dispatch.rs b/aimdb-core/src/session/aimx/dispatch.rs index d2c0ab4b..c3d3f256 100644 --- a/aimdb-core/src/session/aimx/dispatch.rs +++ b/aimdb-core/src/session/aimx/dispatch.rs @@ -226,8 +226,8 @@ impl AimxSession { } let value = match method { - // WI1 version gate: refuse an incompatible/absent client version at - // the handshake rather than letting it trip over the new wire shapes. + // Version gate: refuse an incompatible/absent client version at the + // handshake rather than letting it trip over the new wire shapes. "hello" => self.hello(params), "record.list" => Ok(json!(self.db.list_records())), "record.set" => self.record_set(params), diff --git a/aimdb-websocket-connector/benches/fanout_socket.rs b/aimdb-websocket-connector/benches/fanout_socket.rs index 26d8b21a..6b5c068e 100644 --- a/aimdb-websocket-connector/benches/fanout_socket.rs +++ b/aimdb-websocket-connector/benches/fanout_socket.rs @@ -1,4 +1,5 @@ -//! H-A socket-driven fan-out benchmark (design 048 WI4). +//! H-A socket-driven fan-out benchmark. +//! See `docs/design/048-fanout-event-encoding.md`. //! //! Unlike the `aimdb-bench` encoding microbench (which isolates the codec), //! this drives the **real** converged path end-to-end: a live WebSocket server diff --git a/docs/design/048-fanout-event-encoding.md b/docs/design/048-fanout-event-encoding.md new file mode 100644 index 00000000..dbe7e95a --- /dev/null +++ b/docs/design/048-fanout-event-encoding.md @@ -0,0 +1,390 @@ +# 048 — Fan-out event encoding + +**Status:** complete — encoding microbench (§3–5) and end-to-end socket harness +(§6) both run; H-A/H-B/H-C answered. Improvement A implemented in `AimxCodec` +and re-measured (§7): ~14× faster encode, but only ~3% end-to-end — kept as +hygiene, not a hot-path fix. + +**Scope:** the fan-out event-encoding fast path on the converged AimX/WS +transport — design [047](047-retire-ws-protocol-converge-on-aimx.md). A +benchmark-driven analysis: it measures the per-broadcast encode cost and records +the decision to keep the fast path as hygiene rather than a hot-path fix. + +**Related:** design [047](047-retire-ws-protocol-converge-on-aimx.md) +(the convergence this measures) and design 046 (CBOR self-describing remote +access — record representation; a separate in-flight doc on PR #195, not yet in +this branch; shares the `aimdb-bench` harness and the byte-heavy-as-JSON +observation). + +**Measured on:** `feat/retire-aimdb-ws-protocol`. Encoding microbench: +`aimdb-bench/src/fanout_encode.rs`, `benches/b0_alloc_fanout_encode.rs`, +`benches/b1_b2_fanout_encode.rs`. Socket harness: +`aimdb-websocket-connector/benches/fanout_socket.rs`. + +**Environment:** Docker Desktop VM — Linux 6.10.14-linuxkit x86_64, Intel Core +i5-7267U @ 3.10 GHz (4 cores visible), rustc 1.91.1, `cargo bench` (optimized). +This is a **virtualized, noisy** host: absolutes run high and Criterion +confidence intervals are wide. The numbers below are point-estimate medians and +are evidence for **relative direction**, not portable latency guarantees. + +--- + +## 1. Purpose + +The converged AimX-over-WS path builds the `event` envelope **per subscriber**: +each connection stamps `{"t":"event","seq":N,"topic":T,"sub":S,"data":V}` with +its own `sub` (id-routed demux) and `seq` (drop detection). The retired +`aimdb-ws-protocol` path pre-serialized **one** `Data{topic,payload,ts}` frame +and shared the finished bytes, because that frame carried no per-subscriber +fields. `sub`/`seq` are the capabilities the convergence exists to add (047 +§2.2, §3.1), so "serialize once" was a property of a thinner frame, not a bar +to clear — the convergence is already decided **Go** (047 §5). + +This benchmark therefore does **not** compare against the deleted crate. It +measures the converged encode path **against itself** — the current +per-subscriber codec versus an "encode once, patch the per-subscriber fields" +fast path — to answer two questions: does the encode cost scale cleanly, and is +it worth optimizing? + +It is a pure CPU/allocation microbenchmark. It excludes the socket write, task +scheduling, the `ClientManager` fan-out, and backpressure — all of which both +strategies share. One timed unit is **one broadcast to N subscribers** (N +frames). + +--- + +## 2. Hypotheses (pre-registered) + +- **H-B — linear scaling.** Per-broadcast encode cost grows ~linearly in N with + a small constant per-subscriber increment; no super-linear knee. *Falsified + by* any O(N²)/contention curve. +- **H-C — optimization payoff.** The shared-suffix fast path reduces + per-broadcast cost by a meaningful fraction, and the benchmark localizes + *where* (expected: byte-heavy payloads and high fan-out) versus where the + naive path is already cheap (expected: small payloads). + +- **H-A — absolute acceptability & encode share.** Whether 500-client fan-out + is fast enough in absolute terms, and whether the encode is a real fraction of + the per-subscriber cost relative to the **socket write** both paths pay. The + CPU microbench cannot settle this; it is answered by the socket-driven harness + in §6. + +--- + +## 3. Method + +**Two payload shapes** (already-serialized JSON, as a record codec hands the +session layer): + +| Shape | Record payload | Event frame | +|---|--:|--:| +| `small_structured` | 110 B object | 191 B | +| `byte_heavy` | 3657 B JSON integer array (a byte record — JSON has no byte type, 046 §6.1) | 3738 B | + +**Two strategies, byte-identical output:** + +- **`naive`** — the production `AimxCodec::encode(Outbound::Event)`, re-run once + per subscriber. Each call re-escapes `topic`/`sub`, re-validates the record + value (`as_raw`), and allocates serde intermediates. +- **`shared_suffix`** — the fast path. Escapes `topic` once per broadcast into a + reused middle segment; per subscriber only formats the two varying scalars + (`sub`, `seq`) and splices the shared record bytes. The current AimX field + order (`t, seq, topic, sub, data`) already lets this stay **byte-identical** + to the codec, so no wire change is needed. + +**Faithfulness gate.** A test (`fast_path_is_byte_identical_to_codec`) asserts +the fast path reproduces the codec's bytes for both shapes across 512 +subscribers. This is what licenses treating the two strategies as measuring the +same output rather than comparing different frames. + +**Config.** Criterion: 30 samples, 1 s warmup, 3 s measurement. Allocation +bench: 50 warmup + 500 measured broadcasts per row, `CountingAllocator` global, +positive control fired after all windows. + +**Reproduce:** + +```bash +cargo test -p aimdb-bench fanout +cargo bench -p aimdb-bench --bench b0_alloc_fanout_encode +cargo bench -p aimdb-bench --bench b1_b2_fanout_encode +``` + +--- + +## 4. Results + +### 4.1 Allocation (deterministic) + +Allocation calls and bytes per broadcast; `allocs/frame` is per-subscriber. + +All three strategies at N=500 (`allocs/frame` is per-subscriber): + +| Shape | Strategy | allocs/frame | bytes/broadcast | +|---|---|--:|--:| +| small | naive | 3.00 | 288,390 | +| small | validate_once | 2.002 | 192,110 | +| small | shared_suffix | 1.002 | 99,438 | +| byte-heavy | naive | 4.00 | 7,542,060 | +| byte-heavy | validate_once | 3.002 | 5,675,827 | +| byte-heavy | shared_suffix | 1.002 | 1,872,938 | + +The naive path allocates **3–4× per frame** (serde intermediates + the output +buffer); hoisting validation removes one alloc/frame; the shared-suffix path +converges on **~1 alloc/frame** (just the output buffer) plus one per-broadcast +segment. For byte-heavy at 500 subscribers, allocation bytes drop **7.5 MB → +1.9 MB per broadcast** (naive → shared_suffix), with the validation hoist +accounting for the first ~1.9 MB of that. + +### 4.2 Latency (medians, µs per broadcast) + +Three strategies from one internally-consistent run (`naive` → `validate_once` += Improvement A alone; `validate_once` → `shared_suffix` = Improvement B on +top). Absolutes vary run-to-run on this virtualized host (see §7), so the +comparison uses a single run. + +**`small_structured` (191 B frame):** + +| N | naive | validate_once | shared_suffix | +|--:|--:|--:|--:| +| 1 | 1.44 | 1.65 | 0.30 | +| 10 | 9.46 | 4.78 | 2.10 | +| 100 | 99.6 | 70.7 | 19.7 | +| 500 | 530 | 293 | 107 | + +**`byte_heavy` (3738 B frame):** + +| N | naive | validate_once | shared_suffix | +|--:|--:|--:|--:| +| 1 | 15.4 | 24.2 | 0.47 | +| 10 | 148 | 23.7 | 3.47 | +| 100 | 1713 | 112 | 42.8 | +| 500 | 7587 | 347 | 143 | + +At N=1 `validate_once` can trail `naive` (its single validation is unamortized, +plus host noise); the comparison only matters at fan-out (N ≫ 1). + +--- + +## 5. Findings and encode-path improvements + +### 5.1 H-B — scaling is clean (for the encode loop) + +`naive` µs/frame is roughly constant (2.4–3.3 small; 14–22 byte-heavy); +`shared_suffix` µs/frame *decreases* with N as the once-per-broadcast precompute +amortizes. No super-linear knee. **H-B holds for the encode loop.** The caveat: +this microbench does not include the `ClientManager` `DashMap` fan-out, so the +true O(N²)/lock-contention check for the *bus* still requires the socket-driven +bench. + +### 5.2 H-C — the payoff, isolated + +The win decomposes into two independent improvements, and the `validate_once` +strategy isolates them. The root cause of the large byte-heavy gap is in the +codec: `AimxCodec::encode(Outbound::Event)` calls `as_raw(&data)` +(`aimdb-core/src/session/aimx/codec.rs:174`), which runs `serde_json` over the +**entire** record value to validate it is one JSON value. The naive path +therefore **re-validates the whole 1024-element array once per subscriber** — +O(payload) × N. + +The N=500 decomposition (µs per broadcast, this run): + +| Shape | naive | → validate_once (A) | → shared_suffix (B) | A share | B share | +|---|--:|--:|--:|--:|--:| +| small | 530 | 293 | 107 | ~56% | ~44% | +| byte-heavy | 7587 | 347 | 143 | **~97%** | ~3% | + +**Improvement A — hoist the payload validation out of the per-subscriber loop.** +The record value is identical for every subscriber of a broadcast; validating it +once instead of once per subscriber removes N−1 full `serde_json` passes over +the payload. For byte-heavy this **alone is a 21.8× reduction** (7587 → 347 µs) +— essentially the entire win. It is a **small, self-contained change** that +needs none of the shared-suffix machinery, and it is the highest +value-to-effort item this benchmark surfaced. For small payloads it is a ~1.8× +cut (validation is cheap when the payload is tiny). + +**Improvement B — the shared-suffix fast path.** Escape `topic` once per +broadcast and stamp per subscriber only the two varying scalars, rather than +re-running the full serde frame serialization each time. On top of A it adds a +further ~2.4× (byte-heavy) / ~2.7× (small) and cuts allocations to ~1/frame — it +carries **most of the small-payload win** but little of the byte-heavy one. It +applies cleanly to **exact-topic** fan-out (many clients on one topic share +`topic`/`data`); wildcard subscriptions, where each client matched a different +record, have no shared suffix and keep the per-frame path (047 §3.1). + +Both are on the same contiguous-frame model as today: the payload is still +copied into each subscriber's frame buffer (a WS text frame embeds it), so +neither improvement removes that copy — they remove *serialization and +validation*, not the memcpy. True zero-copy would need vectored writes, which +WebSocket framing via tokio-tungstenite does not cleanly expose. + +--- + +## 6. End-to-end socket-driven fan-out (H-A) + +A separate harness (`aimdb-websocket-connector/benches/fanout_socket.rs`) drives +the **real** converged path: a live `WebSocketConnector` server fanning one +record update out to N raw `tokio-tungstenite` clients (not aimdb client +connectors — 500 of those would put the cost on the client side). One round sets +the record to a fresh value and waits until every client observes it; the +elapsed time is the end-to-end fan-out latency of one broadcast, **including** +the socket write, Tokio scheduling, the `ClientManager` bus, and client +receive/parse. 100 rounds/config, same Docker VM. + +### 6.1 Results (median latency per broadcast) + +| Shape | N | median | per-delivery | msgs/s | +|---|--:|--:|--:|--:| +| small | 1 | 156 µs | 156 µs | 5.6k | +| small | 10 | 547 µs | 55 µs | 17k | +| small | 100 | 5.1 ms | 51 µs | 14k | +| small | 500 | 20.8 ms | 42 µs | 22k | +| byte-heavy | 1 | 287 µs | 287 µs | 3.1k | +| byte-heavy | 10 | 732 µs | 73 µs | 11k | +| byte-heavy | 100 | 4.7 ms | 47 µs | 19k | +| byte-heavy | 500 | 22.9 ms | 46 µs | 18k | + +### 6.2 What it says + +- **Absolute (H-A):** one broadcast reaches all 500 clients in ~21–23 ms + (~42–46 µs/delivery, ~18–22k deliveries/s). For a dashboard-style workload + (a topic a few hundred browsers watch, updating a few times per second) that + is comfortable. +- **System-level scaling (H-B, with the real bus):** per-delivery falls then + flattens (~42–55 µs) as fixed costs amortize; total latency is roughly linear + (5.1 ms → 20.8 ms from N=100 → 500). No super-linear knee — the microbench + could not see the `ClientManager`/`DashMap` fan-out, and end-to-end it shows no + O(N²) pathology. +- **The encode share — the headline.** The microbench made the byte-heavy encode + look dominant (7.6 ms/broadcast at N=500). End-to-end, byte-heavy fan-out + (22.9 ms) is only **~2 ms slower** than small (20.8 ms) at N=500 — the ~7 ms + isolated encode difference **compresses to ~2 ms** once embedded in the real + system, because the per-subscriber encodes overlap with socket I/O and spread + across the multi-threaded runtime instead of running back-to-back on one core. + Per-delivery is dominated by the socket write, scheduling, and client-side + receive (~42 µs), not the encode (~1–15 µs isolated). + +So the isolated encode wins do **not** translate proportionally. Improvement A's +realistic end-to-end payoff is **~9% for byte-heavy at high fan-out** (removing +~2 ms of a ~23 ms broadcast) and negligible for small payloads — worthwhile for +byte-heavy / bandwidth-bound workloads, not a critical hotspot. This vindicates +the original intuition that the socket write dominates at the system level. + +### 6.3 Harness caveats + +- **4-core VM.** 500 client tasks contend with the server for cores, so absolute + per-delivery is inflated by client-side contention — not pure server fan-out. + The **relative** small-vs-byte-heavy gap is the robust signal. +- **Latency, not sustained throughput.** Rounds are serialized (set → await full + delivery → next), so `msgs/s` is the effective rate under isolated broadcasts, + not pipelined load; true sustained throughput would be higher. +- **Poll granularity.** The driver busy-polls with `yield_now` (small fixed + per-round overhead); p99 tails (up to ~150 ms) are scheduling pauses on the + loaded VM. + +--- + +## 7. Improvement A implemented and re-measured + +Improvement A was implemented in `AimxCodec`: the `Outbound::Event` arm now +serializes the frame scaffolding (`t`/`seq`/`topic`/`sub`) and **splices the +record payload verbatim**, instead of validating it into a `RawValue` via +`as_raw` on every encode. The payload is trusted-valid (record-serializer +output), so the per-encode O(payload) validation was redundant — and on a +fan-out subscription it ran once *per subscriber* over the shared payload. The +change is byte-identical to the previous wire (asserted by +`event_encode_splices_data_byte_for_byte`), benefits every transport, and also +removes one payload copy. + +### 7.1 Microbench — confirms the hoist landed + +`naive` calls the real codec, so it now reflects the change: + +| Shape (N=500) | naive before | naive after | validate_once | +|---|--:|--:|--:| +| small | 530 µs | 449 µs | 275 µs | +| byte-heavy | 7587 µs | **540 µs** | 504 µs | + +Byte-heavy `naive` dropped **~14×** and now tracks `validate_once` — the +per-encode validation is gone. Small barely moved (validation was cheap there). + +### 7.2 End-to-end socket harness — the real payoff + +| Shape (N=500) | before | after | Δ | +|---|--:|--:|--:| +| small | 20.8 ms | 20.6 ms | ~noise | +| byte-heavy | 22.9 ms | 22.2 ms | ~0.7 ms (~3%) | +| byte-heavy − small gap | 2.1 ms | 1.6 ms | ~0.5 ms closed | + +Per-delivery, byte-heavy improved ~1 µs (of ~44 µs). So the **14× microbench win +translates to ~3% end-to-end** — even below the ~9% H-A estimate. Removing the +encode validation closed only ~0.5 ms of the ~2 ms byte-heavy-vs-small gap; the +**remaining gap is payload size** (3.7 KB more to write and parse per delivery), +not encode. The improvement is real but small enough to sit near this VM's +run-to-run noise. + +**Verdict:** Improvement A is a clean, byte-identical, safe change worth keeping +— it removes redundant work on *every* transport and cuts an allocation — but +its end-to-end fan-out benefit is marginal, consistent with H-A. It is justified +as hygiene, not as a fix for a hot path that turned out not to be hot. + +--- + +## 8. Conclusions + +1. **H-B: confirmed** — linear in both the isolated encode loop and, per §6, the + real `ClientManager` bus end-to-end. No O(N²) pathology. +2. **H-C: confirmed, and isolated.** The `validate_once` variant splits the win: + for byte-heavy, **~97% is the validation hoist (Improvement A)** and only ~3% + is the shared-suffix structure (Improvement B); for small payloads the two + are comparable (~56% / ~44%). +3. **H-A: answered — the isolated wins do not translate proportionally.** The + encode is a minor fraction of end-to-end fan-out (§6): per-delivery is + socket/scheduling-dominated, and the byte-heavy encode's ~7 ms isolated cost + compresses to ~2 ms end-to-end. 500-client fan-out at ~21 ms/broadcast is + comfortable for dashboard-style workloads. +4. **Improvement A implemented (§7).** The per-subscriber validation was removed + from `AimxCodec` — a clean, byte-identical change. Microbench confirms it + (byte-heavy encode ~14× faster), but end-to-end it moved fan-out only ~3% + (~0.7 ms of ~23 ms), *below* the ~9% estimate: the residual byte-heavy penalty + is payload size (bytes on the wire), not encode. Keep it as hygiene, not as a + performance fix. +5. **Improvement B (shared-suffix) is not worth its complexity** given H-A and + §7: its extra gain over A is small in isolation and vanishes end-to-end. + +--- + +## 9. Threats to validity + +- **Virtualized host with high run-to-run variance.** Docker Desktop VM; wide + Criterion CIs, and absolutes shift materially between runs — small `naive`/500 + measured 1.24 ms in one run and 530 µs in another. Cross-run absolutes are + unreliable; the decomposition uses a single run where all strategies were + measured back-to-back, and the **proportions** (A vs B share) are the robust + result, not the precise multipliers. +- **Excludes transport.** No socket write, scheduling, `ClientManager`, or + backpressure — the microbench isolates encoding only. +- **Fast path trusts the payload.** `shared_suffix` performs no re-validation; + `validate_once` performs exactly one validation per broadcast, which is the + production-safe form of Improvement A. At N=1 that single validation can make + `validate_once` trail `naive`; at realistic fan-out (N ≫ 1) it amortizes away. + +--- + +## 10. Next steps + +1. ~~**Isolate A vs B.**~~ Done — the `validate_once` strategy measures the split + (§5.2): byte-heavy is ~97% validation hoist. +2. ~~**Run H-A.**~~ Done — the socket harness (§6) shows the encode is a minor + end-to-end fraction. +3. ~~**Implement Improvement A.**~~ Done (§7) — implemented in `AimxCodec` as a + verbatim payload splice (simpler than the anticipated once-per-broadcast + plumbing, since the payload is trusted-valid), byte-identical, ~14× faster + encode but only ~3% end-to-end. Kept as hygiene. +4. **Do not implement Improvement B (shared-suffix) on this evidence** — its + marginal gain over A is small in isolation and negligible end-to-end (§6.2, + §7.2). Revisit only if a concrete high-frequency small-payload exact-topic + workload appears. +5. **Optional guardrail follow-up.** The splice trusts the record serializer's + output; if defensiveness against a misconfigured custom serializer is wanted, + validate once at the record-serialize / broadcast boundary rather than per + encode — but this benchmark gives no performance reason to. From a5ac2e25daeaf33f56d82b51d8aeb4c15ab67b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 21:00:28 +0000 Subject: [PATCH 29/45] feat: enhance broadcast mechanism to surface dropped updates as seq gaps --- aimdb-websocket-connector/CHANGELOG.md | 10 ++++ .../src/server/client_manager.rs | 59 +++++++++++++++++-- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/aimdb-websocket-connector/CHANGELOG.md b/aimdb-websocket-connector/CHANGELOG.md index ad8b4fd9..26e1564f 100644 --- a/aimdb-websocket-connector/CHANGELOG.md +++ b/aimdb-websocket-connector/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **Fan-out broadcast drops are now observable as `seq` gaps.** When a + subscription's bounded channel is full, `ClientManager::broadcast` still drops + the update (slow-client protection) but now records it and folds the count + into the next delivered update's `skipped`, so a broadcast-stage drop surfaces + as a `seq` gap downstream — the same loss signal buffer lag and the connection + funnel already emit. Previously this drop happened upstream of where the pump + assigns `seq`, so a slow fan-out consumer silently under-reported its loss. + ### Security - **Subscribe ACL now checks pattern *containment*, not topic matching.** A diff --git a/aimdb-websocket-connector/src/server/client_manager.rs b/aimdb-websocket-connector/src/server/client_manager.rs index 9739b67a..c602e677 100644 --- a/aimdb-websocket-connector/src/server/client_manager.rs +++ b/aimdb-websocket-connector/src/server/client_manager.rs @@ -27,6 +27,13 @@ struct SubEntry { pattern: String, /// Bounded; `broadcast` drops on a full channel (slow-client protection). tx: mpsc::Sender, + /// Updates dropped (full channel) since the last one that got through, to be + /// folded into the next delivered [`SubUpdate::skipped`] so a broadcast-stage + /// drop still surfaces as a `seq` gap downstream — the same loss signal the + /// per-record buffer lag and the connection funnel already emit. Without + /// this, a drop *here* (upstream of where the pump assigns `seq`) would be + /// silent, and a slow fan-out consumer would under-report its loss. + dropped: AtomicU64, } /// Shared per-topic broadcast bus. Cloning is cheap (all clones share state). @@ -86,6 +93,7 @@ impl ClientManager { SubEntry { pattern: pattern.to_string(), tx, + dropped: AtomicU64::new(0), }, ); let stream = futures_util::stream::unfold(rx, |mut rx| async move { @@ -100,6 +108,11 @@ impl ClientManager { /// The payload and the topic tag are `Arc`-shared to every matching /// subscription (refcount bumps, no per-subscriber copies); the per-frame /// envelope is applied downstream by each connection's codec. + /// + /// A full channel drops the update (slow-client protection) but records it on + /// the subscription's [`dropped`](SubEntry::dropped) counter, folded into the + /// next delivered update's `skipped` so the loss still surfaces as a `seq` + /// gap — see that field. pub async fn broadcast(&self, topic: &str, payload_bytes: &[u8]) { let payload = Payload::from(payload_bytes); let tag: Arc = Arc::from(topic); @@ -108,13 +121,21 @@ impl ClientManager { if !topic_matches(&entry.pattern, topic) { continue; } + // Carry any drops accumulated since the last delivered update, so a + // broadcast-stage loss rides this update's `skipped` into a `seq` + // gap. Take them now; restore on failure so nothing is lost. + let carried = entry.dropped.swap(0, Ordering::Relaxed); + let update = SubUpdate::tagged(tag.clone(), payload.clone()).with_skipped(carried); // Bounded: drop on a full queue (slow-client protection), prune only // when the receiver is gone (stream dropped). - if let Err(mpsc::error::TrySendError::Closed(_)) = entry - .tx - .try_send(SubUpdate::tagged(tag.clone(), payload.clone())) - { - dead.push(*entry.key()); + match entry.tx.try_send(update) { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(_)) => { + // This update is dropped too: restore the carried count and + // add one for it, to ride the next delivered update. + entry.dropped.fetch_add(carried + 1, Ordering::Relaxed); + } + Err(mpsc::error::TrySendError::Closed(_)) => dead.push(*entry.key()), } } for id in dead { @@ -174,6 +195,34 @@ mod tests { assert!(stream.next().now_or_never().is_none()); } + #[tokio::test] + async fn full_channel_drops_surface_as_skipped_on_next_delivery() { + // Capacity 1: the second and third broadcasts have nowhere to go and are + // dropped, but the loss must ride the next delivered update's `skipped` + // so it becomes a `seq` gap downstream (not a silent hole). + let mgr = ClientManager::new(1); + let (_id, mut stream) = mgr.subscribe("#"); + + mgr.broadcast("t", b"1").await; // fills the one slot + mgr.broadcast("t", b"2").await; // full → dropped (counter = 1) + mgr.broadcast("t", b"3").await; // full → dropped (counter = 2) + + // First delivery is the update that got through, lossless. + let first = stream.next().await.expect("first update"); + assert_eq!(&first.data[..], b"1"); + assert_eq!(first.skipped, 0); + + // With the slot now free, the next broadcast is delivered and carries the + // two drops that happened while the channel was full. + mgr.broadcast("t", b"4").await; + let second = stream.next().await.expect("second update"); + assert_eq!(&second.data[..], b"4"); + assert_eq!( + second.skipped, 2, + "the two full-channel drops must be reported" + ); + } + #[tokio::test] async fn fan_out_to_n_subscribers() { let mgr = ClientManager::new(256); From baf1327933bba20bc429e92510c052e8ce160c9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 21:02:26 +0000 Subject: [PATCH 30/45] fix: update subscription topic syntax from 'sensors/#' to 'sensors.#' in examples --- aimdb-wasm-adapter/src/bindings.rs | 2 +- aimdb-wasm-adapter/src/react/useAimDb.tsx | 2 +- aimdb-wasm-adapter/src/ws_bridge.rs | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/aimdb-wasm-adapter/src/bindings.rs b/aimdb-wasm-adapter/src/bindings.rs index c9122208..a19c40ac 100644 --- a/aimdb-wasm-adapter/src/bindings.rs +++ b/aimdb-wasm-adapter/src/bindings.rs @@ -289,7 +289,7 @@ impl WasmDb { /// # Example (TypeScript) /// ```ts /// const bridge = db.connectBridge('wss://api.example.com/ws', { - /// subscribeTopics: ['sensors/#'], + /// subscribeTopics: ['sensors.#'], /// autoReconnect: true, /// }); /// bridge.onStatusChange((status) => console.log(status)); diff --git a/aimdb-wasm-adapter/src/react/useAimDb.tsx b/aimdb-wasm-adapter/src/react/useAimDb.tsx index cade45c9..dcbe0b5a 100644 --- a/aimdb-wasm-adapter/src/react/useAimDb.tsx +++ b/aimdb-wasm-adapter/src/react/useAimDb.tsx @@ -16,7 +16,7 @@ * { key: 'sensors.humidity.vienna', schemaType: 'humidity', buffer: 'SingleLatest' }, * ], * // Optional: connect to server - * bridge: { url: 'wss://api.cloud.aimdb.dev/ws', subscribeTopics: ['sensors/#'] }, + * bridge: { url: 'wss://api.cloud.aimdb.dev/ws', subscribeTopics: ['sensors.#'] }, * }}> * * diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index a12adb3d..cb8dd610 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -327,7 +327,7 @@ impl Dialer for WasmWsDialer { /// # Example (TypeScript) /// ```ts /// const bridge = db.connectBridge('wss://api.example.com/ws', { -/// subscribeTopics: ['sensors/#'], +/// subscribeTopics: ['sensors.#'], /// autoReconnect: true, /// }); /// bridge.onStatusChange((status) => updateIndicator(status)); @@ -564,11 +564,14 @@ impl WsBridge { } } -/// Human-readable form of the engine's 3-code error vocabulary. +/// Human-readable form of the engine's `RpcError` vocabulary. fn rpc_err_str(e: &RpcError) -> &'static str { match e { RpcError::NotFound => "not_found", RpcError::Denied => "denied", + RpcError::VersionMismatch => { + "version_mismatch (server rejected the client protocol version)" + } _ => "internal (engine stopped, disconnected, or server error)", } } From 7bcdcf99d40f77e81748c3592a1f66790fb88a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 21:05:45 +0000 Subject: [PATCH 31/45] fix: clarify documentation for SendFuture and its safety guarantees in wasm32 context --- aimdb-wasm-adapter/src/time.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/aimdb-wasm-adapter/src/time.rs b/aimdb-wasm-adapter/src/time.rs index 6396cd69..f8db712c 100644 --- a/aimdb-wasm-adapter/src/time.rs +++ b/aimdb-wasm-adapter/src/time.rs @@ -15,23 +15,22 @@ use core::pin::Pin; #[cfg(feature = "wasm-runtime")] use core::task::{Context, Poll}; -/// A wrapper that unconditionally implements `Send` for a future, so a +/// A wrapper that implements `Send` for a future **on wasm32 only**, so a /// JS-touching (`!Send`) future can satisfy the engine's `Send` bounds and box /// into the `dyn Future + Send` transport shapes. /// /// # Safety /// -/// `wasm-runtime` is a wasm32-only feature in practice: the JS-driven bridge it -/// gates (`ws_bridge`) holds `Rc`/`Closure` state that is genuinely `!Send`, so -/// the crate only compiles for `wasm32-unknown-unknown` once the feature is on -/// (the native `cargo test` lane runs `--no-default-features`, which compiles -/// this type and the bridge out entirely). That leaves -/// `wasm32-unknown-unknown` without the `atomics` / shared-memory proposal as -/// the only configuration where the `Send` impl is live — single-threaded by -/// construction, so the inner future is never actually sent between threads. -/// The `compile_error!` guard below is the backstop: enabling wasm threads -/// invalidates that reasoning and fails the build rather than silently -/// producing UB. +/// The blanket `unsafe impl Send for SendFuture` is sound only where +/// execution is single-threaded, so it is gated on `target_arch = "wasm32"` +/// (matching the per-type `unsafe impl Send` gates in `ws_bridge` and +/// `buffer`). The gate is the guarantee, not the incidental fact that the +/// crate's JS bridge fails to compile off-wasm: on any other target +/// `SendFuture` remains a plain `Future` that is only `Send` when `F: Send`, +/// the correct default. The `compile_error!` guard below is the backstop for +/// the one unsound wasm configuration — `wasm32` *with* the `atomics` / +/// shared-memory proposal (multi-threaded) — failing the build rather than +/// silently producing UB. #[cfg(feature = "wasm-runtime")] pub(crate) struct SendFuture(pub(crate) F); @@ -43,9 +42,10 @@ compile_error!( Disable the `atomics` target feature or provide a thread-safe implementation." ); -// SAFETY: see the type's docs — with `wasm-runtime` on, the only target this -// compiles for is wasm32 (without atomics), which is single-threaded. -#[cfg(feature = "wasm-runtime")] +// SAFETY: single-threaded wasm32 (without atomics) — the future is never +// actually sent between threads. Arch-gated so it can't satisfy a `Send` bound +// on a multi-threaded native target. +#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] unsafe impl Send for SendFuture {} #[cfg(feature = "wasm-runtime")] From ce990f14ab186b97d3cc0bd7f25830a60737bd4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Sat, 25 Jul 2026 21:30:51 +0000 Subject: [PATCH 32/45] feat: add CancelToken and CancelHandle for wasm-runtime support --- aimdb-wasm-adapter/src/buffer.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/aimdb-wasm-adapter/src/buffer.rs b/aimdb-wasm-adapter/src/buffer.rs index 9b3d86ac..0392e2f5 100644 --- a/aimdb-wasm-adapter/src/buffer.rs +++ b/aimdb-wasm-adapter/src/buffer.rs @@ -19,7 +19,9 @@ use alloc::boxed::Box; use alloc::collections::VecDeque; use alloc::rc::Rc; use alloc::vec::Vec; -use core::cell::{Cell, RefCell}; +#[cfg(feature = "wasm-runtime")] +use core::cell::Cell; +use core::cell::RefCell; use core::task::{Context, Poll, Waker}; use aimdb_core::buffer::{Buffer, BufferCfg, BufferReader, DynBuffer}; @@ -389,6 +391,7 @@ fn wake_all(wakers: &mut Vec) { // ============================================================================ /// Shared state between [`CancelToken`] and [`CancelHandle`]. +#[cfg(feature = "wasm-runtime")] struct CancelInner { cancelled: Cell, waker: RefCell>, @@ -399,6 +402,7 @@ struct CancelInner { /// Polled in a `futures_util::future::select` alongside `reader.recv()`. /// When [`CancelHandle::cancel()`] fires, the stored waker is woken and /// `is_cancelled()` returns `true`, causing the select to resolve. +#[cfg(feature = "wasm-runtime")] pub(crate) struct CancelToken { inner: Rc, } @@ -407,6 +411,7 @@ pub(crate) struct CancelToken { /// /// Calling [`cancel()`](CancelHandle::cancel) sets the flag and wakes the /// subscription task so it exits immediately — even if `recv()` is blocked. +#[cfg(feature = "wasm-runtime")] pub(crate) struct CancelHandle { inner: Rc, } @@ -414,16 +419,17 @@ pub(crate) struct CancelHandle { // SAFETY: wasm32 is single-threaded — no concurrent access possible. // Gated to wasm32 — `buffer.rs` is not feature-gated and compiles on any // host target, where `Rc`-backed types are not actually Send/Sync. -#[cfg(target_arch = "wasm32")] +#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] unsafe impl Send for CancelToken {} -#[cfg(target_arch = "wasm32")] +#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] unsafe impl Sync for CancelToken {} -#[cfg(target_arch = "wasm32")] +#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] unsafe impl Send for CancelHandle {} -#[cfg(target_arch = "wasm32")] +#[cfg(all(feature = "wasm-runtime", target_arch = "wasm32"))] unsafe impl Sync for CancelHandle {} /// Create a linked cancel token/handle pair. +#[cfg(feature = "wasm-runtime")] pub(crate) fn cancel_pair() -> (CancelToken, CancelHandle) { let inner = Rc::new(CancelInner { cancelled: Cell::new(false), @@ -437,6 +443,7 @@ pub(crate) fn cancel_pair() -> (CancelToken, CancelHandle) { ) } +#[cfg(feature = "wasm-runtime")] impl CancelToken { /// Returns `true` if [`CancelHandle::cancel()`] has been called. pub(crate) fn is_cancelled(&self) -> bool { @@ -449,6 +456,7 @@ impl CancelToken { } } +#[cfg(feature = "wasm-runtime")] impl CancelHandle { /// Signal cancellation and wake the subscription task. /// From 0d35d19fe76a741fcdb49ff9f70de74d0cadb122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 27 Jul 2026 19:59:24 +0000 Subject: [PATCH 33/45] fix: ensure non-terminal `#` in grants only covers its suffix --- aimdb-core/CHANGELOG.md | 1 + aimdb-core/src/session/topic_match.rs | 259 +++++++++++++++---- aimdb-websocket-connector/CHANGELOG.md | 8 + aimdb-websocket-connector/src/server/auth.rs | 13 + 4 files changed, 231 insertions(+), 50 deletions(-) diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index 0aac6865..40e8d9fe 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -86,6 +86,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A non-terminal `#` no longer matches everything after it (`topic_matches`, `pattern_contains`).** Both walks returned `true` the moment they saw a `#`, so every segment following it was dead text: `topic_matches("a.#.secret", "a.public")` and `pattern_contains("a.#.secret", "a.public")` were both `true`, despite the module documenting `#` as usable at any position. On the ACL path this widened coverage — a `Permissions::can_subscribe` grant of that shape behaved like a blanket `a.#` grant over the whole subtree. `#` now absorbs **zero or more** segments with the pattern continuing afterwards (RabbitMQ topic-exchange semantics, as documented), so `a.#.secret` matches `a.secret` and `a.b.secret` but not `a.public`. Both functions share one greedy walk that retains a single `#` backtrack point, bounding it at `O(|pattern| · |subject|)` — relevant because `pattern_contains`'s subject is the client-supplied subscription pattern. Grants and subscriptions using a trailing `#` (`a.#`, `#`) or `*` are unaffected; the change only ever tightens what an interior `#` admits. - **AimX protocol doc rot cleaned up; `remote::PROTOCOL_VERSION` corrected to `"2.0"` and exported.** The `remote` module docs claimed "AimX v1" and linked a spec file that no longer exists; they now describe the v2 NDJSON tagged-frame wire and point at `crate::session::aimx` / `docs/design/remote-access-via-connectors.md`. The AimX dispatch's Welcome uses the constant instead of a hardcoded `"2.0"` (same bytes on the wire). The dead, never-exported v1 `Message` untagged envelope and its helpers were removed from `remote::protocol`. Also de-advertised Kafka/HTTP connector semantics from `ConnectorUrl` docs (the parser is scheme-agnostic; those connectors never existed) and updated the `connector` module docs from the removed `.link()` API to `.link_to()`/`.link_from()`. - **`build()` reports a missing runtime alongside every other configuration error (issue #133 contract).** The missing-runtime check no longer short-circuits: it is collected as a `ConfigError` and returned in the one `DbError::InvalidConfiguration` with all other findings (previously the collected errors were silently dropped and only a `RuntimeError` surfaced). The error type for a runtime-less build changes accordingly from `DbError::RuntimeError` to `DbError::InvalidConfiguration`. diff --git a/aimdb-core/src/session/topic_match.rs b/aimdb-core/src/session/topic_match.rs index 50f80c92..8d35c08c 100644 --- a/aimdb-core/src/session/topic_match.rs +++ b/aimdb-core/src/session/topic_match.rs @@ -9,19 +9,24 @@ //! The one segment separator is `.` — AimDB record keys are dot-delimited //! (`temp.vienna`, `app.config`), so wildcards split on `.`. The grammar //! (dot segments, `*` single-level, `#` multi-level) is RabbitMQ topic-exchange -//! semantics. `/` is an ordinary character here — it belongs to external broker -//! addresses (`mqtt://sensors/temp/x`), not to AimDB's subscription grammar. +//! semantics: `#` absorbs **zero or more** segments and the pattern continues +//! after it, so a `#` in the middle still has to line its suffix up. `/` is an +//! ordinary character here — it belongs to external broker addresses +//! (`mqtt://sensors/temp/x`), not to AimDB's subscription grammar. + +use core::str::Split; /// Returns `true` if `topic` matches `pattern`. /// /// Wildcard conventions over **dot-separated** segments: /// -/// | Pattern | Semantics | -/// |----------|-----------------------------------| -/// | `#` | Multi-level wildcard (all topics) | -/// | `a.#` | Everything under `a.` | -/// | `a.*.c` | Single-level wildcard in segment | -/// | `a.b.c` | Exact match | +/// | Pattern | Semantics | +/// |----------|-----------------------------------------| +/// | `#` | Multi-level wildcard (all topics) | +/// | `a.#` | `a` and everything under `a.` | +/// | `a.#.c` | `a.c` and any `c` at any depth under it | +/// | `a.*.c` | Single-level wildcard in segment | +/// | `a.b.c` | Exact match | pub fn topic_matches(pattern: &str, topic: &str) -> bool { // Fast path: exact match if pattern == topic { @@ -34,8 +39,8 @@ pub fn topic_matches(pattern: &str, topic: &str) -> bool { } // `prefix.#` matches everything under prefix — only when prefix is literal - // (no wildcards in the prefix). When wildcards are present, fall through to - // the segment loop which handles `#` at any position. + // (no wildcards in the prefix). Same answer as the general walk below, just + // without splitting: this is the shape the WS fan-out matches per broadcast. if let Some(prefix) = pattern.strip_suffix(".#") { if !prefix.contains('*') && !prefix.contains('#') { return topic.starts_with(prefix) @@ -44,17 +49,65 @@ pub fn topic_matches(pattern: &str, topic: &str) -> bool { } } - // Segment-by-segment matching with `*` single-level wildcard - let mut pattern_parts = pattern.split('.'); - let mut topic_parts = topic.split('.'); + // A topic segment is always a literal, so a pattern segment covers it when + // it is that same literal or the single-level wildcard. + match_segments(pattern, topic, |p, t| p == "*" || p == t) +} + +/// Segment walk shared by [`topic_matches`] and [`pattern_contains`]: the two +/// differ only in when one `pattern` segment covers one `subject` segment, which +/// is what `covers` decides. `#` is handled here because it is the one construct +/// that spans segments. +/// +/// `#` absorbs **zero or more** subject segments and the pattern *continues* +/// after it — it is not a "match everything from here" escape hatch. That +/// distinction is load-bearing: returning `true` at the `#` would make the +/// suffix of `a.#.secret` dead text and let it cover `a.public`. +/// +/// Only the most recent `#` is retained as a backtrack point. That is the +/// standard glob trick, and it is what bounds this at `O(|pattern| · |subject|)` +/// rather than exponential — worth keeping deliberate, since `subject` is the +/// client-supplied pattern on the [`pattern_contains`] ACL path. +fn match_segments(pattern: &str, subject: &str, covers: F) -> bool +where + F: Fn(&str, &str) -> bool, +{ + let mut pat = pattern.split('.'); + let mut sub = subject.split('.'); + // The pattern tail following the most recent `#`, paired with the subject + // position that `#` currently stops absorbing at. `None` until a `#` is seen. + let mut resume: Option<(Split<'_, char>, Split<'_, char>)> = None; loop { - match (pattern_parts.next(), topic_parts.next()) { - (Some("#"), _) => return true, - (Some("*"), Some(_)) => {} // single-level wildcard — consume one segment - (Some(p), Some(t)) if p == t => {} // literal match - (None, None) => return true, // both exhausted at the same time - _ => return false, + let mut pat_rest = pat.clone(); + let mut sub_rest = sub.clone(); + match (pat_rest.next(), sub_rest.next()) { + // `#` starts out absorbing nothing; `resume` is how it grows. + (Some("#"), _) => { + pat = pat_rest; + resume = Some((pat.clone(), sub.clone())); + } + (Some(p), Some(s)) if covers(p, s) => { + pat = pat_rest; + sub = sub_rest; + } + // Both exhausted together — every segment was covered. + (None, None) => return true, + // Mismatch, or one side ran out while the other still has segments. + // The only way forward is to let the most recent `#` swallow one + // more subject segment and retry its suffix from there. + _ => { + let Some((pat_after_hash, sub_at_hash)) = resume.take() else { + return false; // no `#` to fall back on + }; + let mut grown = sub_at_hash; + if grown.next().is_none() { + return false; // `#` has nothing left to absorb + } + pat = pat_after_hash.clone(); + sub = grown.clone(); + resume = Some((pat_after_hash, grown)); + } } } } @@ -82,40 +135,41 @@ pub fn is_wildcard(pattern: &str) -> bool { /// [`topic_matches`]`(grant, requested)`, so it is a safe drop-in for an ACL /// that must accept both concrete and wildcard subscription requests. /// -/// Grammar is the same dot-separated `*` (single-level) / `#` (multi-level, and -/// as `grant`'s tail, zero-or-more) set as [`topic_matches`]. +/// Grammar is the same dot-separated `*` (single-level) / `#` (multi-level, +/// zero-or-more) set as [`topic_matches`]. A grant's `#` absorbs zero or more +/// *requested* segments and the grant continues after it, so an interior `#` +/// constrains rather than blanket-allows: `a.#.secret` contains `a.b.secret` +/// but not `a.public`. +/// +/// # Conservative by construction +/// +/// This decides containment with a single left-to-right walk, which is sound +/// (it never reports containment that does not hold) but not complete. Deciding +/// the remaining cases means reasoning about every expansion of the *request's* +/// `#`, and that ∀-branch is exactly the intricate, blowup-prone code an ACL on +/// client-supplied input should not carry. +/// +/// The gap is confined to a grant whose `*` meets a request's `#`, where a later +/// segment would have ruled out the short expansions — e.g. `a.*.#` does contain +/// `a.#.a` (every topic the request admits has ≥2 segments), but this returns +/// `false`. Such a pair denies access that could have been allowed; it never +/// allows access that should have been denied. Grants without a `*`/`#` +/// adjacency — every ordinary shape, including `a.#`, `a.*`, `a.#.b`, `#` — are +/// decided exactly. pub fn pattern_contains(grant: &str, requested: &str) -> bool { - let mut g = grant.split('.'); - let mut r = requested.split('.'); - loop { - match (g.next(), r.next()) { - // A `#` in the grant absorbs the entire remaining request tail - // (including an already-exhausted one, e.g. `a.#` contains `a`). - (Some("#"), _) => return true, - // Both exhausted together: an exact structural cover. - (None, None) => return true, - // Request reaches past where the grant stops covering. - (None, Some(_)) => return false, - // Grant still requires ≥1 concrete segment the request never yields. - (Some(_), None) => return false, - // A grant `*` covers exactly one segment. A request `#` here could - // expand to zero or many segments, so `*` cannot contain it; a - // request `*` or literal is exactly one segment and is covered. - (Some("*"), Some(rs)) => { - if rs == "#" { - return false; - } - } + match_segments(grant, requested, |g, r| { + if g == "*" { + // A grant `*` covers exactly one segment. A request `#` could expand + // to zero or many, so `*` cannot contain it; a request `*` or literal + // is exactly one segment and is covered. + r != "#" + } else { // A grant literal covers only itself: the request segment must be // that same literal (a request `*`/`#` here would reach topics the - // literal doesn't, so it is not contained). - (Some(gs), Some(rs)) => { - if gs != rs { - return false; - } - } + // literal doesn't). A grant `#` never arrives here — the walk owns it. + g == r } - } + }) } #[cfg(test)] @@ -184,6 +238,45 @@ mod tests { assert!(!topic_matches("a.*.c.#", "a.b.x.d")); } + #[test] + fn non_terminal_hash_still_matches_its_suffix() { + // Regression: `#` used to return `true` on sight, which made every + // segment after it dead text — `a.#.secret` matched *anything* under + // `a`. It absorbs zero or more segments and the suffix must still line up. + assert!(!topic_matches("a.#.secret", "a.public")); + assert!(!topic_matches("a.#.secret", "a.b.c.public")); + assert!(!topic_matches("a.#.secret", "b.secret")); // prefix still binds + assert!(!topic_matches("a.#.secret", "a.secret.tail")); + // Zero-or-more: the `#` may absorb nothing, one segment, or many. + assert!(topic_matches("a.#.secret", "a.secret")); + assert!(topic_matches("a.#.secret", "a.b.secret")); + assert!(topic_matches("a.#.secret", "a.b.c.d.secret")); + } + + #[test] + fn hash_backtracks_to_find_a_later_suffix() { + // The suffix appears twice: the `#` has to keep growing past the first + // occurrence to line the pattern up on the last one. + assert!(topic_matches("a.#.c", "a.c.b.c")); + assert!(!topic_matches("a.#.c", "a.c.b.d")); + // Several `#` in one pattern each need their own extension. + assert!(topic_matches("a.#.b.#.c", "a.x.y.b.z.c")); + assert!(!topic_matches("a.#.b.#.c", "a.x.y.b.z.d")); + // A trailing `#` after an interior one still absorbs the remainder. + assert!(topic_matches("a.#.b.#", "a.x.b.y.z")); + assert!(topic_matches("a.#.b.#", "a.b")); // both absorb zero + assert!(!topic_matches("a.#.b.#", "a.x.y")); + } + + #[test] + fn hash_does_not_match_a_shorter_topic_than_its_suffix() { + // `#` absorbing zero segments must not let the pattern outrun the topic. + assert!(!topic_matches("a.#.b.c", "a.b")); + assert!(!topic_matches("#.a", "b")); + assert!(topic_matches("#.a", "a")); // leading `#` absorbs zero + assert!(topic_matches("#.a", "x.y.a")); + } + #[test] fn slash_is_not_a_separator() { // `/` is an ordinary character — a slash key is one literal segment, so a @@ -244,17 +337,83 @@ mod tests { assert!(!pattern_contains("sensors.temp", "sensors.#")); } + #[test] + fn containment_honours_a_non_terminal_hash_in_the_grant() { + // Regression: the grant's `#` used to return `true` on sight, so a grant + // written to reach only `secret` leaves widened the ACL to the whole + // subtree. The suffix after the `#` must still constrain the request. + assert!(!pattern_contains("a.#.secret", "a.public")); + assert!(!pattern_contains("a.#.secret", "a.b.public")); + assert!(!pattern_contains("a.#.secret", "a.#")); // no escalation to the subtree + assert!(!pattern_contains("a.#.secret", "a.*")); + // What the grant genuinely covers stays covered. + assert!(pattern_contains("a.#.secret", "a.secret")); + assert!(pattern_contains("a.#.secret", "a.b.secret")); + assert!(pattern_contains("a.#.secret", "a.#.secret")); + assert!(pattern_contains("a.#.secret", "a.b.#.secret")); + } + + #[test] + fn containment_denies_requests_reaching_past_an_interior_hash() { + // A request `#` where the grant demands a concrete tail is unbounded and + // must be denied — it would reach topics the grant's suffix excludes. + assert!(!pattern_contains("a.#.b", "a.#")); + assert!(!pattern_contains("a.#.b", "#")); + assert!(!pattern_contains("a.#.b.c", "a.b")); + // A `*` cannot stand in for a request `#`, before or after a grant `#`. + assert!(!pattern_contains("a.#.*", "a.#")); + assert!(pattern_contains("a.#.*", "a.b.c")); + assert!(!pattern_contains("a.#.*", "a")); + } + + #[test] + fn containment_is_conservative_where_a_grant_star_meets_a_request_hash() { + // Pins the documented soundness/completeness trade-off so a future change + // has to be deliberate. `a.*.#` genuinely contains `a.#.a` — every topic + // the request admits has ≥2 segments, which is exactly what `*.#` + // requires — but deciding that needs reasoning over every expansion of + // the request's `#`, which this walk does not do. It denies instead. + assert!(!pattern_contains("a.*.#", "a.#.a")); + assert!(!pattern_contains("a.#.*", "a.b.#")); + // The direction that matters: the denial is fail-closed, so no topic the + // request admits is reachable in a way the grant would not have allowed. + // (Sanity: the grant really is the broader pattern here.) + for topic in ["a.a", "a.b.a", "a.b.c.a"] { + assert!(topic_matches("a.#.a", topic)); + assert!(topic_matches("a.*.#", topic)); + } + // Ordinary grants — no `*`/`#` adjacency — stay exact, not conservative. + assert!(pattern_contains("a.#", "a.b.#")); + assert!(pattern_contains("a.#.b", "a.x.b")); + assert!(pattern_contains("a.*", "a.b")); + } + #[test] fn containment_matches_topic_matches_on_concrete_requests() { // For a concrete (wildcard-free) request, containment must be exactly // `topic_matches` — the safe-drop-in property the ACL relies on. - let grants = ["#", "sensors.#", "sensors.*", "sensors.temp", "*.temp"]; + let grants = [ + "#", + "sensors.#", + "sensors.*", + "sensors.temp", + "*.temp", + // Interior `#` — the shape that used to short-circuit both walks. + "sensors.#.temp", + "#.temp", + "sensors.#.*", + "a.#.b.#.c", + ]; let topics = [ "sensors.temp", "sensors.temp.vienna", "sensors", "commands.on", "a.temp", + "sensors.rack.temp", + "sensors.rack.shelf.temp", + "a.x.b.y.c", + "a.b.c", ]; for g in grants { for t in topics { diff --git a/aimdb-websocket-connector/CHANGELOG.md b/aimdb-websocket-connector/CHANGELOG.md index 26e1564f..4880d71a 100644 --- a/aimdb-websocket-connector/CHANGELOG.md +++ b/aimdb-websocket-connector/CHANGELOG.md @@ -19,6 +19,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- **A grant with a non-terminal `#` no longer covers its whole subtree.** Both + `Permissions::can_subscribe` (via `pattern_contains`) and + `Permissions::can_write` (via `topic_matches`) stopped matching at the first + `#`, so every segment after it was ignored: a grant of `tenant.#.secret` + admitted `tenant.public`. `#` now absorbs zero or more segments with the + suffix still applying, so that grant covers `tenant.secret` and + `tenant.a.b.secret` only. Grants using a trailing `#` (`sensors.#`, `#`) or + `*` are unaffected — the change only tightens what an interior `#` admits. - **Subscribe ACL now checks pattern *containment*, not topic matching.** A granted subscribe pattern is honored only if it covers the *whole* pattern the client requests. Previously the check matched the requested pattern as if it diff --git a/aimdb-websocket-connector/src/server/auth.rs b/aimdb-websocket-connector/src/server/auth.rs index 62c0ce43..b2b75505 100644 --- a/aimdb-websocket-connector/src/server/auth.rs +++ b/aimdb-websocket-connector/src/server/auth.rs @@ -225,6 +225,19 @@ mod tests { assert!(!p.can_subscribe("sensors.temp.vienna")); // deeper — denied } + #[test] + fn subscribe_grant_with_an_interior_hash_keeps_its_suffix() { + // A grant scoped to `secret` leaves at any depth must not admit the rest + // of the subtree: the `#` used to short-circuit the whole match, so every + // segment after it was ignored and this grant behaved like `tenant.#`. + let p = perms(&["tenant.#.secret"]); + assert!(p.can_subscribe("tenant.secret")); + assert!(p.can_subscribe("tenant.a.b.secret")); + assert!(!p.can_subscribe("tenant.public")); + assert!(!p.can_subscribe("tenant.a.b.public")); + assert!(!p.can_subscribe("tenant.#")); // no escalation to the subtree + } + #[test] fn subscribe_allows_requests_the_grant_actually_covers() { assert!(perms(&["#"]).can_subscribe("sensors.#")); From 443631ba2581cbbff759d374ad78130ab407593a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 27 Jul 2026 20:55:27 +0000 Subject: [PATCH 34/45] feat: implement late-join snapshot sequence numbering to improve loss detection --- aimdb-core/CHANGELOG.md | 16 ++ aimdb-core/src/session/aimx/codec.rs | 33 ++- aimdb-core/src/session/client.rs | 111 ++++++---- aimdb-core/src/session/mod.rs | 12 + aimdb-core/src/session/server.rs | 21 +- aimdb-core/tests/session_engine.rs | 206 +++++++++++++++++- aimdb-websocket-connector/CHANGELOG.md | 9 + aimdb-websocket-connector/tests/e2e.rs | 13 +- ...047-retire-ws-protocol-converge-on-aimx.md | 18 +- 9 files changed, 387 insertions(+), 52 deletions(-) diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index 40e8d9fe..196db135 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -31,6 +31,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 without them are unchanged on the wire). `Session::snapshot` became `Session::snapshots(topic) -> Vec<(String, Payload)>` (one per covered record). +- **Late-join snapshots are sequence-numbered and no longer lost silently.** + `Outbound::Snapshot` gains a required `seq` in the *same* space as the + subscription's events: `run_session` numbers the burst `1..=N` and + `pump_subscription` continues at `N + 1`. The client demux routes snapshots + through the identical gap accounting as events, so one that overruns the + consumer's `SUBSCRIBE_CHANNEL_CAP`-bounded sink is reported as + `SubUpdate::skipped` on the next delivered update — and a loss at the tail of + the burst surfaces on the first event. Previously the client discarded the + snapshot `try_send` result and snapshots carried no `seq`, so a wildcard + subscription over more than 256 matched records silently delivered only 256, + with no error, no gap, and no way for later event `seq`s to reveal which + initial states were missing — quietly breaking "one snapshot per matched + record". Also fixes two smaller leaks on that path: a snapshot dropped by an + encode failure is now counted as loss, and a snapshot for a subscription + whose receiver is gone now prunes the sub instead of lingering. + Wire: `snap` frames carry `"seq"`, and one without it is `Malformed`. - **`AimxCodec` learned the `subscribed` ack frame** (`{"t":"subscribed", "sub":S}`) for servers running `acks_subscribe:true` (the WebSocket connector); UDS/serial/TCP keep the implicit ack. A dedicated `AimxCodec` diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index fc9f2744..1ddd1f1a 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -232,10 +232,16 @@ impl EnvelopeCodec for AimxCodec { // `write_frame_splicing_data`. write_frame_splicing_data(out, &frame, &data) } - Outbound::Snapshot { sub, topic, data } => { + Outbound::Snapshot { + sub, + seq, + topic, + data, + } => { let raw = as_raw(&data)?; let mut frame = Frame::tagged("snap"); frame.sub = Some(sub); + frame.seq = Some(seq); frame.topic = Some(topic); frame.data = Some(raw); write_frame(out, &frame) @@ -308,6 +314,7 @@ impl EnvelopeCodec for AimxCodec { }), "snap" => Ok(Outbound::Snapshot { sub: f.sub.ok_or(CodecError::Malformed)?, + seq: f.seq.ok_or(CodecError::Malformed)?, topic: f.topic.ok_or(CodecError::Malformed)?, data: payload_of(f.data), }), @@ -511,21 +518,39 @@ mod tests { } #[test] - fn snapshot_roundtrip_carries_sub_and_topic() { + fn snapshot_roundtrip_carries_sub_seq_and_topic() { let frame = encode_outbound(Outbound::Snapshot { sub: "8", + seq: 3, topic: "temp.berlin", data: payload(r#"{"c":18.0}"#), }); match AimxCodec.decode_outbound(&frame).unwrap() { - Outbound::Snapshot { sub, topic, data } => { - assert_eq!((sub, topic), ("8", "temp.berlin")); + Outbound::Snapshot { + sub, + seq, + topic, + data, + } => { + assert_eq!((sub, seq, topic), ("8", 3, "temp.berlin")); assert_eq!(&data[..], br#"{"c":18.0}"#); } _ => panic!("expected Snapshot"), } } + /// A `snap` frame without `seq` is not decodable: snapshots share the + /// subscription's sequence space, so an unnumbered one would silently + /// defeat the client's gap accounting. + #[test] + fn snapshot_without_seq_is_malformed() { + let frame = br#"{"t":"snap","sub":"8","topic":"temp.berlin","data":1}"#; + assert!(matches!( + AimxCodec.decode_outbound(frame), + Err(CodecError::Malformed) + )); + } + #[test] fn subscribed_ack_roundtrip() { let frame = encode_outbound(Outbound::Subscribed { sub: "13" }); diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index 098123b0..b4c30f56 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -32,8 +32,9 @@ use crate::AimDb; /// Capacity of a subscription's client-side event sink. Bounded (was /// `unbounded`) so a slow consumer can't grow memory without limit under a fast /// wildcard set; matches the server's per-connection `EVENT_BUFFER`. On overflow -/// the run loop drops the event and lets the loss surface as a `seq` gap -/// (`SubUpdate::skipped`) on the next delivered update. +/// the run loop drops the update and lets the loss surface as a `seq` gap +/// (`SubUpdate::skipped`) on the next delivered update — for late-join snapshots +/// too, which a wildcard subscription can burst well past this cap. const SUBSCRIBE_CHANNEL_CAP: usize = 256; /// Client engine knobs. Durations are in **milliseconds** so the engine stays @@ -225,6 +226,49 @@ impl Drop for DrainOnExit<'_> { } } +/// Route one subscription update (snapshot or event) to its sink, folding any +/// loss since the last delivery into its [`SubUpdate::skipped`]. +/// +/// Server `seq` counts true production across the *whole* subscription — the +/// late-join snapshot burst (`1..=N`) and then its events (`N + 1`…), with +/// server-side buffer lag folded in via `+= skipped + 1` and funnel drops +/// bumping the counter before dropping. So one delta against the last delivered +/// seq captures every loss point, plus any prior local full-channel drop that +/// left `last_seq` behind. +fn deliver( + subs: &mut HashMap>>, + last_seq: &mut HashMap, + sub: &str, + seq: u64, + topic: Option>, + data: Payload, +) { + let prev = last_seq.get(sub).copied().unwrap_or(0); + let skipped = seq.saturating_sub(prev + 1); + // `None` here is a late update for a dropped sub — ignore. + if let Some(tx) = subs.get(sub) { + let update = SubUpdate { + topic, + data, + skipped, + }; + match tx.try_send(Ok(update)) { + // Delivered — advance the per-sub cursor. + Ok(()) => { + last_seq.insert(sub.to_string(), seq); + } + // Slow consumer: drop this update but leave `last_seq` so the + // shortfall folds into the next delivered update's `skipped`. + Err(e) if e.is_full() => {} + // Receiver gone — the sub was dropped; prune it. + Err(_) => { + subs.remove(sub); + last_seq.remove(sub); + } + } + } +} + /// What [`drive_connection`]'s `select_biased!` decided this iteration — extracted /// so the work runs after the arm futures' borrow of `conn` releases. enum ClientStep { @@ -423,43 +467,32 @@ where seq, topic, data, - }) => { - // Loss = the shortfall between the last delivered seq and - // this one. Server `seq` counts true production (buffer lag - // folded in via `+= skipped + 1`, funnel drops bump then - // drop), so one delta captures every loss point, plus any - // prior local full-channel drop that left `last_seq` behind. - let prev = last_seq.get(sub).copied().unwrap_or(0); - let skipped = seq.saturating_sub(prev + 1); - // `None` here is a late event for a dropped sub — ignore. - if let Some(tx) = subs.get(sub) { - let update = SubUpdate { - topic: topic.map(Arc::from), - data, - skipped, - }; - match tx.try_send(Ok(update)) { - // Delivered — advance the per-sub cursor. - Ok(()) => { - last_seq.insert(sub.to_string(), seq); - } - // Slow consumer: drop this event but leave - // `last_seq` so the shortfall folds into the next - // delivered update's `skipped`. - Err(e) if e.is_full() => {} - // Receiver gone — the sub was dropped; prune it. - Err(_) => { - subs.remove(sub); - last_seq.remove(sub); - } - } - } - } - Ok(Outbound::Snapshot { sub, topic, data }) => { - if let Some(tx) = subs.get(sub) { - let _ = tx.try_send(Ok(SubUpdate::tagged(Arc::from(topic), data))); - } - } + }) => deliver( + &mut subs, + &mut last_seq, + sub, + seq, + topic.map(Arc::from), + data, + ), + // Snapshots ride the same accounting as events (they share + // the subscription's `seq` space), so a late-join burst that + // overruns a slow consumer's sink is reported as `skipped` + // instead of vanishing — "one snapshot per matched record" + // stays checkable by the subscriber. + Ok(Outbound::Snapshot { + sub, + seq, + topic, + data, + }) => deliver( + &mut subs, + &mut last_seq, + sub, + seq, + Some(Arc::from(topic)), + data, + ), Ok(Outbound::Pong) => {} // Explicit subscribe ack — informational; the sink already exists. Ok(Outbound::Subscribed { .. }) => {} diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index ff911598..d587f7f4 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -316,6 +316,13 @@ pub enum Outbound<'a> { /// it like an [`Event`](Outbound::Event) (a wildcard subscription's /// snapshots carry topics that don't string-match the pattern key). sub: &'a str, + /// Monotonic sequence number, in the **same space** as this + /// subscription's [`Event`](Outbound::Event)s: the burst is numbered + /// `1..=N` and the first event continues at `N + 1`. So a snapshot lost + /// anywhere between here and the subscriber surfaces as a gap in the + /// next delivered update's [`SubUpdate::skipped`] — including a loss at + /// the tail of the burst, which the first event reveals. + seq: u64, /// Topic the snapshot is for. topic: &'a str, /// Unparsed record value. @@ -429,6 +436,11 @@ pub trait Session: Send { /// [`Outbound::Snapshot`] right after a successful /// [`subscribe`](Session::subscribe) and before the first event. Defaulted /// to empty (no snapshots). + /// + /// The engine numbers the returned burst `1..=N` and starts the event + /// stream at `N + 1`, so "one snapshot per matched record" is *auditable* + /// downstream rather than merely intended: any snapshot dropped in transit + /// shows up as [`SubUpdate::skipped`] on the next delivered update. fn snapshots(&mut self, topic: &str) -> Vec<(String, Payload)> { let _ = topic; Vec::new() diff --git a/aimdb-core/src/session/server.rs b/aimdb-core/src/session/server.rs index bbaec450..e794d8cc 100644 --- a/aimdb-core/src/session/server.rs +++ b/aimdb-core/src/session/server.rs @@ -220,14 +220,24 @@ pub async fn run_session( } } // Optional late-join snapshots (one per covered - // record), before the first event. + // record), before the first event. They share + // the subscription's `seq` space — numbered + // `1..=N` here, events continue at `N + 1` — so + // every snapshot is accounted for end-to-end: + // one dropped by a slow client sink (or skipped + // by the encode failure below) leaves a hole the + // client reports as `skipped`, and a loss at the + // tail of the burst surfaces on the first event. + let mut seq: u64 = 0; let mut send_failed = false; for (snap_topic, data) in session.snapshots(&topic) { + seq += 1; out.clear(); if codec .encode( Outbound::Snapshot { sub: &sub_id, + seq, topic: &snap_topic, data, }, @@ -250,6 +260,7 @@ pub async fn run_session( stream, event_tx.clone(), cancel_rx, + seq, ))); } Err(e) => { @@ -311,6 +322,11 @@ async fn send_reply_err( /// tagging each update with a monotonic `seq`. Ends when the stream finishes or /// the cancel handle is dropped/fired (Unsubscribe or connection teardown). /// +/// `start_seq` is the last sequence number the late-join snapshot burst used +/// (`0` when there were none), so events continue the *same* counter and a +/// snapshot lost at the tail of the burst still shows up as a gap on the first +/// event. +/// /// Returns its `sub_id` so [`run_session`] can prune the `cancels` entry for a /// pump that ended on its own; the Unsubscribe path already removed it, so that /// later prune is a no-op. @@ -319,13 +335,14 @@ async fn pump_subscription( mut stream: BoxStream<'static, SubUpdate>, tx: Sender, cancel: oneshot::Receiver<()>, + start_seq: u64, ) -> String { // Fuse the cancel receiver: a bare `oneshot::Receiver` reports // `is_terminated()` once its sender drops, and `select_biased!` skips // terminated arms — so the cancel would never fire. `Fuse` keeps the arm // polled until it actually resolves. let mut cancel = cancel.fuse(); - let mut seq: u64 = 0; + let mut seq: u64 = start_seq; loop { // Independent arms, so a direct `select_biased!` is fine here. let update = select_biased! { diff --git a/aimdb-core/tests/session_engine.rs b/aimdb-core/tests/session_engine.rs index e8797317..a450480a 100644 --- a/aimdb-core/tests/session_engine.rs +++ b/aimdb-core/tests/session_engine.rs @@ -195,8 +195,13 @@ impl EnvelopeCodec for LineCodec { utf8(&data)? ) } - Outbound::Snapshot { sub, topic, data } => { - format!("SNAP\n{}\n{}\n{}", sub, topic, utf8(&data)?) + Outbound::Snapshot { + sub, + seq, + topic, + data, + } => { + format!("SNAP\n{}\n{}\n{}\n{}", sub, seq, topic, utf8(&data)?) } Outbound::Pong => "PONG".to_string(), Outbound::Subscribed { sub } => format!("SUBSCRIBED\n{}", sub), @@ -250,9 +255,11 @@ impl EnvelopeCodec for LineCodec { } "SNAP" => { let (sub, r) = rest.split_once('\n').ok_or(CodecError::Malformed)?; + let (seq, r) = r.split_once('\n').ok_or(CodecError::Malformed)?; let (topic, data) = r.split_once('\n').unwrap_or((r, "")); Ok(Outbound::Snapshot { sub, + seq: seq.parse().map_err(|_| CodecError::Malformed)?, topic, data: payload_from(data), }) @@ -338,6 +345,85 @@ impl Session for EchoSession { } } +// =========================================================================== +// Late-join snapshot dispatch (Layer 3) — a wildcard subscription whose +// snapshot burst is larger than the client's per-subscription sink. +// =========================================================================== + +/// Records covered by the wildcard subscription below. Deliberately larger than +/// the client's private `SUBSCRIBE_CHANNEL_CAP` (256) so the late-join burst +/// *must* overrun a consumer that hasn't started draining yet. +const SNAPSHOT_RECORDS: usize = 300; + +/// The subscription's live event source, handed over once at `subscribe` so the +/// test can push events *after* draining the snapshot burst. +type EventFeed = Arc>>>; + +struct SnapshotDispatch { + feed: EventFeed, +} + +impl Dispatch for SnapshotDispatch { + fn authenticate<'a>( + &'a self, + _peer: &'a PeerInfo, + _first: Option<&'a [u8]>, + ) -> BoxFut<'a, Result> { + Box::pin(async { Ok(SessionCtx::default()) }) + } + + fn open(&self, _ctx: &SessionCtx) -> Box { + Box::new(SnapshotSession { + feed: self.feed.clone(), + }) + } +} + +struct SnapshotSession { + feed: EventFeed, +} + +impl Session for SnapshotSession { + fn call<'a>( + &'a mut self, + _method: &'a str, + params: Payload, + ) -> BoxFut<'a, Result> { + Box::pin(async move { Ok(params) }) + } + + fn snapshots(&mut self, _topic: &str) -> Vec<(String, Payload)> { + (1..=SNAPSHOT_RECORDS) + .map(|i| (format!("rec.{i}"), payload_from(&format!("snap#{i}")))) + .collect() + } + + fn subscribe<'a>( + &'a mut self, + _topic: &'a str, + ) -> BoxFut<'a, Result, RpcError>> { + let feed = self.feed.clone(); + Box::pin(async move { + // One subscription per test run; a second would find the feed taken. + let rx = feed.lock().unwrap().take().ok_or(RpcError::Internal)?; + let stream = + futures::stream::unfold( + rx, + |mut rx| async move { rx.recv().await.map(|u| (u, rx)) }, + ); + Ok(Box::pin(stream) as BoxStream<'static, SubUpdate>) + }) + } + + fn write<'a>( + &'a mut self, + _topic: &'a str, + _payload: Payload, + ) -> BoxFut<'a, Result<(), RpcError>> { + Box::pin(async { Ok(()) }) + } +} + // =========================================================================== // The exit-criterion test // =========================================================================== @@ -470,6 +556,122 @@ async fn failed_subscribe_ends_stream_via_ack() { server.abort(); } +/// A late-join snapshot burst larger than the client's per-subscription sink +/// must not vanish silently. Regression for the bug where the client demux +/// discarded the snapshot `try_send` result *and* snapshots carried no `seq`: a +/// wildcard subscription over 300 records delivered exactly +/// `SUBSCRIBE_CHANNEL_CAP` (256) updates with no error and no gap, and because +/// event `seq` then restarted at 1, no later event could reveal which initial +/// states were lost — quietly breaking "one snapshot per matched record". +/// +/// Snapshots now share the subscription's `seq` space (`1..=N`, events continue +/// at `N + 1`), so every dropped snapshot is reported as `skipped` on the next +/// delivered update — including the tail of the burst, which the first event +/// accounts for. +#[tokio::test] +async fn oversized_snapshot_burst_surfaces_as_a_gap() { + let (listener, dialer) = transport_pair(); + let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel::(); + let dispatch = Arc::new(SnapshotDispatch { + feed: Arc::new(Mutex::new(Some(event_rx))), + }); + let server = tokio::spawn(serve( + listener, + Arc::new(LineCodec), + dispatch, + SessionConfig::default(), + )); + let (handle, client_fut) = run_client( + dialer, + LineCodec, + ClientConfig { + reconnect: false, + reconnect_delay: 10, + max_reconnect_delay: 10, + max_reconnect_attempts: 0, + keepalive_interval: None, + max_offline_queue: usize::MAX, + sends_hello: false, + }, + Arc::new(TestClock), + ); + let client = tokio::spawn(client_fut); + + let mut stream = handle.subscribe("rec.#").unwrap(); + + // RPC barrier. The server serves frames in order and emits the whole burst + // inline while handling the Subscribe, so this reply is the *last* frame + // behind all 300 snapshots: once it resolves, the client demux has already + // pushed every snapshot at a sink nothing has drained yet — a deterministic + // overflow rather than a timing race. + let reply = handle.call("echo", payload_from("barrier")).await.unwrap(); + assert_eq!(&*reply, b"barrier"); + + // Drain whatever actually landed in the sink. + let mut delivered = Vec::new(); + while let Ok(next) = tokio::time::timeout(Duration::from_millis(50), stream.next()).await { + let item = next.expect("stream must stay open while the subscription lives"); + delivered.push(item.expect("a snapshot burst must not be a terminal error")); + } + + // Guard against a vacuous pass: raising SUBSCRIBE_CHANNEL_CAP past + // SNAPSHOT_RECORDS would make the burst fit and test nothing. + assert!( + delivered.len() < SNAPSHOT_RECORDS, + "burst must overrun the sink for this regression to mean anything; \ + all {} snapshots fit — raise SNAPSHOT_RECORDS above SUBSCRIBE_CHANNEL_CAP", + delivered.len() + ); + + // What did land is the contiguous head of the burst, in order and untagged + // by loss — the drops are all at the tail. + for (i, update) in delivered.iter().enumerate() { + assert_eq!( + update.topic.as_deref(), + Some(format!("rec.{}", i + 1).as_str()), + "snapshot {i} should carry its concrete record topic, in burst order" + ); + assert_eq!(&*update.data, format!("snap#{}", i + 1).as_bytes()); + assert_eq!( + update.skipped, 0, + "the delivered head of the burst is contiguous" + ); + } + + // The sink is empty again, so one live event gets through — and it must + // report every snapshot lost at the tail of the burst. Pre-fix this event + // arrived with `skipped == 0`. + event_tx + .send(SubUpdate::new(payload_from("live"))) + .expect("subscription stream should still be feeding"); + let event = tokio::time::timeout(Duration::from_secs(2), stream.next()) + .await + .expect("the live event must arrive") + .expect("stream must still be open") + .expect("a live event must not be a terminal error"); + + assert_eq!(&*event.data, b"live"); + assert_eq!( + event.skipped, + (SNAPSHOT_RECORDS - delivered.len()) as u64, + "the first event must report the snapshots dropped at the tail of the burst" + ); + + // The whole point, stated as a balance: every matched record is either + // delivered or accounted for as skipped — nothing goes missing quietly. + let reported: u64 = delivered.iter().map(|u| u.skipped).sum::() + event.skipped; + assert_eq!( + delivered.len() as u64 + reported, + SNAPSHOT_RECORDS as u64, + "every matched record must be either delivered or reported lost" + ); + + drop(handle); + drop(stream); + let _ = client.await; + server.abort(); +} + /// A subscription whose source stream *ends on its own* (the echo yields three /// updates, then completes) must free its slot against /// `max_subs_per_connection` once its pump drains. Regression for the bug where diff --git a/aimdb-websocket-connector/CHANGELOG.md b/aimdb-websocket-connector/CHANGELOG.md index 4880d71a..5c18d522 100644 --- a/aimdb-websocket-connector/CHANGELOG.md +++ b/aimdb-websocket-connector/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed (breaking, wire) + +- **`snap` frames now carry `seq`.** Late-join snapshots are numbered in the + subscription's sequence space (`1..=N`), and the first `event` continues at + `N + 1` rather than restarting at `1` — so a snapshot dropped by a slow + client is visible as a gap instead of vanishing (see `aimdb-core`). Clients + reading the golden frame shape must expect `"seq"` on `snap` and an event + sequence offset by the snapshot count. + ### Fixed - **Fan-out broadcast drops are now observable as `seq` gaps.** When a diff --git a/aimdb-websocket-connector/tests/e2e.rs b/aimdb-websocket-connector/tests/e2e.rs index 3e40bea2..450eff73 100644 --- a/aimdb-websocket-connector/tests/e2e.rs +++ b/aimdb-websocket-connector/tests/e2e.rs @@ -298,10 +298,12 @@ async fn server_late_join_snapshot() { ws_send(&mut c, json!({"t":"sub","id":4,"topic":"sensors.temp"})).await; assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"4"})); // The snapshot rides between the ack and the first event, tagged with the - // subscription that triggered it. + // subscription that triggered it and numbered in that subscription's `seq` + // space (events continue after the burst) so a dropped snapshot shows up as + // a gap rather than vanishing. assert_eq!( ws_recv(&mut c).await, - json!({"t":"snap","sub":"4","topic":"sensors.temp","data":99}) + json!({"t":"snap","sub":"4","seq":1,"topic":"sensors.temp","data":99}) ); } @@ -596,13 +598,16 @@ async fn golden_wire_frames() { assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"1"})); assert_eq!( ws_recv(&mut c).await, - json!({"t":"snap","sub":"1","topic":"t","data":5}) + json!({"t":"snap","sub":"1","seq":1,"topic":"t","data":5}) ); + // The event continues the snapshot burst's `seq` (one snapshot, so `seq:2`) + // — a single counter over the whole subscription, so loss anywhere in it is + // visible as a gap. inject(&db, "t", json!(42)); assert_eq!( ws_recv(&mut c).await, - json!({"t":"event","sub":"1","seq":1,"topic":"t","data":42}) + json!({"t":"event","sub":"1","seq":2,"topic":"t","data":42}) ); ws_send(&mut c, json!({"t":"ping"})).await; diff --git a/docs/design/047-retire-ws-protocol-converge-on-aimx.md b/docs/design/047-retire-ws-protocol-converge-on-aimx.md index 686a6229..42196305 100644 --- a/docs/design/047-retire-ws-protocol-converge-on-aimx.md +++ b/docs/design/047-retire-ws-protocol-converge-on-aimx.md @@ -73,7 +73,7 @@ AimX wire tags from `session/aimx/codec.rs`; ws-protocol messages from | Subscribe ack | `Subscribed{topics}` | `{"t":"subscribed","sub":S}` (**new**) | §3.2. | | Unsubscribe | `Unsubscribe{topics}` | `{"t":"unsub","sub":S}` | By sub id, not topic. | | Live data | `Data{topic,payload,ts}` | `{"t":"event","sub":S,"seq":N,"topic":T,"data":V}` | `topic` is **new**, present when the server tags it (always on WS; on wildcard subs elsewhere). Server-side `ts` is dropped — timestamps belong to the record layer / query results. `seq` is new for WS clients (drop detection). | -| Late-join snapshot | `Snapshot{topic,payload}` | `{"t":"snap","sub":S,"topic":T,"data":V}` | `sub` is **new** (§3.3). | +| Late-join snapshot | `Snapshot{topic,payload}` | `{"t":"snap","sub":S,"seq":N,"topic":T,"data":V}` | `sub` and `seq` are **new** (§3.3). `seq` shares the subscription's event sequence space: the burst is `1..=N` and the first `event` continues at `N+1`, so a dropped snapshot is a detectable gap. | | Client write | `Write{topic,payload}` | `{"t":"write","topic":T,"payload":V}` | Identical semantics (fire-and-forget, producer/arbiter path). | | Keepalive | `Ping`/`Pong` | `{"t":"ping"}` / `{"t":"pong"}` | Identical. | | Discovery | `ListTopics` over a raw socket | `record.list` req over a raw socket | UI's `discoverTopics` and `WasmDb.discover` reissue as AimX. | @@ -148,6 +148,22 @@ engine hook `Session::snapshot(topic) -> Option` becomes `Session::snapshots(topic) -> Vec<(String, Payload)>` ("one snapshot per matched record"); the default stays "none". +"One snapshot per matched record" is a claim the subscriber must be able to +*check*, because the burst is the one place a subscription can exceed its +bounded sink by a wide margin: a wildcard over N records emits N updates +before the consumer has run at all, and N is unbounded while the sink is +`SUBSCRIBE_CHANNEL_CAP` (256). Blocking the client demux until the consumer +drains is not an option — that loop also carries every other subscription and +pending RPC on the connection, so a subscriber that (say) awaits an RPC before +draining would deadlock itself. So snapshots take the same route as events: +they are numbered in the subscription's sequence space (`1..=N`, events +continuing at `N+1`), dropped on a full sink, and the shortfall folds into the +next delivered update's `SubUpdate::skipped`. Numbering them jointly with +events is what makes a loss at the *tail* of the burst recoverable — the first +event's `seq` reveals it. The residual blind spot is a tail loss on a +subscription that never fires another event; closing that needs an explicit +end-of-burst marker, deliberately deferred. + ### 3.4 Query / list result shapes (DECIDED) - `record.query` params stay `{name, limit, start, end}` (`name` accepts From eaab01d3d26d35d56d235b3bee93c21b97ad9110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 27 Jul 2026 21:13:19 +0000 Subject: [PATCH 35/45] feat: implement loss-aware subscription updates and reporting for AimDB client and CLI --- aimdb-client/CHANGELOG.md | 2 + aimdb-client/src/engine.rs | 182 ++++++++++++++++++++++++-- aimdb-client/src/lib.rs | 2 +- aimdb-client/tests/aimx_session.rs | 20 +++ aimdb-wasm-adapter/CHANGELOG.md | 7 + aimdb-wasm-adapter/README.md | 8 ++ aimdb-wasm-adapter/src/ws_bridge.rs | 67 ++++++++++ tools/aimdb-cli/CHANGELOG.md | 2 + tools/aimdb-cli/src/commands/watch.rs | 19 ++- tools/aimdb-cli/src/output/live.rs | 32 ++++- 10 files changed, 318 insertions(+), 23 deletions(-) diff --git a/aimdb-client/CHANGELOG.md b/aimdb-client/CHANGELOG.md index d55ea060..3e34aff2 100644 --- a/aimdb-client/CHANGELOG.md +++ b/aimdb-client/CHANGELOG.md @@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Loss-aware subscription stream: `AimxConnection::subscribe_updates`.** Yields `RecordUpdate { topic, value, skipped }`, exposing the delivery gap the engine already tracks (`SubUpdate::skipped`) — previously reachable only from raw `ClientHandle` streams, so ordinary client/CLI/bridge consumers could not tell a buffer overrun from an idle producer. `subscribe` and `subscribe_with_topics` are unchanged value-only views over the same stream. A payload that fails to decode is itself counted as a gap and folded into the next delivered update's `skipped`, rather than being dropped silently. + - **Transport-agnostic endpoint resolver — pick the transport at runtime via a `scheme://` URL (Issue #123, follow-up to #39 / #122).** New `endpoint` module: `parse_endpoint` (pure, feature-independent grammar) and `dial(url) -> Box` map an endpoint string to a transport `Dialer`, the way records already pick one for links. Schemes: `unix://PATH` / `uds://PATH`, a bare path (the `unix://` shorthand), and `serial://DEVICE?baud=N`. An unknown scheme — or one whose transport isn't compiled in — is rejected with a clear error. New `AimxConnection::connect_over(dialer)` / `connect_over_with_timeout` dial over an explicit `Dialer`, bypassing resolution. (Rides a new `impl Dialer for Box` in `aimdb-core`.) ### Fixed diff --git a/aimdb-client/src/engine.rs b/aimdb-client/src/engine.rs index df013ecb..98c56c7f 100644 --- a/aimdb-client/src/engine.rs +++ b/aimdb-client/src/engine.rs @@ -46,6 +46,35 @@ pub struct DrainResponse { pub count: usize, } +/// One decoded update from a subscription, carrying the delivery gap that +/// preceded it. +/// +/// Yielded by [`AimxConnection::subscribe_updates`]; the value-only +/// [`subscribe`](AimxConnection::subscribe) / +/// [`subscribe_with_topics`](AimxConnection::subscribe_with_topics) streams +/// project this down to the parts they expose. +#[derive(Debug, Clone, PartialEq)] +pub struct RecordUpdate { + /// Concrete record that fired. `Some` on wildcard subscriptions, which fan + /// in many records under one subscription; `None` when the server left the + /// event untagged (exact-topic subscribe). + pub topic: Option, + /// The decoded record value. + pub value: serde_json::Value, + /// Updates lost between the previous delivered update and this one — `0` + /// when delivery was lossless. Non-zero means the server-side buffer + /// overran a slow consumer (or the payload was undecodable), so the value + /// stream has a hole immediately before this item. + pub skipped: u64, +} + +impl RecordUpdate { + /// Whether updates were lost immediately before this one. + pub fn has_gap(&self) -> bool { + self.skipped > 0 + } +} + /// A live connection to an AimDB instance over the shared session engine. /// /// Holds the cheap-clone [`ClientHandle`] (use [`handle`](Self::handle) to issue @@ -210,34 +239,55 @@ impl AimxConnection { Ok(serde_json::from_slice(&reply)?) } + /// Subscribe to a record or topic pattern, yielding the full + /// [`RecordUpdate`] — decoded value, per-record topic, **and** the + /// [`skipped`](RecordUpdate::skipped) gap count. + /// + /// This is the loss-aware form: the engine tracks how many updates the + /// server-side buffer dropped between deliveries, and a consumer that cares + /// about completeness (a watcher, a mirror, an archiver) must observe it. + /// [`subscribe`](Self::subscribe) and + /// [`subscribe_with_topics`](Self::subscribe_with_topics) are value-only + /// views over this stream and silently discard the gap. + /// + /// Wildcards (`#`, `*`) are supported: one subscription fans in every + /// matching record, each update tagged with the record that fired. A + /// terminal rejection item ends the stream. Dropping the stream stops local + /// delivery. + pub fn subscribe_updates( + &self, + pattern: &str, + ) -> ClientResult> { + let raw = self.handle.subscribe(pattern).map_err(rpc_err)?; + Ok(decode_updates(raw)) + } + /// Subscribe to a record's updates. Returns a stream of decoded JSON values; /// the engine routes events back by the request id it owns, so there is no /// `subscription_id` to track. Dropping the stream stops local delivery. + /// + /// Delivery gaps are **not** visible here — use + /// [`subscribe_updates`](Self::subscribe_updates) to observe them. For the + /// per-record topic (wildcard subscriptions), see + /// [`subscribe_with_topics`](Self::subscribe_with_topics). pub fn subscribe(&self, name: &str) -> ClientResult> { - let raw = self.handle.subscribe(name).map_err(rpc_err)?; - // Decode each update's payload into a JSON value; drop any that fail to - // parse. A terminal rejection item ends the stream. For the per-record - // topic (wildcard subscriptions), see - // [`subscribe_with_topics`](Self::subscribe_with_topics). - let decoded = raw.filter_map(|u| async move { serde_json::from_slice(&u.ok()?.data).ok() }); - Ok(Box::pin(decoded)) + Ok(Box::pin(self.subscribe_updates(name)?.map(|u| u.value))) } /// Subscribe to a topic pattern (wildcards supported: `#`, `*`), yielding /// `(topic, value)` pairs. One wildcard subscription fans in every matching /// record; each update names the record that fired. The topic is `None` /// only when the server left the event untagged (exact-topic subscribe). + /// + /// Delivery gaps are **not** visible here — use + /// [`subscribe_updates`](Self::subscribe_updates) to observe them. pub fn subscribe_with_topics( &self, pattern: &str, ) -> ClientResult, serde_json::Value)>> { - let raw = self.handle.subscribe(pattern).map_err(rpc_err)?; - let decoded = raw.filter_map(|u| async move { - let u = u.ok()?; - let value = serde_json::from_slice(&u.data).ok()?; - Some((u.topic.as_deref().map(String::from), value)) - }); - Ok(Box::pin(decoded)) + Ok(Box::pin( + self.subscribe_updates(pattern)?.map(|u| (u.topic, u.value)), + )) } /// Fire-and-forget write to a record (no reply; routes through the server's @@ -323,6 +373,42 @@ impl Drop for AimxConnection { } } +/// Decode a raw engine subscription stream into [`RecordUpdate`]s. +/// +/// Each update's payload is decoded into a JSON value; a terminal rejection item +/// ends the stream. An undecodable payload never reaches the caller, so it is +/// itself a gap: its own `skipped` count plus one for the dropped item are +/// carried into the next delivered update rather than lost with it. +fn decode_updates( + raw: BoxStream<'static, Result>, +) -> BoxStream<'static, RecordUpdate> { + let decoded = raw + .scan(0u64, |carry, item| { + let update = match item { + Ok(u) => match serde_json::from_slice(&u.data) { + Ok(value) => { + let skipped = *carry + u.skipped; + *carry = 0; + Some(RecordUpdate { + topic: u.topic.as_deref().map(String::from), + value, + skipped, + }) + } + Err(_) => { + *carry += u.skipped + 1; + None + } + }, + // Terminal rejection: end the stream. + Err(_) => return futures::future::ready(None), + }; + futures::future::ready(Some(update)) + }) + .filter_map(futures::future::ready); + Box::pin(decoded) +} + /// Serialize a value into a record-value [`Payload`]. fn to_payload(value: &T) -> ClientResult { Ok(Payload::from(serde_json::to_vec(value)?.as_slice())) @@ -349,3 +435,71 @@ fn rpc_err(e: RpcError) -> ClientError { _ => ClientError::server_error("internal", "engine/transport failure", None), } } + +#[cfg(test)] +mod tests { + use super::*; + use aimdb_core::session::SubUpdate; + use serde_json::json; + + /// Build a raw engine stream from a fixed list of subscription items. + fn raw_stream( + items: Vec>, + ) -> BoxStream<'static, Result> { + Box::pin(futures::stream::iter(items)) + } + + fn ok_update(payload: &str, skipped: u64) -> Result { + Ok(SubUpdate::new(Payload::from(payload.as_bytes())).with_skipped(skipped)) + } + + #[tokio::test] + async fn gap_counts_reach_the_caller() { + let raw = raw_stream(vec![ok_update(r#"{"n":1}"#, 0), ok_update(r#"{"n":2}"#, 7)]); + let out: Vec<_> = decode_updates(raw).collect().await; + + assert_eq!(out.len(), 2); + assert_eq!(out[0].value, json!({ "n": 1 })); + assert_eq!(out[0].skipped, 0); + assert!(!out[0].has_gap()); + assert_eq!(out[1].value, json!({ "n": 2 })); + assert_eq!(out[1].skipped, 7, "the engine's gap count is preserved"); + assert!(out[1].has_gap()); + } + + #[tokio::test] + async fn undecodable_payloads_fold_into_the_next_gap() { + // A payload the caller never sees is itself a loss: it (and the gap it + // reported) must show up on the next delivered update, not vanish. + let raw = raw_stream(vec![ + ok_update(r#"{"n":1}"#, 0), + ok_update("not json", 3), + ok_update("also not json", 0), + ok_update(r#"{"n":2}"#, 1), + ]); + let out: Vec<_> = decode_updates(raw).collect().await; + + assert_eq!(out.len(), 2, "undecodable items are not delivered"); + assert_eq!(out[1].value, json!({ "n": 2 })); + // 3 reported + 1 dropped item + 1 dropped item + 1 reported = 6 + assert_eq!(out[1].skipped, 6); + } + + #[tokio::test] + async fn topics_ride_along_and_rejection_ends_the_stream() { + let raw = raw_stream(vec![ + Ok( + SubUpdate::tagged(Arc::from("temp.vienna"), Payload::from(&b"1"[..])) + .with_skipped(2), + ), + Err(RpcError::Denied), + ok_update("99", 0), // past the terminal item — never delivered + ]); + let out: Vec<_> = decode_updates(raw).collect().await; + + assert_eq!(out.len(), 1); + assert_eq!(out[0].topic.as_deref(), Some("temp.vienna")); + assert_eq!(out[0].value, json!(1)); + assert_eq!(out[0].skipped, 2); + } +} diff --git a/aimdb-client/src/lib.rs b/aimdb-client/src/lib.rs index 869c0af1..19b70348 100644 --- a/aimdb-client/src/lib.rs +++ b/aimdb-client/src/lib.rs @@ -49,6 +49,6 @@ pub mod protocol; #[cfg(feature = "transport-uds")] pub use discovery::{discover_instances, find_instance, InstanceInfo}; pub use endpoint::{dial, parse_endpoint, ParsedEndpoint, Scheme}; -pub use engine::{AimxConnection, DrainResponse}; +pub use engine::{AimxConnection, DrainResponse, RecordUpdate}; pub use error::{ClientError, ClientResult}; pub use protocol::{RecordMetadata, WelcomeMessage, CLIENT_NAME, PROTOCOL_VERSION}; diff --git a/aimdb-client/tests/aimx_session.rs b/aimdb-client/tests/aimx_session.rs index f0e790b0..94b12443 100644 --- a/aimdb-client/tests/aimx_session.rs +++ b/aimdb-client/tests/aimx_session.rs @@ -117,6 +117,26 @@ async fn aimx_roundtrip_over_uds_production_server() { assert!(ev.get("n").is_some(), "event carries a Reading: {ev}"); } + // The loss-aware form carries the same values plus the engine's gap count; + // a keeping-up consumer sees no gaps. + let mut updates = conn + .subscribe_updates("events") + .expect("loss-aware subscribe"); + for _ in 0..3 { + let update = tokio::time::timeout(Duration::from_secs(2), updates.next()) + .await + .expect("update within timeout") + .expect("update"); + assert!( + update.value.get("n").is_some(), + "update carries a Reading: {}", + update.value + ); + assert_eq!(update.skipped, 0, "consumer keeps up, so no gap"); + assert!(!update.has_gap()); + } + drop(updates); + // Graph introspection wrappers. let nodes = conn.graph_nodes().await.expect("graph nodes"); assert!( diff --git a/aimdb-wasm-adapter/CHANGELOG.md b/aimdb-wasm-adapter/CHANGELOG.md index 5347a05a..d03bd329 100644 --- a/aimdb-wasm-adapter/CHANGELOG.md +++ b/aimdb-wasm-adapter/CHANGELOG.md @@ -56,6 +56,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`WsBridge` surfaces subscription delivery gaps: `onGap(topic, skipped)` + + `droppedUpdates()`.** The pump reads the engine's per-update `skipped` count + and reports it (console warning always; the registered handler in addition), + instead of routing the payload and discarding the gap. A mirror that jumped + ahead because the server-side buffer overran a slow consumer is no longer + indistinguishable from an idle producer. + - **Host-run buffer unit tests + shared contract suite (Design 040).** `buffer.rs` now has native (`cargo test`) unit tests for the fresh-subscriber and `peek()` semantics, plus the `aimdb-core` `buffer::test_support` conformance suite run diff --git a/aimdb-wasm-adapter/README.md b/aimdb-wasm-adapter/README.md index 66cd23e1..5c22d54c 100644 --- a/aimdb-wasm-adapter/README.md +++ b/aimdb-wasm-adapter/README.md @@ -81,6 +81,14 @@ bridge.onStatusChange((status) => { console.log('Connection:', status); // 'Connected' | 'Reconnecting' | ... }); +// Delivery gaps: the server sent updates the mirror never received (a slow +// consumer overran the server-side buffer). Without this, a gap looks exactly +// like an idle producer. +bridge.onGap((topic, skipped) => { + console.warn(`${topic}: lost ${skipped} update(s)`); +}); +bridge.droppedUpdates(); // cumulative count since connect + bridge.write('commands.setpoint', { target: 21.0 }); bridge.disconnect(); ``` diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index cb8dd610..bc1fa532 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -118,6 +118,10 @@ impl Default for BridgeOptions { struct BridgeShared { status: Cell, on_status: RefCell>, + /// Notified when a subscription reports a delivery gap. + on_gap: RefCell>, + /// Cumulative updates the server sent that never reached the local mirror. + dropped_total: Cell, /// Set by `disconnect()` — stops redials and ends the pumps. stopped: Cell, /// Whether a connection ever succeeded (Connecting vs Reconnecting). @@ -137,6 +141,13 @@ impl BridgeShared { emit_status(&self.on_status, status); } + /// Record a delivery gap on `topic` and notify JS. + fn report_gap(&self, topic: &str, skipped: u64) { + self.dropped_total + .set(self.dropped_total.get().saturating_add(skipped)); + emit_gap(&self.on_gap, topic, skipped); + } + /// The status to report when a connection ends. fn drop_status(&self) -> ConnectionStatus { if self.stopped.get() || !self.auto_reconnect { @@ -366,6 +377,28 @@ impl WsBridge { *self.shared.on_status.borrow_mut() = Some(callback); } + /// Register a callback for subscription delivery gaps. + /// + /// Fires with `(topic, skipped)` whenever the server-side buffer dropped + /// updates before the one just mirrored — the local record jumped ahead and + /// intermediate values are gone for good. Without this, a gap is + /// indistinguishable from an idle producer. + /// + /// ```ts + /// bridge.onGap((topic, skipped) => console.warn(`${topic}: lost ${skipped}`)); + /// ``` + #[wasm_bindgen(js_name = "onGap")] + pub fn on_gap(&self, callback: js_sys::Function) { + *self.shared.on_gap.borrow_mut() = Some(callback); + } + + /// Total updates dropped before reaching the local mirror since the bridge + /// was created (`0` while delivery has been lossless). + #[wasm_bindgen(js_name = "droppedUpdates")] + pub fn dropped_updates(&self) -> f64 { + self.shared.dropped_total.get() as f64 + } + /// Send a value to the server for a given topic (AimX `write` frame). /// /// While disconnected the command is queued by the engine (up to @@ -471,6 +504,8 @@ impl WsBridge { let shared = Rc::new(BridgeShared { status: Cell::new(ConnectionStatus::Connecting), on_status: RefCell::new(None), + on_gap: RefCell::new(None), + dropped_total: Cell::new(0), stopped: Cell::new(false), ever_connected: Cell::new(false), auto_reconnect: config.auto_reconnect, @@ -609,6 +644,12 @@ async fn pump_pattern( // Wildcard events carry the concrete record topic; an exact-topic // subscription may leave it implicit. let topic = update.topic.as_deref().unwrap_or(&pattern); + // A non-zero gap means the mirror missed values the server did send: + // the local record jumps ahead. Surface it — a silent hole looks + // identical to an idle producer from JS. + if update.skipped > 0 { + shared.report_gap(topic, update.skipped); + } route_update(&db, &schema_map, ®istry, topic, &update.data); } // Stream ended without a rejection — a disconnect. Pace the re-subscribe @@ -723,6 +764,32 @@ fn emit_status(on_status: &RefCell>, status: Connection dispatch_status_event(status); } +/// Notify JS that `skipped` updates were lost on `topic`. +/// +/// Always warns on the console (a gap is a data-loss event a developer should +/// see even with no handler registered), then calls the registered handler — +/// deferred to a microtask for the same re-entrancy reason as [`emit_status`], +/// since the pump is polled from inside the WebSocket message callback. +fn emit_gap(on_gap: &RefCell>, topic: &str, skipped: u64) { + web_sys::console::warn_1( + &format!("[WsBridge] delivery gap: {skipped} update(s) dropped for topic='{topic}'").into(), + ); + let Some(cb) = on_gap.borrow().as_ref().cloned() else { + return; + }; + let topic = JsValue::from_str(topic); + let skipped = JsValue::from_f64(skipped as f64); + wasm_bindgen_futures::spawn_local(async move { + let _ = + wasm_bindgen_futures::JsFuture::from(js_sys::Promise::resolve(&JsValue::NULL)).await; + if let Err(e) = cb.call2(&JsValue::NULL, &topic, &skipped) { + web_sys::console::error_1( + &format!("[WsBridge] emit_gap callback threw: {:?}", e).into(), + ); + } + }); +} + /// Dispatch a `CustomEvent("aimdb:status")` on `window` with the status /// string as `event.detail`. fn dispatch_status_event(status: ConnectionStatus) { diff --git a/tools/aimdb-cli/CHANGELOG.md b/tools/aimdb-cli/CHANGELOG.md index 93187a77..c319f789 100644 --- a/tools/aimdb-cli/CHANGELOG.md +++ b/tools/aimdb-cli/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`aimdb watch` reports delivery gaps.** The watcher subscribes via the loss-aware `AimxConnection::subscribe_updates`, printing a `⚠️ gap: N update(s) dropped` line before the event that follows a gap, plus a session total when watching stops. A silent hole in the value stream is no longer indistinguishable from an idle producer. + - **Global `--connect ` flag + `AIMDB_CONNECT` env (Issue #123).** Choose the target instance by `scheme://` URL — `unix://PATH`, `serial://DEVICE?baud=N` (with the `transport-serial` feature), or a bare path (the `unix://` shorthand). Precedence: `--connect` → `AIMDB_CONNECT` → UDS auto-discovery. `instance info`/`ping` now work over any endpoint (not just discovered sockets). New `transport-serial` feature (off by default; pulls libudev) adds the serial transport to the resolver. ### Changed (breaking) diff --git a/tools/aimdb-cli/src/commands/watch.rs b/tools/aimdb-cli/src/commands/watch.rs index d1b7a7d0..ba30751d 100644 --- a/tools/aimdb-cli/src/commands/watch.rs +++ b/tools/aimdb-cli/src/commands/watch.rs @@ -50,8 +50,9 @@ async fn watch_record( let conn = connect_endpoint(endpoint).await?; // Subscribe to the record (the engine routes updates back by request id; no - // server-allocated subscription id to track). - let mut stream = conn.subscribe(record_name)?; + // server-allocated subscription id to track). The loss-aware form, so the + // watcher can report gaps instead of skipping over them silently. + let mut stream = conn.subscribe_updates(record_name)?; live::print_watch_start(record_name); @@ -64,17 +65,23 @@ async fn watch_record( }); // Receive updates. The reshaped wire carries no server sequence, so the - // watcher counts locally. + // watcher counts locally; `skipped` is what the engine observed as lost + // between deliveries, tallied for the closing summary. let mut count: u64 = 0; + let mut skipped_total: u64 = 0; let unlimited = max_count == 0; loop { tokio::select! { next = stream.next() => { match next { - Some(data) => { + Some(update) => { count += 1; - live::print_event(count, &data, show_full); + if update.skipped > 0 { + skipped_total += update.skipped; + live::print_gap(update.skipped); + } + live::print_event(count, &update.value, show_full); if !unlimited && count >= max_count as u64 { break; } @@ -87,6 +94,6 @@ async fn watch_record( } // Dropping the stream stops local delivery (no explicit unsubscribe needed). - live::print_watch_stop(); + live::print_watch_stop(skipped_total); Ok(()) } diff --git a/tools/aimdb-cli/src/output/live.rs b/tools/aimdb-cli/src/output/live.rs index d9a1954e..a1923521 100644 --- a/tools/aimdb-cli/src/output/live.rs +++ b/tools/aimdb-cli/src/output/live.rs @@ -34,6 +34,21 @@ pub fn print_event(seq: u64, data: &serde_json::Value, show_full: bool) { println!("{}", format_event(seq, data, show_full)); } +/// Format a delivery-gap notice: `skipped` updates were lost between the +/// previous printed event and the next one. +pub fn format_gap(skipped: u64) -> String { + let plural = if skipped == 1 { "update" } else { "updates" }; + format!("⚠️ gap: {skipped} {plural} dropped before the next event") + .yellow() + .to_string() +} + +/// Print a delivery-gap notice to stdout (immediately before the event that +/// follows the gap). +pub fn print_gap(skipped: u64) { + println!("{}", format_gap(skipped)); +} + /// Print subscription start message pub fn print_watch_start(record_name: &str) { println!("📡 Watching record: {}", record_name.bold()); @@ -41,9 +56,16 @@ pub fn print_watch_start(record_name: &str) { println!(); } -/// Print subscription stop message -pub fn print_watch_stop() { +/// Print subscription stop message, reporting the total updates lost across the +/// session (`0` stays silent — the common lossless case). +pub fn print_watch_stop(skipped_total: u64) { println!(); + if skipped_total > 0 { + println!( + "{}", + format!("⚠️ {skipped_total} update(s) dropped during this session").yellow() + ); + } println!("{}", "✅ Stopped watching".green()); } @@ -60,6 +82,12 @@ mod tests { assert!(formatted.contains("temperature")); } + #[test] + fn test_format_gap() { + assert!(format_gap(1).contains("1 update dropped")); + assert!(format_gap(7).contains("7 updates dropped")); + } + #[test] fn test_format_event_with_dropped() { // AimX-v2 wire does not carry dropped counts; format_event receives From 182bd4787b79c8d2736ec0e26b374f68e251a4e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 27 Jul 2026 21:41:29 +0000 Subject: [PATCH 36/45] feat: enhance WebSocket connection handling and add integration tests for transform join --- aimdb-wasm-adapter/src/ws_bridge.rs | 168 ++++++++++++++++-- .../tests/transform_join_integration_tests.rs | 20 ++- 2 files changed, 171 insertions(+), 17 deletions(-) diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index bc1fa532..15594dcd 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -148,6 +148,22 @@ impl BridgeShared { emit_gap(&self.on_gap, topic, skipped); } + /// Publish a freshly-opened socket as the live one, moving to `Connected`. + /// + /// Returns `false` when [`WsBridge::disconnect`] ran while the handshake was + /// still pending: back then `self.ws` was `None`, so `disconnect()` had no + /// socket to close — the caller must abandon this one instead of installing + /// it, or a completed disconnect would silently come back as `Connected`. + fn accept_dial(&self, ws: &web_sys::WebSocket) -> bool { + if self.stopped.get() { + return false; + } + self.ever_connected.set(true); + *self.ws.borrow_mut() = Some(ws.clone()); + self.set_status(ConnectionStatus::Connected); + true + } + /// The status to report when a connection ends. fn drop_status(&self) -> ConnectionStatus { if self.stopped.get() || !self.auto_reconnect { @@ -201,15 +217,24 @@ impl Connection for WasmWsConnection { } } +/// Detach every JS callback and close `ws`. +/// +/// Detaching first matters: the `Closure`s are dropped right after (they live in +/// the dial future or the connection), and a socket still holding them would +/// call into freed WASM closures on its final `close`/`error` event. +fn shutdown_socket(ws: &web_sys::WebSocket) { + ws.set_onopen(None); + ws.set_onmessage(None); + ws.set_onclose(None); + ws.set_onerror(None); + let _ = ws.close(); +} + impl Drop for WasmWsConnection { fn drop(&mut self) { - // Detach the JS callbacks before they drop, and close the socket if the - // engine lets go of a still-open connection (graceful stop). - self.ws.set_onopen(None); - self.ws.set_onmessage(None); - self.ws.set_onclose(None); - self.ws.set_onerror(None); - let _ = self.ws.close(); + // Close the socket if the engine lets go of a still-open connection + // (graceful stop). + shutdown_socket(&self.ws); if self .shared .ws @@ -308,13 +333,24 @@ impl Dialer for WasmWsDialer { ws.set_onerror(Some(on_error.as_ref().unchecked_ref())); plain_callbacks.push(on_error); + // Handshake failed (`onclose` before `onopen`, or the sender was + // dropped). The socket is closing but still holds the callbacks, + // which are freed when this future returns — detach them first so a + // trailing `close`/`error` event cannot invoke a dropped `Closure`. if open_rx.await != Ok(true) { + shutdown_socket(&ws); return Err(TransportError::Io); } - shared.ever_connected.set(true); - *shared.ws.borrow_mut() = Some(ws.clone()); - shared.set_status(ConnectionStatus::Connected); + // `disconnect()` may have run while this handshake was pending — it + // could not close a socket that was not published yet. Abandon it + // rather than reporting `Connected` after a completed disconnect; + // `Closed` is terminal, so the engine stops and the subscription + // pumps release their `ClientHandle` clones. + if !shared.accept_dial(&ws) { + shutdown_socket(&ws); + return Err(TransportError::Closed); + } Ok(Box::new(WasmWsConnection { ws, @@ -803,3 +839,115 @@ fn dispatch_status_event(status: ConnectionStatus) { let _ = web_sys::EventTarget::from(window).dispatch_event(&event); } } + +// ─── Tests ───────────────────────────────────────────────────────────────── + +// Needs a real `web_sys::WebSocket`, so this is the browser lane only +// (`make wasm-test`); off-target the socket constructor is an unusable stub. +#[cfg(all(test, target_arch = "wasm32"))] +mod dial_tests { + use super::*; + use wasm_bindgen_test::wasm_bindgen_test; + + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + /// TEST-NET-1 (RFC 5737) on a port the browser does not block: the address + /// is unroutable, so the handshake stays pending for the duration of a test + /// and a socket built from it never opens or closes on its own — nothing can + /// race the assertions. + const PENDING_URL: &str = "ws://192.0.2.1:8443/aimdb-test"; + + fn shared() -> Rc { + Rc::new(BridgeShared { + status: Cell::new(ConnectionStatus::Connecting), + on_status: RefCell::new(None), + on_gap: RefCell::new(None), + dropped_total: Cell::new(0), + stopped: Cell::new(false), + ever_connected: Cell::new(false), + auto_reconnect: true, + ws: RefCell::new(None), + }) + } + + #[wasm_bindgen_test] + fn accept_dial_publishes_the_socket_while_running() { + let shared = shared(); + let ws = web_sys::WebSocket::new(PENDING_URL).unwrap(); + + assert!(shared.accept_dial(&ws)); + assert!(shared.ws.borrow().is_some()); + assert!(shared.ever_connected.get()); + assert_eq!(shared.status.get(), ConnectionStatus::Connected); + + shutdown_socket(&ws); + shared.ws.borrow_mut().take(); + } + + /// Regression: `disconnect()` during a pending handshake. + /// + /// While the dial is in flight `shared.ws` is still `None`, so `disconnect()` + /// has no socket to close. When `onopen` lands afterwards the dial must be + /// abandoned — otherwise the socket is installed and the status flips back to + /// `Connected` after the bridge was already disconnected. + #[wasm_bindgen_test] + fn accept_dial_is_refused_after_disconnect_during_handshake() { + let shared = shared(); + + // What `disconnect()` can do mid-handshake: no live socket to close. + assert!(shared.ws.borrow().is_none()); + shared.stopped.set(true); + shared.set_status(ConnectionStatus::Disconnected); + + // The in-flight socket completes its handshake only now. + let ws = web_sys::WebSocket::new(PENDING_URL).unwrap(); + assert!( + !shared.accept_dial(&ws), + "a stopped bridge must not adopt a late socket" + ); + assert!( + shared.ws.borrow().is_none(), + "late socket must not become the live one" + ); + assert_eq!( + shared.status.get(), + ConnectionStatus::Disconnected, + "a late onopen must not undo a completed disconnect" + ); + assert!(!shared.ever_connected.get()); + + shutdown_socket(&ws); + } + + /// `disconnect()` while the bridge is still connecting settles on + /// `Disconnected` and rejects further commands, with no socket left behind. + #[wasm_bindgen_test] + async fn bridge_disconnect_while_connecting_settles_disconnected() { + // No records configured, so the runner owns no futures — dropping it is + // the whole of "running" this database. + let (db, _runner) = aimdb_core::AimDbBuilder::new() + .runtime(Arc::new(WasmAdapter)) + .build() + .await + .unwrap(); + let bridge = WsBridge::new_internal( + db, + BTreeMap::new(), + SchemaRegistry::new(), + PENDING_URL, + JsValue::NULL, + ) + .unwrap(); + + // Let the engine start its (never-completing) dial. + let _ = + wasm_bindgen_futures::JsFuture::from(js_sys::Promise::resolve(&JsValue::NULL)).await; + assert_eq!(bridge.status(), "connecting"); + + bridge.disconnect(); + assert_eq!(bridge.status(), "disconnected"); + assert!(bridge.shared.stopped.get()); + assert!(bridge.shared.ws.borrow().is_none()); + assert!(bridge.write("test::topic", JsValue::NULL).is_err()); + } +} diff --git a/aimdb-wasm-adapter/tests/transform_join_integration_tests.rs b/aimdb-wasm-adapter/tests/transform_join_integration_tests.rs index a99199f4..de08a4e3 100644 --- a/aimdb-wasm-adapter/tests/transform_join_integration_tests.rs +++ b/aimdb-wasm-adapter/tests/transform_join_integration_tests.rs @@ -36,11 +36,14 @@ async fn transform_join_produces_sum_on_both_inputs() { let runtime = Arc::new(WasmAdapter); let mut builder = AimDbBuilder::new().runtime(runtime); + // SpmcRing inputs (vs SingleLatest) so that values produced before the join + // transform's forwarders subscribe are still buffered and replayed — removes + // a startup race where the test might otherwise need a hand-tuned barrier. builder.configure::("test::A", |reg| { - reg.buffer(BufferCfg::SingleLatest); + reg.buffer(BufferCfg::SpmcRing { capacity: 16 }); }); builder.configure::("test::B", |reg| { - reg.buffer(BufferCfg::SingleLatest); + reg.buffer(BufferCfg::SpmcRing { capacity: 16 }); }); builder.configure::("test::Sum", |reg| { reg.buffer(BufferCfg::SpmcRing { capacity: 16 }) @@ -64,7 +67,10 @@ async fn transform_join_produces_sum_on_both_inputs() { }); }); - let db = builder.build().await.unwrap(); + let (db, runner) = builder.build().await.unwrap(); + // The join transform lives in the runner's future set; on wasm the browser + // microtask queue is the executor, so `spawn_local` is the `tokio::spawn`. + wasm_bindgen_futures::spawn_local(runner.run()); let mut sum_rx = db.subscribe::("test::Sum").unwrap(); // Yield to let the join transform task spawn its input forwarders and subscribe. @@ -73,18 +79,18 @@ async fn transform_join_produces_sum_on_both_inputs() { .unwrap(); // A=1, B=10 → Sum=11 - db.produce::("test::A", ValueA(1)).await.unwrap(); - db.produce::("test::B", ValueB(10)).await.unwrap(); + db.produce::("test::A", ValueA(1)).unwrap(); + db.produce::("test::B", ValueB(10)).unwrap(); let s = sum_rx.recv().await.unwrap(); assert_eq!(s.0, 11, "expected 1+10=11"); // A=2 → Sum=12 (B stays 10) - db.produce::("test::A", ValueA(2)).await.unwrap(); + db.produce::("test::A", ValueA(2)).unwrap(); let s = sum_rx.recv().await.unwrap(); assert_eq!(s.0, 12, "expected 2+10=12"); // B=20 → Sum=22 (A stays 2) - db.produce::("test::B", ValueB(20)).await.unwrap(); + db.produce::("test::B", ValueB(20)).unwrap(); let s = sum_rx.recv().await.unwrap(); assert_eq!(s.0, 22, "expected 2+20=22"); } From 97f475edb962ea33ff71dbf33f7714b3b74ea8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 27 Jul 2026 21:49:03 +0000 Subject: [PATCH 37/45] feat: update WebSocket connection URL to use versioned endpoint --- aimdb-websocket-connector/benches/fanout_socket.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aimdb-websocket-connector/benches/fanout_socket.rs b/aimdb-websocket-connector/benches/fanout_socket.rs index 6b5c068e..b67c518e 100644 --- a/aimdb-websocket-connector/benches/fanout_socket.rs +++ b/aimdb-websocket-connector/benches/fanout_socket.rs @@ -146,7 +146,8 @@ fn spawn_client( ready: mpsc::Sender<()>, ) -> JoinHandle<()> { tokio::spawn(async move { - let (mut ws, _) = tokio_tungstenite::connect_async(format!("ws://{addr}/ws")) + let url = aimdb_core::remote::ws_url_with_version(&format!("ws://{addr}/ws")); + let (mut ws, _) = tokio_tungstenite::connect_async(url) .await .expect("client connect"); ws.send(Message::Text( From 9dfb52db63eef3b89759f9b1c4766fb18053da02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Mon, 27 Jul 2026 22:06:12 +0000 Subject: [PATCH 38/45] feat: implement pending call management to reclaim abandoned calls in the client engine --- aimdb-core/src/session/client.rs | 122 ++++++++++++++++++++++++- aimdb-core/tests/session_engine.rs | 135 ++++++++++++++++++++++++++++ aimdb-wasm-adapter/src/ws_bridge.rs | 4 + 3 files changed, 258 insertions(+), 3 deletions(-) diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index b4c30f56..0c6aab70 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -358,11 +358,43 @@ async fn reconnect_after( true } +/// What a caller awaiting [`ClientHandle::call`] eventually receives. +type CallReply = Result; + +/// The engine's in-flight call table: correlation `id` → the caller's reply +/// channel. +type PendingCalls = HashMap>; + +/// Drop every entry whose caller has gone away. +/// +/// A caller that gives up on [`ClientHandle::call`] — a timeout racing the +/// future, or any other cancellation — drops only its `oneshot::Receiver`; the +/// matching sender stays in the engine's map, since a peer that holds the +/// connection open but never answers produces neither a `Reply` nor a +/// disconnect. `futures_channel` records the drop on the sender, so this sweep +/// is a flag check per entry with no wakeups or allocation. +fn prune_canceled(pending: &mut PendingCalls) { + pending.retain(|_, tx| !tx.is_canceled()); +} + +/// Register a call's reply channel, first reclaiming any abandoned entry. +/// +/// Pruning here (rather than per frame) is what bounds the table: the sweep is +/// O(in-flight calls) and runs only when a new call is issued, so a stream of +/// timed-out requests over a live connection costs one lingering entry each +/// until the next call, instead of one for the life of the connection. +fn track_call(pending: &mut PendingCalls, id: u64, reply: oneshot::Sender) { + prune_canceled(pending); + pending.insert(id, reply); +} + /// Drive one dialed [`Connection`]: optional handshake, then `biased` demux of /// server frames (resolve `Reply` by `id`, route `Event`/`Snapshot` to their /// subscription channels) interleaved with caller commands. Pending state is /// per-connection: a disconnect fails outstanding calls (their `oneshot` -/// senders drop → callers see [`RpcError::Internal`]). +/// senders drop → callers see [`RpcError::Internal`]). Calls the *caller* +/// abandoned are reclaimed by [`prune_canceled`], so an unanswering peer that +/// keeps the link open can't grow the table. async fn drive_connection( mut conn: Box, codec: &C, @@ -374,7 +406,7 @@ where C: EnvelopeCodec + ?Sized, { let mut next_id: u64 = 1; - let mut pending: HashMap>> = HashMap::new(); + let mut pending: PendingCalls = HashMap::new(); // sub-id → event sink. The sub-id is `id.to_string()` of the opening // request, matching the server's derivation so `Event.sub` routes back. let mut subs: HashMap>> = HashMap::new(); @@ -501,6 +533,10 @@ where } ClientStep::Keepalive => { + // Also the reclaim tick: a caller that times out and then stops + // issuing calls would otherwise hold its entries until the next + // call or the disconnect. + prune_canceled(&mut pending); // `keepalive_timer` is `Some` whenever this step fires. let interval_ms = keepalive_ms.unwrap_or(0); let idle_ms = clock.now_nanos().saturating_sub(last_activity) / 1_000_000; @@ -540,9 +576,16 @@ where params, reply, } => { + // The caller may have given up while the command sat in + // the queue — don't spend a wire request (or an `id`) on + // a reply nobody is waiting for. + if reply.is_canceled() { + prune_canceled(&mut pending); + continue; + } let id = next_id; next_id += 1; - pending.insert(id, reply); + track_call(&mut pending, id, reply); out.clear(); let sent = codec .encode_inbound(Inbound::Request { id, method, params }, &mut out) @@ -666,3 +709,76 @@ pub fn pump_client(db: &AimDb, scheme: &str, handle: &ClientHandle) -> Vec (oneshot::Sender, oneshot::Receiver) { + oneshot::channel() + } + + /// A caller that gave up is reclaimed; one still awaiting its reply is not, + /// and stays deliverable. + #[test] + fn prune_canceled_drops_only_abandoned_calls() { + let mut pending = PendingCalls::new(); + let (tx1, rx1) = call_slot(); + let (tx2, rx2) = call_slot(); + pending.insert(1, tx1); + pending.insert(2, tx2); + + drop(rx1); // caller 1 timed out and dropped its `call` future + prune_canceled(&mut pending); + + assert_eq!(pending.len(), 1, "the abandoned call must be reclaimed"); + assert!(pending.contains_key(&2), "a live call must be kept"); + pending + .remove(&2) + .unwrap() + .send(Ok(Payload::from(&b"ok"[..]))) + .expect("the retained sender must still reach its caller"); + drop(rx2); + } + + /// Regression: a peer that keeps the connection open but never replies used + /// to cost one permanent entry per timed-out request, since only a `Reply` + /// or a disconnect removed one. Repeated timeouts must not grow the table. + #[test] + fn repeated_timeouts_do_not_grow_pending() { + let mut pending = PendingCalls::new(); + for id in 1..=1_000u64 { + let (tx, rx) = call_slot(); + track_call(&mut pending, id, tx); + // The caller times out immediately; nothing ever replies. + drop(rx); + assert!( + pending.len() <= 1, + "pending grew to {} at id {id}; abandoned calls are leaking", + pending.len() + ); + } + } + + /// The reclaim sweep must not disturb calls that are still outstanding, so + /// a long-lived call survives a burst of timed-out ones around it. + #[test] + fn track_call_keeps_outstanding_calls_across_prunes() { + let mut pending = PendingCalls::new(); + let (slow_tx, slow_rx) = call_slot(); + track_call(&mut pending, 1, slow_tx); + + for id in 2..=100u64 { + let (tx, rx) = call_slot(); + track_call(&mut pending, id, tx); + drop(rx); + } + + assert!( + pending.contains_key(&1), + "an outstanding call must survive the reclaim sweep" + ); + assert_eq!(pending.len(), 2, "only the outstanding calls remain"); + drop(slow_rx); + } +} diff --git a/aimdb-core/tests/session_engine.rs b/aimdb-core/tests/session_engine.rs index a450480a..bb67964b 100644 --- a/aimdb-core/tests/session_engine.rs +++ b/aimdb-core/tests/session_engine.rs @@ -761,3 +761,138 @@ async fn ended_subscription_frees_its_cap_slot() { let _ = client.await; server.abort(); } + +// =========================================================================== +// Abandoned calls (Layer 4) — a peer that keeps the link open but never replies +// =========================================================================== + +/// Engine config for the silent-peer tests: one connection, no redial, no +/// keepalive, so the only thing under test is the pending-call bookkeeping. +fn silent_peer_config() -> ClientConfig { + ClientConfig { + reconnect: false, + keepalive_interval: None, + ..Default::default() + } +} + +/// Read the next `REQ` frame the client engine put on the wire, as +/// `(id, method, params)`. +async fn read_request(peer: &mut Box) -> (u64, String, String) { + let frame = tokio::time::timeout(Duration::from_secs(2), peer.recv()) + .await + .expect("the client must send its request") + .expect("the pipe must stay open") + .expect("the pipe must stay open"); + let text = String::from_utf8(frame).expect("test frames are utf-8"); + let mut parts = text.splitn(4, '\n'); + assert_eq!( + parts.next(), + Some("REQ"), + "expected a request frame: {text}" + ); + let id = parts + .next() + .and_then(|id| id.parse().ok()) + .expect("request id"); + let method = parts.next().expect("request method").to_string(); + (id, method, parts.next().unwrap_or("").to_string()) +} + +/// Regression: callers that time out and drop their `call` future must not +/// accumulate in the engine's pending-call map. +/// +/// A `Reply` or a disconnect used to be the only things that removed an entry, +/// so a peer holding the connection open while never answering (the shape the +/// WASM bridge's `query_timeout_ms` produces) cost one live `oneshot::Sender` +/// per timed-out request for the life of the connection. The engine now reclaims +/// abandoned entries; this test drives that path end-to-end and pins the +/// property it must not break — reply correlation still works afterwards. +#[tokio::test] +async fn repeated_call_timeouts_leave_the_connection_usable() { + let (mut listener, dialer) = transport_pair(); + let (handle, client_fut) = + run_client(dialer, LineCodec, silent_peer_config(), Arc::new(TestClock)); + let client = tokio::spawn(client_fut); + // The test *is* the peer: it reads every request and answers only the last. + let mut peer = listener.accept().await.expect("the client must dial"); + + for i in 0..64 { + let call = handle.call("hang", payload_from(&format!("{i}"))); + assert!( + tokio::time::timeout(Duration::from_millis(20), call) + .await + .is_err(), + "an unanswered call must not resolve" + ); + let (_, method, _) = read_request(&mut peer).await; + assert_eq!(method, "hang", "each request must still reach the peer"); + } + + // The demux survived the reclaim sweeps: a call the peer *does* answer + // resolves with its own reply, so no live entry was pruned by mistake. + let answered = tokio::spawn({ + let handle = handle.clone(); + async move { handle.call("ping", payload_from("hi")).await } + }); + let (id, method, params) = read_request(&mut peer).await; + assert_eq!((method.as_str(), params.as_str()), ("ping", "hi")); + peer.send(format!("REPLY\n{id}\nOK\npong").as_bytes()) + .await + .expect("the pipe must stay open"); + let reply = tokio::time::timeout(Duration::from_secs(2), answered) + .await + .expect("the answered call must resolve") + .expect("the call task must not panic") + .expect("the peer replied with OK"); + assert_eq!(&*reply, b"pong"); + + drop(handle); + drop(peer); + let _ = client.await; +} + +/// A call abandoned while its command still sits in the engine's queue is +/// dropped instead of dialed out: nobody is left to receive the reply, so the +/// request would only make the peer do work whose answer is discarded. +#[tokio::test] +async fn a_call_abandoned_before_dispatch_is_never_sent() { + let (mut listener, dialer) = transport_pair(); + let (handle, client_fut) = + run_client(dialer, LineCodec, silent_peer_config(), Arc::new(TestClock)); + + // Zero deadline: `timeout` polls the call once — enough to enqueue the + // command — then gives up and drops it, all before the engine is driven. + assert!( + tokio::time::timeout(Duration::ZERO, handle.call("ghost", payload_from("x"))) + .await + .is_err(), + "the call must be abandoned before the engine dequeues it" + ); + + let client = tokio::spawn(client_fut); + let mut peer = listener.accept().await.expect("the client must dial"); + + let answered = tokio::spawn({ + let handle = handle.clone(); + async move { handle.call("real", payload_from("hi")).await } + }); + let (id, method, _) = read_request(&mut peer).await; + assert_eq!( + method, "real", + "an abandoned call must not be sent to the peer" + ); + peer.send(format!("REPLY\n{id}\nOK\npong").as_bytes()) + .await + .expect("the pipe must stay open"); + let reply = tokio::time::timeout(Duration::from_secs(2), answered) + .await + .expect("the answered call must resolve") + .expect("the call task must not panic") + .expect("the peer replied with OK"); + assert_eq!(&*reply, b"pong"); + + drop(handle); + drop(peer); + let _ = client.await; +} diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index 15594dcd..b93018d9 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -614,6 +614,10 @@ impl WsBridge { let timeout = WasmAdapter.sleep(core::time::Duration::from_millis(timeout_ms as u64)); futures_util::pin_mut!(call); + // Losing the race drops `call`, i.e. the reply channel's + // receiver. The engine reclaims its side on the next call or + // keepalive tick, so timing out against an unanswering peer + // doesn't accumulate pending entries for the connection's life. match futures_util::future::select(call, timeout).await { futures_util::future::Either::Left((reply, _)) => reply, futures_util::future::Either::Right(((), _)) => { From 5b5b0812a1db0bdad5f744b3bdeeff2dbac329b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 28 Jul 2026 08:10:45 +0000 Subject: [PATCH 39/45] feat: enhance late-join snapshot handling with sequence numbering and final frame flag --- aimdb-core/CHANGELOG.md | 46 +++++---- aimdb-core/src/session/aimx/codec.rs | 35 ++++++- aimdb-core/src/session/client.rs | 42 +++++++- aimdb-core/src/session/mod.rs | 29 +++++- aimdb-core/src/session/server.rs | 8 +- aimdb-core/tests/session_engine.rs | 95 ++++++++++++++----- aimdb-websocket-connector/CHANGELOG.md | 15 +-- aimdb-websocket-connector/tests/e2e.rs | 21 +++- ...047-retire-ws-protocol-converge-on-aimx.md | 28 ++++-- 9 files changed, 256 insertions(+), 63 deletions(-) diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index 196db135..aac28dee 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -31,22 +31,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 without them are unchanged on the wire). `Session::snapshot` became `Session::snapshots(topic) -> Vec<(String, Payload)>` (one per covered record). -- **Late-join snapshots are sequence-numbered and no longer lost silently.** - `Outbound::Snapshot` gains a required `seq` in the *same* space as the - subscription's events: `run_session` numbers the burst `1..=N` and - `pump_subscription` continues at `N + 1`. The client demux routes snapshots - through the identical gap accounting as events, so one that overruns the - consumer's `SUBSCRIBE_CHANNEL_CAP`-bounded sink is reported as - `SubUpdate::skipped` on the next delivered update — and a loss at the tail of - the burst surfaces on the first event. Previously the client discarded the - snapshot `try_send` result and snapshots carried no `seq`, so a wildcard - subscription over more than 256 matched records silently delivered only 256, - with no error, no gap, and no way for later event `seq`s to reveal which - initial states were missing — quietly breaking "one snapshot per matched - record". Also fixes two smaller leaks on that path: a snapshot dropped by an - encode failure is now counted as loss, and a snapshot for a subscription - whose receiver is gone now prunes the sub instead of lingering. - Wire: `snap` frames carry `"seq"`, and one without it is `Malformed`. +- **Late-join snapshots are sequence-numbered, terminated, and no longer lost + silently.** `Outbound::Snapshot` gains a required `seq` in the *same* space as + the subscription's events (`run_session` numbers the burst `1..=N`, + `pump_subscription` continues at `N + 1`) plus a `last` flag on the burst's + final frame. The client demux routes snapshots through the identical gap + accounting as events, so one that overruns the consumer's + `SUBSCRIBE_CHANNEL_CAP`-bounded sink is reported as `SubUpdate::skipped`. + Previously the client discarded the snapshot `try_send` result and snapshots + carried no `seq`, so a wildcard subscription over more than 256 matched + records silently delivered only 256, with no error and no gap — quietly + breaking "one snapshot per matched record". + + Because a gap only reaches a subscriber on an update that is *actually + delivered*, mid-burst snapshots now stop one slot short of filling the sink, + reserving it for the flagged final snapshot. That one is therefore always + delivered, arriving as the single update with the new + `SubUpdate::snapshot_end` set and carrying the burst's whole loss count. A + subscriber can tell a complete initial state from a truncated one the moment + the burst ends — without waiting for a live event, which a *static* + subscription may never produce. (A `topic` matching no records emits no burst, + so nothing carries the flag.) + + Also fixes two smaller leaks on that path: a snapshot dropped by an encode + failure is now counted as loss, and a snapshot for a subscription whose + receiver is gone now prunes the sub instead of lingering. + + Wire: `snap` frames carry `"seq"` (one without it is `Malformed`) and the + burst's last frame adds `"last":true`; mid-burst frames are byte-identical to + before. API: `SubUpdate` gains the `snapshot_end` field and a + `with_snapshot_end()` builder. - **`AimxCodec` learned the `subscribed` ack frame** (`{"t":"subscribed", "sub":S}`) for servers running `acks_subscribe:true` (the WebSocket connector); UDS/serial/TCP keep the implicit ack. A dedicated `AimxCodec` diff --git a/aimdb-core/src/session/aimx/codec.rs b/aimdb-core/src/session/aimx/codec.rs index 1ddd1f1a..c6e7f378 100644 --- a/aimdb-core/src/session/aimx/codec.rs +++ b/aimdb-core/src/session/aimx/codec.rs @@ -44,6 +44,9 @@ struct Frame<'a> { id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] seq: Option, + /// Marks the final `snap` of a late-join burst; absent everywhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + last: Option, #[serde(default, borrow, skip_serializing_if = "Option::is_none")] method: Option<&'a str>, #[serde(default, borrow, skip_serializing_if = "Option::is_none")] @@ -69,6 +72,7 @@ impl<'a> Frame<'a> { t, id: None, seq: None, + last: None, method: None, topic: None, sub: None, @@ -235,6 +239,7 @@ impl EnvelopeCodec for AimxCodec { Outbound::Snapshot { sub, seq, + last, topic, data, } => { @@ -242,6 +247,9 @@ impl EnvelopeCodec for AimxCodec { let mut frame = Frame::tagged("snap"); frame.sub = Some(sub); frame.seq = Some(seq); + // Only the burst's final frame carries it, so mid-burst frames + // stay byte-identical to before. + frame.last = last.then_some(true); frame.topic = Some(topic); frame.data = Some(raw); write_frame(out, &frame) @@ -315,6 +323,7 @@ impl EnvelopeCodec for AimxCodec { "snap" => Ok(Outbound::Snapshot { sub: f.sub.ok_or(CodecError::Malformed)?, seq: f.seq.ok_or(CodecError::Malformed)?, + last: f.last.unwrap_or(false), topic: f.topic.ok_or(CodecError::Malformed)?, data: payload_of(f.data), }), @@ -522,23 +531,47 @@ mod tests { let frame = encode_outbound(Outbound::Snapshot { sub: "8", seq: 3, + last: false, topic: "temp.berlin", data: payload(r#"{"c":18.0}"#), }); + // A mid-burst snapshot omits `last` entirely. + assert_eq!( + frame, + br#"{"t":"snap","seq":3,"topic":"temp.berlin","sub":"8","data":{"c":18.0}}"# + ); match AimxCodec.decode_outbound(&frame).unwrap() { Outbound::Snapshot { sub, seq, + last, topic, data, } => { - assert_eq!((sub, seq, topic), ("8", 3, "temp.berlin")); + assert_eq!((sub, seq, last, topic), ("8", 3, false, "temp.berlin")); assert_eq!(&data[..], br#"{"c":18.0}"#); } _ => panic!("expected Snapshot"), } } + /// The burst's final snapshot is flagged, so the client can attach the + /// burst's loss total to an update it is guaranteed to deliver. + #[test] + fn last_snapshot_of_a_burst_is_flagged() { + let frame = encode_outbound(Outbound::Snapshot { + sub: "8", + seq: 4, + last: true, + topic: "temp.berlin", + data: payload("1"), + }); + match AimxCodec.decode_outbound(&frame).unwrap() { + Outbound::Snapshot { seq, last, .. } => assert_eq!((seq, last), (4, true)), + _ => panic!("expected Snapshot"), + } + } + /// A `snap` frame without `seq` is not decodable: snapshots share the /// subscription's sequence space, so an unnumbered one would silently /// defeat the client's gap accounting. diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index 0c6aab70..f4783e53 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -226,6 +226,21 @@ impl Drop for DrainOnExit<'_> { } } +/// How an inbound update competes for room in its subscription's sink. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Delivery { + /// A live event: dropped when the sink is full. + Event, + /// A snapshot with more of its burst to come: dropped when the sink is + /// full, and additionally stops one slot short so [`Delivery::BurstEnd`] + /// always fits. + BurstBody, + /// The burst's final snapshot: guaranteed delivery into the slot + /// [`Delivery::BurstBody`] kept free, carrying the burst's whole loss count + /// and [`SubUpdate::snapshot_end`]. + BurstEnd, +} + /// Route one subscription update (snapshot or event) to its sink, folding any /// loss since the last delivery into its [`SubUpdate::skipped`]. /// @@ -235,6 +250,11 @@ impl Drop for DrainOnExit<'_> { /// bumping the counter before dropping. So one delta against the last delivered /// seq captures every loss point, plus any prior local full-channel drop that /// left `last_seq` behind. +/// +/// A gap only *reaches* the subscriber on an update that is actually delivered, +/// which is why the burst reserves a slot for its final snapshot: a burst +/// truncated at the tail would otherwise stay silent until some later event +/// closed the sequence, and a static subscription may never produce one. fn deliver( subs: &mut HashMap>>, last_seq: &mut HashMap, @@ -242,15 +262,24 @@ fn deliver( seq: u64, topic: Option>, data: Payload, + mode: Delivery, ) { let prev = last_seq.get(sub).copied().unwrap_or(0); let skipped = seq.saturating_sub(prev + 1); // `None` here is a late update for a dropped sub — ignore. if let Some(tx) = subs.get(sub) { + // Hold the last slot back for the burst's final snapshot. Only this + // loop ever sends, so `len()` can only *fall* before the `try_send` + // below — a slot seen free here stays free, making that final delivery + // infallible rather than merely likely. + if mode == Delivery::BurstBody && tx.capacity().is_some_and(|cap| tx.len() + 1 >= cap) { + return; // folds into the next delivered update's `skipped` + } let update = SubUpdate { topic, data, skipped, + snapshot_end: mode == Delivery::BurstEnd, }; match tx.try_send(Ok(update)) { // Delivered — advance the per-sub cursor. @@ -506,15 +535,19 @@ where seq, topic.map(Arc::from), data, + Delivery::Event, ), // Snapshots ride the same accounting as events (they share // the subscription's `seq` space), so a late-join burst that // overruns a slow consumer's sink is reported as `skipped` - // instead of vanishing — "one snapshot per matched record" - // stays checkable by the subscriber. + // instead of vanishing. The burst's final snapshot is + // delivered unconditionally, so "one snapshot per matched + // record" is checkable the moment the burst ends — no live + // event required. Ok(Outbound::Snapshot { sub, seq, + last, topic, data, }) => deliver( @@ -524,6 +557,11 @@ where seq, Some(Arc::from(topic)), data, + if last { + Delivery::BurstEnd + } else { + Delivery::BurstBody + }, ), Ok(Outbound::Pong) => {} // Explicit subscribe ack — informational; the sink already exists. diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index d587f7f4..1429f7c5 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -80,6 +80,8 @@ pub struct SubUpdate { pub data: Payload, /// Count of skipped records pub skipped: u64, + /// Set on the last update of the late-join snapshot burst, carrying its total `skipped` + pub snapshot_end: bool, } impl SubUpdate { @@ -89,6 +91,7 @@ impl SubUpdate { topic: None, data, skipped: 0, + snapshot_end: false, } } @@ -98,6 +101,7 @@ impl SubUpdate { topic: Some(topic), data, skipped: 0, + snapshot_end: false, } } @@ -108,6 +112,13 @@ impl SubUpdate { self.skipped = n; self } + + /// Mark this update as the last of the late-join snapshot burst (see + /// [`snapshot_end`](Self::snapshot_end)). + pub fn with_snapshot_end(mut self) -> Self { + self.snapshot_end = true; + self + } } /// Result of a transport-layer operation. @@ -320,9 +331,14 @@ pub enum Outbound<'a> { /// subscription's [`Event`](Outbound::Event)s: the burst is numbered /// `1..=N` and the first event continues at `N + 1`. So a snapshot lost /// anywhere between here and the subscriber surfaces as a gap in the - /// next delivered update's [`SubUpdate::skipped`] — including a loss at - /// the tail of the burst, which the first event reveals. + /// next delivered update's [`SubUpdate::skipped`]. seq: u64, + /// Set on the last snapshot of the burst, so the loss total lands on an + /// update the subscriber is *guaranteed* to see. Without it a burst + /// truncated at its tail would stay silent until some later event + /// happened to close the sequence — which on a static subscription may + /// be never. The client engine reserves a sink slot for this frame. + last: bool, /// Topic the snapshot is for. topic: &'a str, /// Unparsed record value. @@ -441,6 +457,15 @@ pub trait Session: Send { /// stream at `N + 1`, so "one snapshot per matched record" is *auditable* /// downstream rather than merely intended: any snapshot dropped in transit /// shows up as [`SubUpdate::skipped`] on the next delivered update. + /// + /// The burst's last snapshot is flagged, reaching the subscriber as the one + /// update with [`SubUpdate::snapshot_end`] set — the client engine reserves + /// a sink slot so it lands even when the rest of the burst overran a slow + /// consumer. Its `skipped` then carries the burst's whole loss, letting a + /// subscriber distinguish a complete initial state from a truncated one + /// *without* waiting for a live event, which a static subscription may + /// never produce. A `topic` matching no records emits no burst, and so no + /// such update. fn snapshots(&mut self, topic: &str) -> Vec<(String, Payload)> { let _ = topic; Vec::new() diff --git a/aimdb-core/src/session/server.rs b/aimdb-core/src/session/server.rs index e794d8cc..384e63f5 100644 --- a/aimdb-core/src/session/server.rs +++ b/aimdb-core/src/session/server.rs @@ -228,9 +228,11 @@ pub async fn run_session( // by the encode failure below) leaves a hole the // client reports as `skipped`, and a loss at the // tail of the burst surfaces on the first event. + let snaps = session.snapshots(&topic); + let total = snaps.len(); let mut seq: u64 = 0; let mut send_failed = false; - for (snap_topic, data) in session.snapshots(&topic) { + for (i, (snap_topic, data)) in snaps.into_iter().enumerate() { seq += 1; out.clear(); if codec @@ -238,6 +240,10 @@ pub async fn run_session( Outbound::Snapshot { sub: &sub_id, seq, + // The client reserves a sink slot + // for this one, so the burst's + // loss total always lands. + last: i + 1 == total, topic: &snap_topic, data, }, diff --git a/aimdb-core/tests/session_engine.rs b/aimdb-core/tests/session_engine.rs index bb67964b..d3ba7dd1 100644 --- a/aimdb-core/tests/session_engine.rs +++ b/aimdb-core/tests/session_engine.rs @@ -198,10 +198,19 @@ impl EnvelopeCodec for LineCodec { Outbound::Snapshot { sub, seq, + last, topic, data, } => { - format!("SNAP\n{}\n{}\n{}\n{}", sub, seq, topic, utf8(&data)?) + // `L` marks the burst's final snapshot, `-` a mid-burst one. + format!( + "SNAP\n{}\n{}\n{}\n{}\n{}", + sub, + seq, + if last { "L" } else { "-" }, + topic, + utf8(&data)? + ) } Outbound::Pong => "PONG".to_string(), Outbound::Subscribed { sub } => format!("SUBSCRIBED\n{}", sub), @@ -256,10 +265,12 @@ impl EnvelopeCodec for LineCodec { "SNAP" => { let (sub, r) = rest.split_once('\n').ok_or(CodecError::Malformed)?; let (seq, r) = r.split_once('\n').ok_or(CodecError::Malformed)?; + let (last, r) = r.split_once('\n').ok_or(CodecError::Malformed)?; let (topic, data) = r.split_once('\n').unwrap_or((r, "")); Ok(Outbound::Snapshot { sub, seq: seq.parse().map_err(|_| CodecError::Malformed)?, + last: last == "L", topic, data: payload_from(data), }) @@ -564,12 +575,14 @@ async fn failed_subscribe_ends_stream_via_ack() { /// event `seq` then restarted at 1, no later event could reveal which initial /// states were lost — quietly breaking "one snapshot per matched record". /// -/// Snapshots now share the subscription's `seq` space (`1..=N`, events continue -/// at `N + 1`), so every dropped snapshot is reported as `skipped` on the next -/// delivered update — including the tail of the burst, which the first event -/// accounts for. +/// Sharing the `seq` space fixed that only once *some* later update arrived, so +/// this test deliberately checks the whole accounting **before any event is +/// injected**: a static subscription may never produce one, and a consumer must +/// still be able to tell a complete initial state from a truncated one. The +/// engine therefore reserves a sink slot for the burst's final snapshot, which +/// arrives flagged `snapshot_end` and carrying the burst's whole loss count. #[tokio::test] -async fn oversized_snapshot_burst_surfaces_as_a_gap() { +async fn oversized_snapshot_burst_completes_without_a_live_event() { let (listener, dialer) = transport_pair(); let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel::(); let dispatch = Arc::new(SnapshotDispatch { @@ -623,9 +636,40 @@ async fn oversized_snapshot_burst_surfaces_as_a_gap() { delivered.len() ); - // What did land is the contiguous head of the burst, in order and untagged - // by loss — the drops are all at the tail. - for (i, update) in delivered.iter().enumerate() { + // --- everything below holds with NO event ever produced ---------------- + + // The burst terminates on an update the subscriber is guaranteed to see, + // and it is the *last* thing delivered — not buried mid-stream. + let (end_idx, end) = delivered + .iter() + .enumerate() + .find(|(_, u)| u.snapshot_end) + .expect( + "the burst must deliver its final snapshot even when it overran the sink — \ + otherwise a static subscription can never tell truncated from complete", + ); + assert_eq!( + end_idx, + delivered.len() - 1, + "the snapshot_end update must be the last one delivered" + ); + assert_eq!( + delivered.iter().filter(|u| u.snapshot_end).count(), + 1, + "exactly one update may close the burst" + ); + + // It is genuinely the burst's last record, so "initial state complete" + // means what it says. + assert_eq!( + end.topic.as_deref(), + Some(format!("rec.{SNAPSHOT_RECORDS}").as_str()) + ); + assert_eq!(&*end.data, format!("snap#{SNAPSHOT_RECORDS}").as_bytes()); + + // What landed before it is the contiguous head of the burst; all the loss + // is folded into the closing update. + for (i, update) in delivered[..end_idx].iter().enumerate() { assert_eq!( update.topic.as_deref(), Some(format!("rec.{}", i + 1).as_str()), @@ -637,10 +681,22 @@ async fn oversized_snapshot_burst_surfaces_as_a_gap() { "the delivered head of the burst is contiguous" ); } + assert!( + end.skipped > 0, + "this burst overran the sink, so the closing update must report the loss" + ); + + // The whole point, stated as a balance the consumer can compute itself, + // without waiting for anything further: every matched record is either + // delivered or accounted for as skipped. + let reported: u64 = delivered.iter().map(|u| u.skipped).sum(); + assert_eq!( + delivered.len() as u64 + reported, + SNAPSHOT_RECORDS as u64, + "every matched record must be either delivered or reported lost" + ); - // The sink is empty again, so one live event gets through — and it must - // report every snapshot lost at the tail of the burst. Pre-fix this event - // arrived with `skipped == 0`. + // --- and the sequence is closed, so live events resume clean ------------ event_tx .send(SubUpdate::new(payload_from("live"))) .expect("subscription stream should still be feeding"); @@ -651,19 +707,10 @@ async fn oversized_snapshot_burst_surfaces_as_a_gap() { .expect("a live event must not be a terminal error"); assert_eq!(&*event.data, b"live"); + assert!(!event.snapshot_end, "a live event does not close a burst"); assert_eq!( - event.skipped, - (SNAPSHOT_RECORDS - delivered.len()) as u64, - "the first event must report the snapshots dropped at the tail of the burst" - ); - - // The whole point, stated as a balance: every matched record is either - // delivered or accounted for as skipped — nothing goes missing quietly. - let reported: u64 = delivered.iter().map(|u| u.skipped).sum::() + event.skipped; - assert_eq!( - delivered.len() as u64 + reported, - SNAPSHOT_RECORDS as u64, - "every matched record must be either delivered or reported lost" + event.skipped, 0, + "the closing snapshot already accounted for the burst; the event adds no phantom gap" ); drop(handle); diff --git a/aimdb-websocket-connector/CHANGELOG.md b/aimdb-websocket-connector/CHANGELOG.md index 5c18d522..9659988e 100644 --- a/aimdb-websocket-connector/CHANGELOG.md +++ b/aimdb-websocket-connector/CHANGELOG.md @@ -9,12 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed (breaking, wire) -- **`snap` frames now carry `seq`.** Late-join snapshots are numbered in the - subscription's sequence space (`1..=N`), and the first `event` continues at - `N + 1` rather than restarting at `1` — so a snapshot dropped by a slow - client is visible as a gap instead of vanishing (see `aimdb-core`). Clients - reading the golden frame shape must expect `"seq"` on `snap` and an event - sequence offset by the snapshot count. +- **`snap` frames now carry `seq`, and the burst's last one carries `last`.** + Late-join snapshots are numbered in the subscription's sequence space + (`1..=N`), and the first `event` continues at `N + 1` rather than restarting + at `1` — so a snapshot dropped by a slow client is visible as a gap instead of + vanishing. The final `snap` of a burst adds `"last":true`, which the client + engine reserves a sink slot for; it surfaces as `SubUpdate::snapshot_end` and + closes out the initial state without needing a live event (see `aimdb-core`). + Clients reading the golden frame shape must expect `"seq"` on every `snap`, + `"last"` on the final one, and an event sequence offset by the snapshot count. ### Fixed diff --git a/aimdb-websocket-connector/tests/e2e.rs b/aimdb-websocket-connector/tests/e2e.rs index 450eff73..9d62218e 100644 --- a/aimdb-websocket-connector/tests/e2e.rs +++ b/aimdb-websocket-connector/tests/e2e.rs @@ -300,10 +300,11 @@ async fn server_late_join_snapshot() { // The snapshot rides between the ack and the first event, tagged with the // subscription that triggered it and numbered in that subscription's `seq` // space (events continue after the burst) so a dropped snapshot shows up as - // a gap rather than vanishing. + // a gap rather than vanishing. `last` closes the burst — here the only + // snapshot is also the final one. assert_eq!( ws_recv(&mut c).await, - json!({"t":"snap","sub":"4","seq":1,"topic":"sensors.temp","data":99}) + json!({"t":"snap","sub":"4","seq":1,"last":true,"topic":"sensors.temp","data":99}) ); } @@ -317,15 +318,25 @@ async fn server_wildcard_late_join_snapshots_per_match() { let mut c = ws_connect(addr).await; ws_send(&mut c, json!({"t":"sub","id":9,"topic":"sensors.#"})).await; assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"9"})); - // One snapshot per cached record under the pattern (order is map order). + // One snapshot per cached record under the pattern (order is map order), + // numbered 1..=N with only the final one flagged `last` — that flag is what + // lets a client close out the burst without waiting for a live event. let mut snaps = Vec::new(); - for _ in 0..2 { + let mut flags = Vec::new(); + for i in 1..=2 { let s = ws_recv_tag(&mut c, "snap").await; assert_eq!(s["sub"], "9"); + assert_eq!(s["seq"], i, "snapshots are numbered in burst order"); snaps.push(s["topic"].as_str().unwrap().to_string()); + flags.push(s["last"].as_bool().unwrap_or(false)); } snaps.sort(); assert_eq!(snaps, vec!["sensors.humidity", "sensors.temp"]); + assert_eq!( + flags, + vec![false, true], + "only the burst's final snapshot carries `last`" + ); } #[tokio::test] @@ -598,7 +609,7 @@ async fn golden_wire_frames() { assert_eq!(ws_recv(&mut c).await, json!({"t":"subscribed","sub":"1"})); assert_eq!( ws_recv(&mut c).await, - json!({"t":"snap","sub":"1","seq":1,"topic":"t","data":5}) + json!({"t":"snap","sub":"1","seq":1,"last":true,"topic":"t","data":5}) ); // The event continues the snapshot burst's `seq` (one snapshot, so `seq:2`) diff --git a/docs/design/047-retire-ws-protocol-converge-on-aimx.md b/docs/design/047-retire-ws-protocol-converge-on-aimx.md index 42196305..76df8865 100644 --- a/docs/design/047-retire-ws-protocol-converge-on-aimx.md +++ b/docs/design/047-retire-ws-protocol-converge-on-aimx.md @@ -73,7 +73,7 @@ AimX wire tags from `session/aimx/codec.rs`; ws-protocol messages from | Subscribe ack | `Subscribed{topics}` | `{"t":"subscribed","sub":S}` (**new**) | §3.2. | | Unsubscribe | `Unsubscribe{topics}` | `{"t":"unsub","sub":S}` | By sub id, not topic. | | Live data | `Data{topic,payload,ts}` | `{"t":"event","sub":S,"seq":N,"topic":T,"data":V}` | `topic` is **new**, present when the server tags it (always on WS; on wildcard subs elsewhere). Server-side `ts` is dropped — timestamps belong to the record layer / query results. `seq` is new for WS clients (drop detection). | -| Late-join snapshot | `Snapshot{topic,payload}` | `{"t":"snap","sub":S,"seq":N,"topic":T,"data":V}` | `sub` and `seq` are **new** (§3.3). `seq` shares the subscription's event sequence space: the burst is `1..=N` and the first `event` continues at `N+1`, so a dropped snapshot is a detectable gap. | +| Late-join snapshot | `Snapshot{topic,payload}` | `{"t":"snap","sub":S,"seq":N,"topic":T,"data":V}`, final frame adds `"last":true` | `sub`, `seq` and `last` are **new** (§3.3). `seq` shares the subscription's event sequence space: the burst is `1..=N` and the first `event` continues at `N+1`, so a dropped snapshot is a detectable gap. `last` terminates the burst so the gap lands even with no event. | | Client write | `Write{topic,payload}` | `{"t":"write","topic":T,"payload":V}` | Identical semantics (fire-and-forget, producer/arbiter path). | | Keepalive | `Ping`/`Pong` | `{"t":"ping"}` / `{"t":"pong"}` | Identical. | | Discovery | `ListTopics` over a raw socket | `record.list` req over a raw socket | UI's `discoverTopics` and `WasmDb.discover` reissue as AimX. | @@ -158,11 +158,27 @@ pending RPC on the connection, so a subscriber that (say) awaits an RPC before draining would deadlock itself. So snapshots take the same route as events: they are numbered in the subscription's sequence space (`1..=N`, events continuing at `N+1`), dropped on a full sink, and the shortfall folds into the -next delivered update's `SubUpdate::skipped`. Numbering them jointly with -events is what makes a loss at the *tail* of the burst recoverable — the first -event's `seq` reveals it. The residual blind spot is a tail loss on a -subscription that never fires another event; closing that needs an explicit -end-of-burst marker, deliberately deferred. +next delivered update's `SubUpdate::skipped`. + +Numbering alone is not enough, because a gap only *reaches* the subscriber on an +update that is actually delivered. If the burst is truncated at its tail, the +next delivered update is the first live event — and a **static** subscription +may never produce one, leaving the consumer unable to tell a complete initial +state from a truncated one. So the burst is explicitly terminated: the server +flags its final snapshot (`"last":true`), and the client holds one sink slot in +reserve through the burst so that frame is always deliverable. Because only the +demux loop sends, an observed free slot cannot be taken by anyone else — the +final delivery is infallible, not merely likely. + +The subscriber therefore sees exactly one update with `SubUpdate::snapshot_end` +set, carrying the burst's whole loss count in `skipped`, and can audit "one +snapshot per matched record" the moment the burst ends. Backpressure was +rejected as the alternative: the client demux loop also carries every other +subscription and every pending RPC on the connection, so blocking it on a full +sink would deadlock any consumer that awaits an RPC before draining. The one +remaining gap is definitional — a pattern matching *no* records emits no burst, +so "nothing matched" and "nothing yet" stay indistinguishable on a silent +subscription. ### 3.4 Query / list result shapes (DECIDED) From 153a9da0ca690deeefa86bd87a93dd28d36d7f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 28 Jul 2026 08:26:41 +0000 Subject: [PATCH 40/45] feat: enhance snapshot handling and error logging in client-server communication --- aimdb-core/CHANGELOG.md | 23 +++++---- aimdb-core/src/session/client.rs | 8 +++ aimdb-core/src/session/mod.rs | 12 ++++- aimdb-core/src/session/server.rs | 49 ++++++++++++------- ...047-retire-ws-protocol-converge-on-aimx.md | 31 +++++++++--- 5 files changed, 88 insertions(+), 35 deletions(-) diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index aac28dee..d4c670a3 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -45,17 +45,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Because a gap only reaches a subscriber on an update that is *actually delivered*, mid-burst snapshots now stop one slot short of filling the sink, - reserving it for the flagged final snapshot. That one is therefore always - delivered, arriving as the single update with the new - `SubUpdate::snapshot_end` set and carrying the burst's whole loss count. A - subscriber can tell a complete initial state from a truncated one the moment - the burst ends — without waiting for a live event, which a *static* - subscription may never produce. (A `topic` matching no records emits no burst, - so nothing carries the flag.) + reserving it for the flagged final snapshot — which therefore survives an + overrun, arriving as the single update with the new `SubUpdate::snapshot_end` + set and carrying the burst's whole loss count. A subscriber can tell a + complete initial state from a truncated one the moment the burst ends, without + waiting for a live event that a *static* subscription may never produce. + + The flag is not an unconditional promise, so do not *block* on it: it rides + the final snapshot's frame and is lost with it if that frame fails to encode + or is rejected as malformed, and a `topic` matching no records emits no burst + at all. Loss accounting survives all of these — only the end-of-burst signal + goes missing. Also fixes two smaller leaks on that path: a snapshot dropped by an encode - failure is now counted as loss, and a snapshot for a subscription whose - receiver is gone now prunes the sub instead of lingering. + failure is now counted as loss (and logged, rather than skipped in silence), + and a snapshot for a subscription whose receiver is gone now prunes the sub + instead of lingering. Wire: `snap` frames carry `"seq"` (one without it is `Malformed`) and the burst's last frame adds `"last":true`; mid-burst frames are byte-identical to diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index f4783e53..5df80e13 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -154,6 +154,14 @@ impl ClientHandle { /// terminal `Err(`[`RpcError`]`)` item, letting the caller distinguish a /// denied subscription (do not retry) from a disconnect-shaped stream end /// (retry to resume). + /// + /// Late-join snapshots arrive first, the last of them flagged + /// [`SubUpdate::snapshot_end`] with the burst's total loss in `skipped` (see + /// [`Session::snapshots`](super::Session::snapshots)). Read that flag to + /// close out the initial state, but drive the stream normally rather than + /// looping *until* it: a subscription matching no records never emits one, + /// and neither does one whose final snapshot frame was lost — the stream + /// stays open and simply proceeds to live events. pub fn subscribe( &self, topic: impl Into, diff --git a/aimdb-core/src/session/mod.rs b/aimdb-core/src/session/mod.rs index 1429f7c5..0a8856a9 100644 --- a/aimdb-core/src/session/mod.rs +++ b/aimdb-core/src/session/mod.rs @@ -464,8 +464,16 @@ pub trait Session: Send { /// consumer. Its `skipped` then carries the burst's whole loss, letting a /// subscriber distinguish a complete initial state from a truncated one /// *without* waiting for a live event, which a static subscription may - /// never produce. A `topic` matching no records emits no burst, and so no - /// such update. + /// never produce. + /// + /// That covers the slow consumer, which is the case worth engineering for, + /// but it is not an unconditional promise and a subscriber must not *block* + /// on it: no such update arrives when `topic` matches no records (there is + /// no burst), nor when the final snapshot fails to encode or its frame is + /// rejected as malformed — the flag rides that frame and is lost with it. + /// Loss accounting itself survives all of these (the shortfall still folds + /// into the next delivered update's `skipped`); only the end-of-burst + /// signal is missing. Treat end-of-stream as terminal too. fn snapshots(&mut self, topic: &str) -> Vec<(String, Payload)> { let _ = topic; Vec::new() diff --git a/aimdb-core/src/session/server.rs b/aimdb-core/src/session/server.rs index 384e63f5..72a619d0 100644 --- a/aimdb-core/src/session/server.rs +++ b/aimdb-core/src/session/server.rs @@ -235,23 +235,38 @@ pub async fn run_session( for (i, (snap_topic, data)) in snaps.into_iter().enumerate() { seq += 1; out.clear(); - if codec - .encode( - Outbound::Snapshot { - sub: &sub_id, - seq, - // The client reserves a sink slot - // for this one, so the burst's - // loss total always lands. - last: i + 1 == total, - topic: &snap_topic, - data, - }, - &mut out, - ) - .is_ok() - && conn.send(&out).await.is_err() - { + let encoded = codec.encode( + Outbound::Snapshot { + sub: &sub_id, + seq, + // The client reserves a sink slot + // for this one, so the burst's + // loss total always lands. + last: i + 1 == total, + topic: &snap_topic, + data, + }, + &mut out, + ); + if let Err(_e) = encoded { + // A record whose serialized bytes aren't + // valid for this envelope — broken for + // every read of it, not just here, so say + // so rather than dropping it in silence. + // `seq` already advanced, so the client + // still counts it as loss; if this was the + // burst's last snapshot the `last` flag + // goes with it and no `snapshot_end` + // reaches the subscriber. + log_warn!( + "snapshot encode failed, skipping: sub={} topic={} err={:?}", + sub_id, + snap_topic, + _e + ); + continue; + } + if conn.send(&out).await.is_err() { send_failed = true; break; } diff --git a/docs/design/047-retire-ws-protocol-converge-on-aimx.md b/docs/design/047-retire-ws-protocol-converge-on-aimx.md index 76df8865..129a427a 100644 --- a/docs/design/047-retire-ws-protocol-converge-on-aimx.md +++ b/docs/design/047-retire-ws-protocol-converge-on-aimx.md @@ -166,19 +166,36 @@ next delivered update is the first live event — and a **static** subscription may never produce one, leaving the consumer unable to tell a complete initial state from a truncated one. So the burst is explicitly terminated: the server flags its final snapshot (`"last":true`), and the client holds one sink slot in -reserve through the burst so that frame is always deliverable. Because only the -demux loop sends, an observed free slot cannot be taken by anyone else — the -final delivery is infallible, not merely likely. +reserve through the burst so that frame always has somewhere to land. Against +sink pressure this is infallible rather than merely likely: only the demux loop +sends on that channel and the consumer only removes, so a slot observed free +stays free; each subscription has its own channel; and the server emits the +whole burst inline in the Subscribe handler before starting the event pump, so +none of that subscription's events can interleave and take the slot. The subscriber therefore sees exactly one update with `SubUpdate::snapshot_end` set, carrying the burst's whole loss count in `skipped`, and can audit "one snapshot per matched record" the moment the burst ends. Backpressure was rejected as the alternative: the client demux loop also carries every other subscription and every pending RPC on the connection, so blocking it on a full -sink would deadlock any consumer that awaits an RPC before draining. The one -remaining gap is definitional — a pattern matching *no* records emits no burst, -so "nothing matched" and "nothing yet" stay indistinguishable on a silent -subscription. +sink would deadlock any consumer that awaits an RPC before draining. + +The reserve bounds *sink overflow*, which is the failure this design exists to +fix, and nothing else. The flag rides the final snapshot's frame, so it is lost +whenever that frame is: if the payload fails to encode (`log_warn` + skip) or +the client rejects the frame as malformed, no `snapshot_end` arrives — and a +pattern matching *no* records emits no burst at all. Loss accounting survives +all three (the shortfall still folds into the next delivered update's +`skipped`); only the end-of-burst signal goes missing, so consumers must drive +the stream normally rather than looping until the flag. Hardening these was +weighed and declined: requiring `"last"` on every frame would tax every +mid-burst snapshot to guard a version skew the release process already +prevents, and a one-frame server-side lookahead would buy an encode failure +that must *also* land on the burst's last record of a *static* subscription +before it differs from an ordinary broken record. Closing them properly needs a +terminator independent of the snapshot payloads — the burst size `N` in the +subscribe ack, or a dedicated `snapend` frame — deferred until something +demands it. ### 3.4 Query / list result shapes (DECIDED) From 4d4afa1c3a1ea7f50372f47a479bab4b08af7fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 28 Jul 2026 10:23:04 +0000 Subject: [PATCH 41/45] feat: implement cancellation for abandoned calls in client engine to prevent pending entry accumulation --- aimdb-core/CHANGELOG.md | 3 + aimdb-core/src/session/client.rs | 355 ++++++++++++++++++++-------- aimdb-core/tests/session_engine.rs | 72 +++++- aimdb-wasm-adapter/src/ws_bridge.rs | 8 +- 4 files changed, 329 insertions(+), 109 deletions(-) diff --git a/aimdb-core/CHANGELOG.md b/aimdb-core/CHANGELOG.md index d4c670a3..72b5945c 100644 --- a/aimdb-core/CHANGELOG.md +++ b/aimdb-core/CHANGELOG.md @@ -123,6 +123,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A non-terminal `#` no longer matches everything after it (`topic_matches`, `pattern_contains`).** Both walks returned `true` the moment they saw a `#`, so every segment following it was dead text: `topic_matches("a.#.secret", "a.public")` and `pattern_contains("a.#.secret", "a.public")` were both `true`, despite the module documenting `#` as usable at any position. On the ACL path this widened coverage — a `Permissions::can_subscribe` grant of that shape behaved like a blanket `a.#` grant over the whole subtree. `#` now absorbs **zero or more** segments with the pattern continuing afterwards (RabbitMQ topic-exchange semantics, as documented), so `a.#.secret` matches `a.secret` and `a.b.secret` but not `a.public`. Both functions share one greedy walk that retains a single `#` backtrack point, bounding it at `O(|pattern| · |subject|)` — relevant because `pattern_contains`'s subject is the client-supplied subscription pattern. Grants and subscriptions using a trailing `#` (`a.#`, `#`) or `*` are unaffected; the change only ever tightens what an interior `#` admits. - **AimX protocol doc rot cleaned up; `remote::PROTOCOL_VERSION` corrected to `"2.0"` and exported.** The `remote` module docs claimed "AimX v1" and linked a spec file that no longer exists; they now describe the v2 NDJSON tagged-frame wire and point at `crate::session::aimx` / `docs/design/remote-access-via-connectors.md`. The AimX dispatch's Welcome uses the constant instead of a hardcoded `"2.0"` (same bytes on the wire). The dead, never-exported v1 `Message` untagged envelope and its helpers were removed from `remote::protocol`. Also de-advertised Kafka/HTTP connector semantics from `ConnectorUrl` docs (the parser is scheme-agnostic; those connectors never existed) and updated the `connector` module docs from the removed `.link()` API to `.link_to()`/`.link_from()`. +- **An abandoned RPC call no longer leaks its pending-call entry (`run_client`).** Dropping a `ClientHandle::call` future — a timeout losing the race, an aborted task — dropped only the caller's `oneshot::Receiver`, which the engine could not observe: a peer that holds the connection open while never answering produces neither a `Reply` nor a disconnect, the only two things that removed an entry. The demux map therefore grew by one live `oneshot::Sender` per timed-out request for the life of the connection — the shape `aimdb-wasm-adapter`'s `query_timeout_ms` produces, and unbounded for a long-lived link against an unresponsive peer. `ClientHandle::call` is now **cancel-safe**: the dropped future emits an internal `CancelCall` naming its own request, and the engine frees the entry by id in `O(1)`. Reclamation is exact and immediate — it needs no later call, no keepalive tick (the fix holds with `keepalive_interval: None`), and no scan of the table, so a concurrent batch of timeouts costs `O(N)`, not `O(N²)`. A call abandoned while its command is still queued is additionally never dialed out: no wire request is spent on a reply nobody awaits. + + To make that race-free, correlation ids move from the engine to `ClientHandle`: the id exists before the command is queued, so the cancellation that follows it on the same FIFO channel always names an entry the engine has already created (with engine-assigned ids there is a window in which the caller gives up before learning its id and has nothing to name). Calls and subscriptions draw from one shared `AtomicUsize` — `usize` rather than `u64` because 64-bit atomics are not native on every supported MCU (thumbv7em), which only bounds how many ids one process can issue, all of them JSON-safe. Wire-visible consequence: ids are **monotonic across reconnects** instead of restarting at `1` per connection, so a redial never reuses an id (`0` stays free as a "no correlation" sentinel). No protocol change — cancellation is engine-internal and the server is never asked to abandon in-flight work. - **`build()` reports a missing runtime alongside every other configuration error (issue #133 contract).** The missing-runtime check no longer short-circuits: it is collected as a `ConfigError` and returned in the one `DbError::InvalidConfiguration` with all other findings (previously the collected errors were silently dropped and only a `RuntimeError` surfaced). The error type for a runtime-less build changes accordingly from `DbError::RuntimeError` to `DbError::InvalidConfiguration`. ### Changed (breaking) diff --git a/aimdb-core/src/session/client.rs b/aimdb-core/src/session/client.rs index 5df80e13..7d108d0a 100644 --- a/aimdb-core/src/session/client.rs +++ b/aimdb-core/src/session/client.rs @@ -16,6 +16,7 @@ use alloc::boxed::Box; use alloc::string::{String, ToString}; use alloc::sync::Arc; use alloc::vec::Vec; +use core::sync::atomic::{AtomicUsize, Ordering}; use async_channel::{Receiver, Sender}; use futures_channel::oneshot; @@ -99,17 +100,39 @@ fn bound_offline_queue(cmd_rx: &Receiver, cap: usize) { #[derive(Clone)] pub struct ClientHandle { cmd_tx: Sender, + /// Correlation ids, allocated caller-side and shared by every clone. + /// + /// The *caller* numbers its requests so that abandoning one is race-free: + /// the id is known before the command is queued, so the cancellation that + /// follows on the same FIFO channel always names an entry the engine has + /// already created (see [`CancelOnDrop`]). With engine-assigned ids there is + /// an unavoidable window in which the caller gives up before learning the id + /// and has nothing to cancel. Calls and subscriptions draw from this one + /// counter because they share an id space on the wire. + /// + /// `usize` rather than `u64`: 64-bit atomics are not native on every + /// supported MCU (thumbv7em), and the width only bounds the ids one process + /// can issue — 2^32 on a 32-bit target, all of them JSON-safe. + next_id: Arc, } -/// Commands the [`ClientHandle`] funnels to the engine (the engine assigns the -/// correlation `id`, so it stays the sole owner of the demux map). +/// Commands the [`ClientHandle`] funnels to the engine. The caller allocates the +/// correlation `id`; the engine owns the demux maps keyed by it. enum ClientCmd { Call { + id: u64, method: String, params: Payload, - reply: oneshot::Sender>, + reply: oneshot::Sender, + }, + /// The caller abandoned call `id` (its future was dropped). Frees the + /// pending-call entry immediately, without waiting for a reply that may + /// never come. + CancelCall { + id: u64, }, Subscribe { + id: u64, topic: String, events: Sender>, }, @@ -119,6 +142,31 @@ enum ClientCmd { }, } +/// Frees a call's engine-side state if the caller stops awaiting it. +/// +/// A dropped [`ClientHandle::call`] future takes the reply channel's receiver +/// with it, which the engine cannot observe: a peer that holds the connection +/// open but never answers produces neither a `Reply` nor a disconnect. The guard +/// turns that silent drop into a [`ClientCmd::CancelCall`], so reclamation is +/// exact and immediate — it needs no later call, no keepalive tick, and no scan +/// of the pending table. +struct CancelOnDrop<'a> { + handle: &'a ClientHandle, + id: u64, + /// Cleared once the call resolves — a completed call has already left the + /// table, so cancelling it would only add channel traffic. + armed: bool, +} + +impl Drop for CancelOnDrop<'_> { + fn drop(&mut self) { + if self.armed { + // A closed channel means the engine is already gone with its table. + let _ = self.handle.enqueue(ClientCmd::CancelCall { id: self.id }); + } + } +} + impl ClientHandle { /// Funnel a command to the engine. The channel is unbounded, so `try_send` /// never blocks and only fails once the engine has stopped (receiver closed). @@ -126,21 +174,42 @@ impl ClientHandle { self.cmd_tx.try_send(cmd).map_err(|_| RpcError::Internal) } + /// Allocate the next correlation id. Monotonic across reconnects, so an id + /// is never reused by a later connection. + fn next_id(&self) -> u64 { + self.next_id.fetch_add(1, Ordering::Relaxed) as u64 + } + /// One-shot RPC: send a request and await its single reply. Returns /// [`RpcError::Internal`] if the engine has stopped or the connection drops /// before the reply arrives. + /// + /// Cancel-safe: dropping this future (a timeout losing the race, an aborted + /// task) releases the engine's pending-call entry rather than leaving it + /// until the connection ends. pub async fn call( &self, method: impl Into, params: Payload, ) -> Result { + let id = self.next_id(); let (reply, rx) = oneshot::channel(); self.enqueue(ClientCmd::Call { + id, method: method.into(), params, reply, })?; - rx.await.map_err(|_| RpcError::Internal)? + // Armed across the await only: from here, any exit path that isn't the + // reply itself tells the engine to drop the entry. + let mut cancel = CancelOnDrop { + handle: self, + id, + armed: true, + }; + let reply = rx.await; + cancel.armed = false; + reply.map_err(|_| RpcError::Internal)? } /// Open a subscription; returns the stream of updates immediately (the engine @@ -169,6 +238,7 @@ impl ClientHandle { let (events, rx) = async_channel::bounded::>(SUBSCRIBE_CHANNEL_CAP); self.enqueue(ClientCmd::Subscribe { + id: self.next_id(), topic: topic.into(), events, })?; @@ -205,7 +275,12 @@ where C: EnvelopeCodec + 'static, { let (cmd_tx, cmd_rx) = async_channel::unbounded(); - let handle = ClientHandle { cmd_tx }; + let handle = ClientHandle { + cmd_tx, + // Ids start at 1: `0` stays free as a "no correlation" sentinel for + // protocols that want one. + next_id: Arc::new(AtomicUsize::new(1)), + }; let fut = Box::pin(client_loop(dialer, codec, config, cmd_rx, clock)); (handle, fut) } @@ -402,36 +477,14 @@ type CallReply = Result; /// channel. type PendingCalls = HashMap>; -/// Drop every entry whose caller has gone away. -/// -/// A caller that gives up on [`ClientHandle::call`] — a timeout racing the -/// future, or any other cancellation — drops only its `oneshot::Receiver`; the -/// matching sender stays in the engine's map, since a peer that holds the -/// connection open but never answers produces neither a `Reply` nor a -/// disconnect. `futures_channel` records the drop on the sender, so this sweep -/// is a flag check per entry with no wakeups or allocation. -fn prune_canceled(pending: &mut PendingCalls) { - pending.retain(|_, tx| !tx.is_canceled()); -} - -/// Register a call's reply channel, first reclaiming any abandoned entry. -/// -/// Pruning here (rather than per frame) is what bounds the table: the sweep is -/// O(in-flight calls) and runs only when a new call is issued, so a stream of -/// timed-out requests over a live connection costs one lingering entry each -/// until the next call, instead of one for the life of the connection. -fn track_call(pending: &mut PendingCalls, id: u64, reply: oneshot::Sender) { - prune_canceled(pending); - pending.insert(id, reply); -} - /// Drive one dialed [`Connection`]: optional handshake, then `biased` demux of /// server frames (resolve `Reply` by `id`, route `Event`/`Snapshot` to their /// subscription channels) interleaved with caller commands. Pending state is /// per-connection: a disconnect fails outstanding calls (their `oneshot` /// senders drop → callers see [`RpcError::Internal`]). Calls the *caller* -/// abandoned are reclaimed by [`prune_canceled`], so an unanswering peer that -/// keeps the link open can't grow the table. +/// abandoned arrive as [`ClientCmd::CancelCall`] and are removed by id in O(1), +/// so an unanswering peer that keeps the link open can't grow the table — with +/// no dependence on later traffic or on the keepalive being enabled. async fn drive_connection( mut conn: Box, codec: &C, @@ -442,7 +495,6 @@ async fn drive_connection( where C: EnvelopeCodec + ?Sized, { - let mut next_id: u64 = 1; let mut pending: PendingCalls = HashMap::new(); // sub-id → event sink. The sub-id is `id.to_string()` of the opening // request, matching the server's derivation so `Event.sub` routes back. @@ -579,10 +631,6 @@ where } ClientStep::Keepalive => { - // Also the reclaim tick: a caller that times out and then stops - // issuing calls would otherwise hold its entries until the next - // call or the disconnect. - prune_canceled(&mut pending); // `keepalive_timer` is `Some` whenever this step fires. let interval_ms = keepalive_ms.unwrap_or(0); let idle_ms = clock.now_nanos().saturating_sub(last_activity) / 1_000_000; @@ -618,20 +666,19 @@ where }; match cmd { ClientCmd::Call { + id, method, params, reply, } => { // The caller may have given up while the command sat in - // the queue — don't spend a wire request (or an `id`) on - // a reply nobody is waiting for. + // the queue — don't spend a wire request on a reply + // nobody is waiting for. Its `CancelCall` is already + // queued behind this one and will find nothing to free. if reply.is_canceled() { - prune_canceled(&mut pending); continue; } - let id = next_id; - next_id += 1; - track_call(&mut pending, id, reply); + pending.insert(id, reply); out.clear(); let sent = codec .encode_inbound(Inbound::Request { id, method, params }, &mut out) @@ -644,9 +691,14 @@ where return Ended::Disconnected; } } - ClientCmd::Subscribe { topic, events } => { - let id = next_id; - next_id += 1; + // The caller abandoned this call: drop its reply channel now + // rather than holding it until a reply that may never arrive + // (or the disconnect). A resolved or never-dispatched call + // is simply absent — removing it is a no-op. + ClientCmd::CancelCall { id } => { + pending.remove(&id); + } + ClientCmd::Subscribe { id, topic, events } => { subs.insert(id.to_string(), events); out.clear(); let sent = codec @@ -759,72 +811,175 @@ pub fn pump_client(db: &AimDb, scheme: &str, handle: &ClientHandle) -> Vec (oneshot::Sender, oneshot::Receiver) { - oneshot::channel() + use crate::session::{CodecError, PeerInfo, TransportResult}; + use core::future::pending; + + /// Sink connection: swallows every frame and never yields one, i.e. a peer + /// that keeps the link open and answers nothing. + #[derive(Default)] + struct SilentPeer { + peer: PeerInfo, } - /// A caller that gave up is reclaimed; one still awaiting its reply is not, - /// and stays deliverable. - #[test] - fn prune_canceled_drops_only_abandoned_calls() { - let mut pending = PendingCalls::new(); - let (tx1, rx1) = call_slot(); - let (tx2, rx2) = call_slot(); - pending.insert(1, tx1); - pending.insert(2, tx2); - - drop(rx1); // caller 1 timed out and dropped its `call` future - prune_canceled(&mut pending); - - assert_eq!(pending.len(), 1, "the abandoned call must be reclaimed"); - assert!(pending.contains_key(&2), "a live call must be kept"); - pending - .remove(&2) - .unwrap() - .send(Ok(Payload::from(&b"ok"[..]))) - .expect("the retained sender must still reach its caller"); - drop(rx2); + impl Connection for SilentPeer { + fn recv(&mut self) -> BoxFut<'_, TransportResult>>> { + Box::pin(pending()) + } + fn send<'a>(&'a mut self, _frame: &'a [u8]) -> BoxFut<'a, TransportResult<()>> { + Box::pin(async { Ok(()) }) + } + fn peer(&self) -> &PeerInfo { + &self.peer + } } - /// Regression: a peer that keeps the connection open but never replies used - /// to cost one permanent entry per timed-out request, since only a `Reply` - /// or a disconnect removed one. Repeated timeouts must not grow the table. - #[test] - fn repeated_timeouts_do_not_grow_pending() { - let mut pending = PendingCalls::new(); - for id in 1..=1_000u64 { - let (tx, rx) = call_slot(); - track_call(&mut pending, id, tx); - // The caller times out immediately; nothing ever replies. - drop(rx); - assert!( - pending.len() <= 1, - "pending grew to {} at id {id}; abandoned calls are leaking", - pending.len() - ); + /// Frames are never inspected in these tests — only the pending-call + /// bookkeeping is. + struct NullCodec; + + impl EnvelopeCodec for NullCodec { + fn decode(&self, _frame: &[u8]) -> Result { + Err(CodecError::Malformed) + } + fn encode(&self, _msg: Outbound<'_>, _out: &mut Vec) -> Result<(), CodecError> { + Ok(()) } + fn encode_inbound(&self, _msg: Inbound, _out: &mut Vec) -> Result<(), CodecError> { + Ok(()) + } + fn decode_outbound<'a>(&self, _frame: &'a [u8]) -> Result, CodecError> { + Err(CodecError::Malformed) + } + } + + fn test_handle() -> (ClientHandle, Receiver) { + let (cmd_tx, cmd_rx) = async_channel::unbounded(); + ( + ClientHandle { + cmd_tx, + next_id: Arc::new(AtomicUsize::new(1)), + }, + cmd_rx, + ) + } + + /// Poll a call once — enough to queue its command — then abandon it, which + /// is what a lost timeout race does to the future. + async fn abandon_call(handle: &ClientHandle, method: &'static str) { + let call = handle.call(method, Payload::from(&b"x"[..])); + assert!( + tokio::time::timeout(core::time::Duration::ZERO, call) + .await + .is_err(), + "the call must not resolve" + ); } - /// The reclaim sweep must not disturb calls that are still outstanding, so - /// a long-lived call survives a burst of timed-out ones around it. - #[test] - fn track_call_keeps_outstanding_calls_across_prunes() { - let mut pending = PendingCalls::new(); - let (slow_tx, slow_rx) = call_slot(); - track_call(&mut pending, 1, slow_tx); - - for id in 2..=100u64 { - let (tx, rx) = call_slot(); - track_call(&mut pending, id, tx); - drop(rx); + /// The caller half of the reclaim path: abandoning a call queues a + /// `CancelCall` naming exactly that call, so the engine never has to guess + /// which entry died. + #[tokio::test] + async fn abandoning_a_call_queues_its_cancellation() { + let (handle, cmd_rx) = test_handle(); + abandon_call(&handle, "one").await; + + match cmd_rx.try_recv() { + Ok(ClientCmd::Call { id: 1, .. }) => {} + _ => panic!("the call must be queued first"), } + match cmd_rx.try_recv() { + Ok(ClientCmd::CancelCall { id: 1 }) => {} + _ => panic!("abandoning the call must queue its cancellation"), + } + assert!(cmd_rx.is_empty(), "no other command is emitted"); + } + /// A call that resolves normally must not also queue a cancellation — the + /// engine already dropped its entry when it routed the reply. + #[tokio::test] + async fn a_resolved_call_queues_no_cancellation() { + let (handle, cmd_rx) = test_handle(); + let call = handle.call("one", Payload::from(&b"x"[..])); + let answer = async { + // Take the reply channel out of the queued command and answer it. + match cmd_rx.recv().await { + Ok(ClientCmd::Call { reply, .. }) => { + let _ = reply.send(Ok(Payload::from(&b"ok"[..]))); + } + _ => panic!("the call must be queued"), + } + }; + let (reply, ()) = futures_util::future::join(call, answer).await; + assert_eq!(&*reply.expect("the call resolves"), b"ok"); assert!( - pending.contains_key(&1), - "an outstanding call must survive the reclaim sweep" + cmd_rx.is_empty(), + "a resolved call must not queue a cancellation" ); - assert_eq!(pending.len(), 2, "only the outstanding calls remain"); - drop(slow_rx); + } + + /// Regression for the *concurrent* timeout batch: every call is issued and + /// tracked while its receiver is live, then all of them are abandoned at + /// once, with keepalive disabled and no later call to trigger a sweep. + /// + /// Each entry the engine frees drops its reply sender, which the (retained) + /// receiver observes as `Canceled` — so this asserts the table actually + /// empties rather than that some reclaim path merely ran. Pre-fix all 1000 + /// entries stayed until the connection ended. + #[tokio::test] + async fn a_concurrent_batch_of_cancellations_frees_every_entry() { + const BATCH: u64 = 1_000; + let (cmd_tx, cmd_rx) = async_channel::unbounded::(); + let config = ClientConfig { + reconnect: false, + // The point of the test: nothing but the cancellations can reclaim. + keepalive_interval: None, + ..Default::default() + }; + let clock = crate::executor::test_support::NoopRuntimeOps; + let engine = drive_connection( + Box::new(SilentPeer::default()), + &NullCodec, + &cmd_rx, + &config, + &clock, + ); + + let exercise = async { + // All in flight at once: every receiver is alive as its call is + // tracked, so nothing is reclaimable until the batch is abandoned. + let mut replies = Vec::new(); + for id in 1..=BATCH { + let (reply, rx) = oneshot::channel(); + cmd_tx + .try_send(ClientCmd::Call { + id, + method: String::from("hang"), + params: Payload::from(&b"x"[..]), + reply, + }) + .expect("the channel is unbounded"); + replies.push(rx); + } + // Then the whole batch times out. + for id in 1..=BATCH { + cmd_tx + .try_send(ClientCmd::CancelCall { id }) + .expect("the channel is unbounded"); + } + for (i, rx) in replies.into_iter().enumerate() { + assert!( + rx.await.is_err(), + "call {} was still held by the engine", + i + 1 + ); + } + }; + + futures_util::pin_mut!(engine); + futures_util::pin_mut!(exercise); + match futures_util::future::select(engine, exercise).await { + futures_util::future::Either::Left(_) => panic!("the engine ended early"), + futures_util::future::Either::Right(((), _)) => {} + } } } diff --git a/aimdb-core/tests/session_engine.rs b/aimdb-core/tests/session_engine.rs index d3ba7dd1..c5bbdc05 100644 --- a/aimdb-core/tests/session_engine.rs +++ b/aimdb-core/tests/session_engine.rs @@ -846,15 +846,17 @@ async fn read_request(peer: &mut Box) -> (u64, String, String) { (id, method, parts.next().unwrap_or("").to_string()) } -/// Regression: callers that time out and drop their `call` future must not -/// accumulate in the engine's pending-call map. +/// Callers that time out and drop their `call` future must not accumulate in +/// the engine's pending-call map. /// /// A `Reply` or a disconnect used to be the only things that removed an entry, /// so a peer holding the connection open while never answering (the shape the /// WASM bridge's `query_timeout_ms` produces) cost one live `oneshot::Sender` -/// per timed-out request for the life of the connection. The engine now reclaims -/// abandoned entries; this test drives that path end-to-end and pins the -/// property it must not break — reply correlation still works afterwards. +/// per timed-out request for the life of the connection. Each abandoned call now +/// cancels itself by id; the emptying of the table is asserted in the engine's +/// own unit tests (`session::client::tests`), which can see it. This drives the +/// same path end-to-end and pins the property the reclaim must not break — +/// reply correlation still works afterwards. #[tokio::test] async fn repeated_call_timeouts_leave_the_connection_usable() { let (mut listener, dialer) = transport_pair(); @@ -943,3 +945,63 @@ async fn a_call_abandoned_before_dispatch_is_never_sent() { drop(peer); let _ = client.await; } + +/// The concurrent shape of the same regression: every call is in flight (so +/// none is reclaimable) and the whole batch then times out at once, with no +/// later call and no keepalive to drive a sweep. Each abandoned call carries its +/// own cancellation, so the engine frees all of them and stays usable. +#[tokio::test] +async fn a_concurrent_batch_of_timeouts_leaves_the_connection_usable() { + const BATCH: usize = 256; + let (mut listener, dialer) = transport_pair(); + let (handle, client_fut) = + run_client(dialer, LineCodec, silent_peer_config(), Arc::new(TestClock)); + let client = tokio::spawn(client_fut); + let mut peer = listener.accept().await.expect("the client must dial"); + + let batch: Vec<_> = (0..BATCH) + .map(|i| { + let handle = handle.clone(); + tokio::spawn(async move { + tokio::time::timeout( + Duration::from_millis(50), + handle.call("hang", payload_from(&format!("{i}"))), + ) + .await + }) + }) + .collect(); + + // Every request reaches the peer while its caller is still waiting. + for _ in 0..BATCH { + let (_, method, _) = read_request(&mut peer).await; + assert_eq!(method, "hang", "each request must reach the peer"); + } + for task in batch { + assert!( + task.await.expect("the call task must not panic").is_err(), + "an unanswered call must not resolve" + ); + } + + // Correlation is intact after the batch was reclaimed. + let answered = tokio::spawn({ + let handle = handle.clone(); + async move { handle.call("ping", payload_from("hi")).await } + }); + let (id, method, params) = read_request(&mut peer).await; + assert_eq!((method.as_str(), params.as_str()), ("ping", "hi")); + peer.send(format!("REPLY\n{id}\nOK\npong").as_bytes()) + .await + .expect("the pipe must stay open"); + let reply = tokio::time::timeout(Duration::from_secs(2), answered) + .await + .expect("the answered call must resolve") + .expect("the call task must not panic") + .expect("the peer replied with OK"); + assert_eq!(&*reply, b"pong"); + + drop(handle); + drop(peer); + let _ = client.await; +} diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index b93018d9..fdb585ff 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -614,10 +614,10 @@ impl WsBridge { let timeout = WasmAdapter.sleep(core::time::Duration::from_millis(timeout_ms as u64)); futures_util::pin_mut!(call); - // Losing the race drops `call`, i.e. the reply channel's - // receiver. The engine reclaims its side on the next call or - // keepalive tick, so timing out against an unanswering peer - // doesn't accumulate pending entries for the connection's life. + // Losing the race drops `call`, which cancels the request in the + // engine (`ClientHandle::call` is cancel-safe: the dropped + // future frees its pending-call entry by id). Timing out against + // an unanswering peer therefore costs nothing per request. match futures_util::future::select(call, timeout).await { futures_util::future::Either::Left((reply, _)) => reply, futures_util::future::Either::Right(((), _)) => { From 1987cc6dd549c0ee6c9835d921e1dcf4356a6e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 28 Jul 2026 10:36:38 +0000 Subject: [PATCH 42/45] feat: limit inbound frame funnel to prevent memory overflow and improve connection stability --- aimdb-wasm-adapter/CHANGELOG.md | 10 ++++++++++ aimdb-wasm-adapter/src/ws_bridge.rs | 31 +++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/aimdb-wasm-adapter/CHANGELOG.md b/aimdb-wasm-adapter/CHANGELOG.md index d03bd329..7bf8a645 100644 --- a/aimdb-wasm-adapter/CHANGELOG.md +++ b/aimdb-wasm-adapter/CHANGELOG.md @@ -47,6 +47,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed (behavioral) +- **`WsBridge`'s inbound frame funnel is bounded (1024 frames) and overflow ends + the connection.** The queue between the browser `onmessage` callback and the + engine's `recv` was unbounded, so a fast server or a main thread stuck in + synchronous JS could grow Rust-owned memory without limit. It now closes the + stream instead of dropping frames: the engine reads that as a disconnect, so + pending `query`/`listTopics` promises reject rather than hanging, and the + redial + re-subscribe re-syncs from scratch — the only sound recovery, since an + overflow gives no way to tell *what* was lost (a reply, events across topics, + part of a snapshot burst). Visible from JS as a `reconnecting` status flap plus + a console warning. - **JS/WASM `SingleLatest` subscribers may observe one extra initial delivery.** As a consequence of the fresh-subscriber fix above, `subscribe` / `subscribe_typed` (bindings and `WsBridge`) now fire an immediate callback with diff --git a/aimdb-wasm-adapter/src/ws_bridge.rs b/aimdb-wasm-adapter/src/ws_bridge.rs index fdb585ff..ac0844d3 100644 --- a/aimdb-wasm-adapter/src/ws_bridge.rs +++ b/aimdb-wasm-adapter/src/ws_bridge.rs @@ -176,8 +176,15 @@ impl BridgeShared { // ─── Transport: web_sys::WebSocket as Connection/Dialer ────────────────── +/// Depth of the funnel between the JS message callback and the engine's `recv`. +/// Bounded so a fast server (or a main thread stuck in synchronous JS) can't +/// grow Rust-owned browser memory without limit before the engine's own bounded +/// subscription sinks apply. Sized well above the engine's `SUBSCRIBE_CHANNEL_CAP` +/// so an ordinary late-join snapshot burst passes through untouched. +const FRAME_QUEUE_CAP: usize = 1024; + /// Frames arriving from JS event callbacks, funneled to the engine's `recv`. -type FrameRx = futures_channel::mpsc::UnboundedReceiver>; +type FrameRx = futures_channel::mpsc::Receiver>; /// A dialed browser WebSocket serving the engine's [`Connection`] contract. struct WasmWsConnection { @@ -279,7 +286,7 @@ impl Dialer for WasmWsDialer { // Frame funnel: onmessage pushes text frames; onclose closes it so // the engine's `recv` observes end-of-stream. - let (frame_tx, frames) = futures_channel::mpsc::unbounded::>(); + let (mut frame_tx, frames) = futures_channel::mpsc::channel::>(FRAME_QUEUE_CAP); // Open handshake: whichever of onopen/onclose fires first wins. let opened: Rc>>> = Rc::new(RefCell::new(None)); @@ -301,10 +308,26 @@ impl Dialer for WasmWsDialer { plain_callbacks.push(on_open); let on_message = { - let frame_tx = frame_tx.clone(); + let mut frame_tx = frame_tx.clone(); Closure::wrap(Box::new(move |event: web_sys::MessageEvent| { if let Some(text) = event.data().as_string() { - let _ = frame_tx.unbounded_send(text.into_bytes()); + // A full funnel means the engine fell far enough behind + // that we can no longer say *what* is being lost — a + // reply, events across topics, part of a snapshot burst. + // So don't drop frames: end the stream, which the engine + // reads as a disconnect (failing pending calls rather + // than leaving them unresolved) and redials. The pumps' + // re-subscribe then re-syncs from scratch — the only + // sound recovery for a loss of unknown shape. + if let Err(e) = frame_tx.try_send(text.into_bytes()) { + if e.is_full() { + web_sys::console::warn_1( + &"WsBridge: frame queue full — dropping the connection to resync" + .into(), + ); + frame_tx.close_channel(); + } + } } }) as Box) }; From ab7755e15e2d44d113acb91678e46612e3ceee9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 28 Jul 2026 18:33:15 +0000 Subject: [PATCH 43/45] feat: add headless browser toolchain for WASM tests and ensure Chrome-chromedriver version compatibility --- .devcontainer/Dockerfile | 28 ++++++++++++++++++ .github/workflows/devcontainer.yml | 17 +++++++++++ Makefile | 47 ++++++++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5081c53f..cac396fe 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -37,6 +37,34 @@ RUN apt-get update && apt-get install -y \ unzip \ && rm -rf /var/lib/apt/lists/* +# -------------------------------------------------------------------- +# Chrome + chromedriver for headless WASM tests (make wasm-test) +# +# chromedriver is pinned to the installed Chrome's major version. wasm-pack +# would otherwise download the newest driver on its own, which refuses to drive +# an older browser, and the bare driver binary also needs Chrome's shared libs +# (libnspr4 & friends) that only the browser package pulls in. +# -------------------------------------------------------------------- +RUN curl -fsSL https://dl.google.com/linux/linux_signing_key.pub \ + | gpg --dearmor -o /usr/share/keyrings/google-chrome.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/google-chrome.gpg] https://dl.google.com/linux/chrome/deb/ stable main" \ + > /etc/apt/sources.list.d/google-chrome.list \ + && apt-get update && apt-get install -y --no-install-recommends google-chrome-stable \ + && rm -rf /var/lib/apt/lists/* \ + && CHROME_MAJOR="$(google-chrome --version | sed -E 's/[^0-9]*([0-9]+).*/\1/')" \ + && DRIVER_URL="$(curl -fsSL https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json \ + | jq -r --arg m "${CHROME_MAJOR}." '[.versions[] | select(.version | startswith($m))] | last | .downloads.chromedriver[] | select(.platform == "linux64") | .url')" \ + && [ -n "$DRIVER_URL" ] && [ "$DRIVER_URL" != "null" ] \ + && curl -fsSL -o /tmp/chromedriver.zip "$DRIVER_URL" \ + && unzip -oq /tmp/chromedriver.zip -d /tmp \ + && install -m 0755 /tmp/chromedriver-linux64/chromedriver /usr/local/bin/chromedriver \ + && rm -rf /tmp/chromedriver.zip /tmp/chromedriver-linux64 \ + && chromedriver --version + +# Points wasm-pack/wasm-bindgen-test at the pinned driver instead of letting them +# download the newest one into ~/.cache/.wasm-pack. +ENV CHROMEDRIVER=/usr/local/bin/chromedriver + # -------------------------------------------------------------------- # Create non-root user (with sudo) as root # -------------------------------------------------------------------- diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 7f86c337..6413e434 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -125,6 +125,23 @@ jobs: pkg-config --version " + - name: Test headless browser toolchain + run: | + docker run --rm aimdb-devcontainer:test bash -c ' + set -e + google-chrome --version + chromedriver --version + # wasm-bindgen-test drives Chrome through chromedriver, which refuses a + # browser from a different major version. + chrome_major="$(google-chrome --version | sed -E "s/[^0-9]*([0-9]+).*/\1/")" + driver_major="$(chromedriver --version | sed -E "s/[^0-9]*([0-9]+).*/\1/")" + [ "$chrome_major" = "$driver_major" ] || { + echo "Chrome $chrome_major and chromedriver $driver_major disagree"; exit 1; + } + [ "$CHROMEDRIVER" = /usr/local/bin/chromedriver ] + google-chrome --headless --no-sandbox --disable-gpu --dump-dom about:blank >/dev/null + ' + - name: Test user permissions run: | docker run --rm aimdb-devcontainer:test bash -c " diff --git a/Makefile b/Makefile index da9d0dce..58bf10d8 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # AimDB Makefile # Simple automation for common development tasks -.PHONY: help build test clean clean-embedded fmt fmt-check clippy doc all check test-embedded test-wasm wasm wasm-test examples deny audit security publish publish-check readme-check codegen-drift check-no-sim +.PHONY: help build test clean clean-embedded fmt fmt-check clippy doc all check test-embedded test-wasm wasm wasm-test wasm-test-deps examples deny audit security publish publish-check readme-check codegen-drift check-no-sim .DEFAULT_GOAL := help # Separate target dir for embedded checks so an interrupted example build @@ -55,6 +55,7 @@ help: @printf " $(YELLOW)WASM Commands:$(NC)\n" @printf " wasm Build WASM adapter with wasm-pack\n" @printf " wasm-test Run WASM tests in headless browser\n" + @printf " wasm-test-deps Install Chrome + matching chromedriver for wasm-test\n" @printf "\n" @printf " $(YELLOW)Convenience:$(NC)\n" @printf " all Build everything\n" @@ -640,8 +641,50 @@ wasm-test: printf "$(YELLOW) ⚠ wasm-pack not found, installing...$(NC)\n"; \ cargo install wasm-pack --locked; \ fi - cd aimdb-wasm-adapter && wasm-pack test --headless --chrome + @if ! command -v google-chrome >/dev/null 2>&1 && ! command -v chromium >/dev/null 2>&1; then \ + printf "$(RED) ✗ No Chrome/Chromium on PATH — headless WASM tests need a browser.$(NC)\n"; \ + printf "$(YELLOW) Run 'make wasm-test-deps' (or rebuild the devcontainer).$(NC)\n"; \ + exit 1; \ + fi + @if ! command -v chromedriver >/dev/null 2>&1; then \ + printf "$(RED) ✗ chromedriver not on PATH.$(NC)\n"; \ + printf "$(YELLOW) Run 'make wasm-test-deps' — the copy wasm-pack downloads itself$(NC)\n"; \ + printf "$(YELLOW) tracks Chrome stable and drifts out of sync with the installed browser.$(NC)\n"; \ + exit 1; \ + fi + cd aimdb-wasm-adapter && CHROMEDRIVER="$$(command -v chromedriver)" wasm-pack test --headless --chrome @printf "$(GREEN)✓ WASM tests passed!$(NC)\n" +# Installs Chrome plus the chromedriver build that matches its major version. +# wasm-pack downloads a chromedriver on its own, but always the newest one, which +# refuses to drive an older Chrome ("only supports Chrome version N"). Pinning the +# driver to the installed browser is what keeps wasm-test reproducible. +wasm-test-deps: + @printf "$(GREEN)Installing headless WASM test dependencies...$(NC)\n" + @if ! command -v google-chrome >/dev/null 2>&1; then \ + printf "$(YELLOW) → Installing google-chrome-stable$(NC)\n"; \ + curl -fsSL https://dl.google.com/linux/linux_signing_key.pub \ + | sudo gpg --dearmor -o /usr/share/keyrings/google-chrome.gpg; \ + echo "deb [arch=$$(dpkg --print-architecture) signed-by=/usr/share/keyrings/google-chrome.gpg] https://dl.google.com/linux/chrome/deb/ stable main" \ + | sudo tee /etc/apt/sources.list.d/google-chrome.list >/dev/null; \ + sudo apt-get update -qq; \ + sudo apt-get install -y -qq google-chrome-stable unzip; \ + fi + @printf "$(YELLOW) → Installing matching chromedriver$(NC)\n" + @set -e; \ + major="$$(google-chrome --version | sed -E 's/[^0-9]*([0-9]+).*/\1/')"; \ + url="$$(curl -fsSL https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json \ + | jq -r --arg m "$$major." '[.versions[] | select(.version | startswith($$m))] | last \ + | .downloads.chromedriver[] | select(.platform == "linux64") | .url')"; \ + if [ -z "$$url" ] || [ "$$url" = "null" ]; then \ + printf "$(RED) ✗ No chromedriver published for Chrome $$major$(NC)\n"; exit 1; \ + fi; \ + tmp="$$(mktemp -d)"; \ + curl -fsSL -o "$$tmp/chromedriver.zip" "$$url"; \ + unzip -oq "$$tmp/chromedriver.zip" -d "$$tmp"; \ + sudo install -m 0755 "$$tmp/chromedriver-linux64/chromedriver" /usr/local/bin/chromedriver; \ + rm -rf "$$tmp" + @printf "$(GREEN)✓ $$(google-chrome --version) / $$(chromedriver --version | cut -d' ' -f1-2)$(NC)\n" + all: build test examples @printf "$(GREEN)Build and test completed!$(NC)\n" From 8e73a325f98a4781c81d04f1375a2d464cf2c492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Tue, 28 Jul 2026 19:30:27 +0000 Subject: [PATCH 44/45] feat: add Docker features to devcontainer configuration for improved build support --- .devcontainer/devcontainer.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b1e3a64e..3a360f1e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,6 +2,13 @@ "name": "Rust Dev-Container", "image": "ghcr.io/aimdb-dev/devcontainer:latest", "workspaceFolder": "/aimdb", + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { + "moby": true, + "installDockerBuildx": true, + "dockerDashComposeVersion": "v2" + } + }, "runArgs": [ "--privileged", "--network=host", @@ -38,4 +45,4 @@ "containerEnv": { "SHELL": "/bin/bash" } -} \ No newline at end of file +} From 81696c7659f44cbc6c988a112de8eaba76b9a8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Schn=C3=B6rch?= Date: Wed, 29 Jul 2026 21:51:38 +0000 Subject: [PATCH 45/45] chore: update subproject commit reference for embassy --- Cargo.lock | 189 +++++++++++++++++++++++----------------------- _external/embassy | 2 +- 2 files changed, 94 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2268bec7..7e1358a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,7 +68,7 @@ dependencies = [ "aimdb-tokio-adapter", "criterion", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "embassy-time-driver", "futures", "portable-atomic", @@ -138,13 +138,13 @@ dependencies = [ "aimdb-derive", "anyhow", "async-channel", - "defmt 1.0.1", + "defmt 1.1.1", "futures", "futures-channel", "futures-core", "futures-util", "hashbrown 0.15.5", - "heapless 0.9.1", + "heapless 0.9.3", "portable-atomic", "serde", "serde_json", @@ -183,7 +183,7 @@ version = "0.6.0" dependencies = [ "aimdb-core", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "embassy-executor", "embassy-net", "embassy-sync", @@ -192,7 +192,7 @@ dependencies = [ "embedded-io-async 0.7.0", "futures", "futures-core", - "heapless 0.9.1", + "heapless 0.9.3", "rand 0.10.1", "tracing", "tracing-test", @@ -206,7 +206,7 @@ dependencies = [ "aimdb-embassy-adapter", "aimdb-tokio-adapter", "async-stream", - "defmt 1.0.1", + "defmt 1.1.1", "embassy-executor", "embassy-futures", "embassy-net", @@ -254,7 +254,7 @@ dependencies = [ "aimdb-embassy-adapter", "aimdb-tokio-adapter", "async-stream", - "defmt 1.0.1", + "defmt 1.1.1", "embassy-executor", "embassy-net", "embassy-sync", @@ -309,7 +309,7 @@ dependencies = [ "aimdb-embassy-adapter", "aimdb-tokio-adapter", "cobs 0.5.1", - "defmt 1.0.1", + "defmt 1.1.1", "embassy-time-driver", "embedded-io-async 0.7.0", "futures", @@ -343,14 +343,14 @@ dependencies = [ "aimdb-embassy-adapter", "aimdb-tokio-adapter", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "embassy-futures", "embassy-net", "embassy-net-driver-channel", "embassy-time-driver", "embedded-io-async 0.7.0", "futures", - "heapless 0.9.1", + "heapless 0.9.3", "serde", "serde_json", "tokio", @@ -665,9 +665,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "block-buffer" @@ -855,7 +855,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd93fd2c1b27acd030440c9dbd9d14c1122aad622374fe05a670b67a4bc034be" dependencies = [ - "heapless 0.9.1", + "heapless 0.9.3", "thiserror 2.0.17", ] @@ -1132,14 +1132,14 @@ version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0963443817029b2024136fc4dd07a5107eb8f977eaf18fcd1fdeb11306b64ad" dependencies = [ - "defmt 1.0.1", + "defmt 1.1.1", ] [[package]] name = "defmt" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "548d977b6da32fa1d1fda2876453da1e7df63ad0304c8b3dae4dbe7b96f39b78" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -1147,12 +1147,11 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d4fc12a85bcf441cfe44344c4b72d58493178ce635338a3f3b78943aceb258e" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" dependencies = [ "defmt-parser", - "proc-macro-error2", "proc-macro2", "quote", "syn 2.0.108", @@ -1174,7 +1173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93d5a25c99d89c40f5676bec8cefe0614f17f0f40e916f98e345dae941807f9e" dependencies = [ "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", ] [[package]] @@ -1195,7 +1194,7 @@ checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" dependencies = [ "const-oid 0.10.2", "der_derive", - "heapless 0.9.1", + "heapless 0.9.3", "time", ] @@ -1250,12 +1249,11 @@ dependencies = [ [[package]] name = "dsp-fixedpoint" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e6691b4edb40c0a1bd948fa524482e3f36f690d782fcabdc39b5d52f6dcf8" +checksum = "731ba87a0802a07eec1ae26620c90990126bc6b72b4abfae51c2b1d84fafcaf8" dependencies = [ "num-traits", - "serde", ] [[package]] @@ -1306,7 +1304,7 @@ dependencies = [ "cortex-m", "cortex-m-rt", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "defmt-rtt", "embassy-executor", "embassy-futures", @@ -1321,7 +1319,7 @@ dependencies = [ name = "embassy-embedded-hal" version = "0.6.0" dependencies = [ - "defmt 1.0.1", + "defmt 1.1.1", "embassy-futures", "embassy-hal-internal", "embassy-sync", @@ -1341,7 +1339,7 @@ dependencies = [ "cordyceps", "cortex-m", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "document-features", "embassy-executor-macros", "embassy-executor-timer-queue", @@ -1371,7 +1369,7 @@ version = "0.5.0" dependencies = [ "cortex-m", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "num-traits", ] @@ -1386,7 +1384,7 @@ dependencies = [ "cortex-m", "cortex-m-rt", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "defmt-rtt", "embassy-executor", "embassy-futures", @@ -1419,7 +1417,7 @@ dependencies = [ "cortex-m", "cortex-m-rt", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "defmt-rtt", "embassy-executor", "embassy-futures", @@ -1445,7 +1443,7 @@ dependencies = [ name = "embassy-net" version = "0.9.1" dependencies = [ - "defmt 1.0.1", + "defmt 1.1.1", "document-features", "embassy-futures", "embassy-net-driver", @@ -1453,7 +1451,7 @@ dependencies = [ "embassy-time", "embedded-io-async 0.7.0", "embedded-nal-async", - "heapless 0.9.1", + "heapless 0.9.3", "managed", "smoltcp", ] @@ -1462,7 +1460,7 @@ dependencies = [ name = "embassy-net-driver" version = "0.2.0" dependencies = [ - "defmt 1.0.1", + "defmt 1.1.1", ] [[package]] @@ -1484,7 +1482,7 @@ dependencies = [ "cortex-m", "cortex-m-rt", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "defmt-rtt", "embassy-executor", "embassy-futures", @@ -1504,13 +1502,13 @@ version = "0.6.0" dependencies = [ "aligned", "bit_field", - "bitflags 2.11.0", + "bitflags 2.13.1", "block-device-driver", "cfg-if", "cortex-m", "cortex-m-rt", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "document-features", "dsp-fixedpoint", "embassy-embedded-hal", @@ -1533,7 +1531,7 @@ dependencies = [ "embedded-storage", "embedded-storage-async", "futures-util", - "heapless 0.9.1", + "heapless 0.9.3", "nb 1.1.0", "once_cell", "proc-macro2", @@ -1558,11 +1556,11 @@ version = "0.8.0" dependencies = [ "cfg-if", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "embedded-io-async 0.7.0", "futures-core", "futures-sink", - "heapless 0.9.1", + "heapless 0.9.3", ] [[package]] @@ -1571,7 +1569,7 @@ version = "0.5.1" dependencies = [ "cfg-if", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "document-features", "embassy-time-driver", "embassy-time-queue-utils", @@ -1593,14 +1591,14 @@ name = "embassy-time-queue-utils" version = "0.3.2" dependencies = [ "embassy-executor-timer-queue", - "heapless 0.9.1", + "heapless 0.9.3", ] [[package]] name = "embassy-usb-driver" version = "0.2.2" dependencies = [ - "defmt 1.0.1", + "defmt 1.1.1", "embedded-io-async 0.6.1", "embedded-io-async 0.7.0", ] @@ -1610,7 +1608,7 @@ name = "embassy-usb-synopsys-otg" version = "0.4.0" dependencies = [ "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "embassy-sync", "embassy-time", "embassy-usb-driver", @@ -1691,7 +1689,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eb1aa714776b75c7e67e1da744b81a129b3ff919c8712b5e1b32252c1f07cc7" dependencies = [ - "defmt 1.0.1", + "defmt 1.1.1", ] [[package]] @@ -1709,7 +1707,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2564b9f813c544241430e147d8bc454815ef9ac998878d30cc3055449f7fd4c0" dependencies = [ - "defmt 1.0.1", + "defmt 1.1.1", "embedded-io 0.7.1", ] @@ -1761,7 +1759,7 @@ dependencies = [ "embedded-io 0.7.1", "embedded-io-async 0.7.0", "generic-array", - "heapless 0.9.1", + "heapless 0.9.3", "hkdf", "hmac", "p256", @@ -1933,9 +1931,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -1943,9 +1941,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" @@ -1960,15 +1958,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", @@ -1977,21 +1975,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -2001,7 +1999,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -2178,11 +2175,11 @@ dependencies = [ [[package]] name = "heapless" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1edcd5a338e64688fbdcb7531a846cfd3476a54784dcb918a0844682bc7ada5" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" dependencies = [ - "defmt 1.0.1", + "defmt 1.1.1", "hash32", "stable_deref_trait", ] @@ -2629,7 +2626,7 @@ version = "0.1.0" dependencies = [ "aimdb-core", "aimdb-knx-connector", - "defmt 1.0.1", + "defmt 1.1.1", "serde", ] @@ -2637,8 +2634,8 @@ dependencies = [ name = "knx-pico" version = "0.3.0" dependencies = [ - "defmt 1.0.1", - "heapless 0.9.1", + "defmt 1.1.1", + "heapless 0.9.3", ] [[package]] @@ -2829,7 +2826,7 @@ dependencies = [ name = "mountain-mqtt-embassy" version = "0.2.0" dependencies = [ - "defmt 1.0.1", + "defmt 1.1.1", "embassy-net", "embassy-sync", "embassy-time", @@ -2842,7 +2839,7 @@ version = "0.1.0" dependencies = [ "aimdb-core", "aimdb-data-contracts", - "defmt 1.0.1", + "defmt 1.1.1", "serde", "serde_json", ] @@ -2896,7 +2893,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -2996,7 +2993,7 @@ version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", @@ -3064,7 +3061,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd402d00b0fb94c5aee000029204a46884b1262e0c443f166d86d2c0747e1a1a" dependencies = [ "cortex-m", - "defmt 1.0.1", + "defmt 1.1.1", ] [[package]] @@ -3246,18 +3243,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.41" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -3361,14 +3358,14 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -3378,9 +3375,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -3389,9 +3386,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "remote-access-demo" @@ -3532,7 +3529,7 @@ version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -3555,7 +3552,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -3645,7 +3642,7 @@ checksum = "442d4946fee518534b4fe4f3b1f37ca440af1d08300dd7bda89a6bfc99bd05b3" dependencies = [ "aligned", "block-device-driver", - "defmt 1.0.1", + "defmt 1.1.1", "embedded-hal 1.0.0", "embedded-hal-async", ] @@ -3675,7 +3672,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -3827,7 +3824,7 @@ version = "4.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4d91116f97173694f1642263b2ff837f80d933aa837e2314969f6728f661df3" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", "cfg-if", "core-foundation 0.10.1", "core-foundation-sys", @@ -3917,7 +3914,7 @@ dependencies = [ "byteorder", "cfg-if", "defmt 0.3.100", - "heapless 0.9.1", + "heapless 0.9.3", "managed", ] @@ -3992,11 +3989,11 @@ dependencies = [ [[package]] name = "stm32-metapac" version = "21.0.0" -source = "git+https://github.com/embassy-rs/stm32-data-generated?tag=stm32-data-1912ac7d73c27f03863fb2d71fbfddf0d9fce062#27bdd2e4982cbc04ba4f4ad9ee7c8824a797944b" +source = "git+https://github.com/embassy-rs/stm32-data-generated?tag=stm32-data-75cfa09e93a6c7275d14ecb5c8b5e95b2ab2dae2#d63f1f3f537b934c3b0c5b9f1ddb155c10c1ffc2" dependencies = [ "cortex-m", "cortex-m-rt", - "defmt 1.0.1", + "defmt 1.1.1", ] [[package]] @@ -4085,7 +4082,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4505,7 +4502,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -4972,7 +4969,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.1", "hashbrown 0.15.5", "indexmap", "semver 1.0.28", @@ -5047,7 +5044,7 @@ dependencies = [ "cortex-m", "cortex-m-rt", "critical-section", - "defmt 1.0.1", + "defmt 1.1.1", "defmt-rtt", "embassy-executor", "embassy-futures", @@ -5491,7 +5488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.0", + "bitflags 2.13.1", "indexmap", "log", "serde", diff --git a/_external/embassy b/_external/embassy index 44200294..03c826f2 160000 --- a/_external/embassy +++ b/_external/embassy @@ -1 +1 @@ -Subproject commit 4420029493483dfb7be20e1f9dccbd75c954ec4a +Subproject commit 03c826f2544220cb44b2dec05f5d636a80d39382