diff --git a/docs/assets/figures/mage-007/kernel-lane-fixes-mobile.svg b/docs/assets/figures/mage-007/kernel-lane-fixes-mobile.svg new file mode 100644 index 0000000..b1538e4 --- /dev/null +++ b/docs/assets/figures/mage-007/kernel-lane-fixes-mobile.svg @@ -0,0 +1,3059 @@ + + + + + + + + RTX 4090; Nsight Systems kernel time, mean over 100 launches per measurement. 10 September 2026, RTX 4090 (sm_89), driver 591.74, WSL2, device idle. Each before/after pair was measured in one session with the same warp-up and the same inputs; Triton and cuTile references are quoted from their own sessions and are marked as such. Hatched bars are references from other sessions. See docs/assets/results/mage-007/kernel-lane-fixes.json for the values. + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/figures/mage-007/kernel-lane-fixes.png b/docs/assets/figures/mage-007/kernel-lane-fixes.png new file mode 100644 index 0000000..e3db5ad Binary files /dev/null and b/docs/assets/figures/mage-007/kernel-lane-fixes.png differ diff --git a/docs/assets/figures/mage-007/kernel-lane-fixes.svg b/docs/assets/figures/mage-007/kernel-lane-fixes.svg new file mode 100644 index 0000000..3a22c4b --- /dev/null +++ b/docs/assets/figures/mage-007/kernel-lane-fixes.svg @@ -0,0 +1,3141 @@ + + + + + + + + RTX 4090; Nsight Systems kernel time, mean over 100 launches per measurement. 10 September 2026, RTX 4090 (sm_89), driver 591.74, WSL2, device idle. Each before/after pair was measured in one session with the same warp-up and the same inputs; Triton and cuTile references are quoted from their own sessions and are marked as such. Hatched bars are references from other sessions. See docs/assets/results/mage-007/kernel-lane-fixes.json for the values. + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/results/mage-007/kernel-lane-fixes.json b/docs/assets/results/mage-007/kernel-lane-fixes.json new file mode 100644 index 0000000..2bab2ff --- /dev/null +++ b/docs/assets/results/mage-007/kernel-lane-fixes.json @@ -0,0 +1,45 @@ +{ + "experiment": "mage-007", + "title": "What three kernel changes measured", + "instrument": "Nsight Systems kernel time, mean over 100 launches per measurement", + "session": "10 September 2026, RTX 4090 (sm_89), driver 591.74, WSL2, device idle. Each before/after pair was measured in one session with the same warp-up and the same inputs; Triton and cuTile references are quoted from their own sessions and are marked as such.", + "changes": [ + { + "operation": "layernorm", + "label": "LayerNorm 4096x4096", + "issue": 54, + "pr": 69, + "note": "The two-warp row kernel was capped at width 2048, so 4096 fell back to the single-warp kernel. Its own soundness condition already held at 4096. Worst full-output error 4.77e-07.", + "bars": [ + { "name": "single-warp (before)", "kind": "before", "us": 255.41 }, + { "name": "two-warp (after)", "kind": "after", "us": 186.80 }, + { "name": "Triton", "kind": "reference", "us": 181.59, "source": "mage-006, different session" } + ] + }, + { + "operation": "neighbor", + "label": "Neighbor aggregation 4096x64x65536", + "issue": 59, + "pr": 71, + "note": "One 128-bit feature quad per thread with the edge walk unrolled by two, so two gathers overlap. Worst full-output error 3.58e-07.", + "bars": [ + { "name": "scalar (before)", "kind": "before", "us": 9.45 }, + { "name": "quad (after)", "kind": "after", "us": 7.06 }, + { "name": "Triton", "kind": "reference", "us": 5.97, "source": "mage-006, different session" } + ] + }, + { + "operation": "gelu", + "label": "Bias + GELU 4096x768", + "issue": 59, + "pr": 71, + "note": "Four features per thread was tried on this kernel and rejected: 2.6x slower than the scalar form, so the scalar kernel stays. Recorded so the attempt is not repeated blind.", + "bars": [ + { "name": "scalar (kept)", "kind": "after", "us": 10.29 }, + { "name": "quad (rejected)", "kind": "rejected", "us": 27.00 }, + { "name": "Triton", "kind": "reference", "us": 7.76, "source": "mage-004, different session" }, + { "name": "cuTile Rust", "kind": "reference", "us": 7.97, "source": "mage-004, different session" } + ] + } + ] +} diff --git a/docs/experiments/mage-007.md b/docs/experiments/mage-007.md new file mode 100644 index 0000000..e57294c --- /dev/null +++ b/docs/experiments/mage-007.md @@ -0,0 +1,80 @@ +# mage-007: what three kernel changes measured + +Status: **two improvements and one rejection**, measured 10 September 2026. This +record belongs to the cuda-oxide kernel lane, which the cuTile track picked up when +its agent exited; mage-006 stays as that agent left it. + +The lane's ranked gaps were wide-row LayerNorm, then GELU and neighbor aggregation. +This round took all three: one closed, one halved, one rejected with its numbers +kept. + +## Method + +| | | +| --- | --- | +| Instrument | GPU kernel time, one Nsight Systems capture per measurement, mean of 100 launches after 25 warm-up launches | +| Host | RTX 4090 (sm_89), driver 591.74, WSL2 Ubuntu 22.04, device idle | +| Session discipline | Each before/after pair was measured in one session, with the same inputs and the same warm-up, on the same binary path; the device was checked idle before each pass | +| Correctness | Full-output check against PyTorch FP32 with TF32 disabled, `rtol = atol = 1e-4`, before any timing was believed | +| References | Triton and cuTile bars are quoted from their own sessions (mage-004, mage-006) and are drawn hatched in the figure because they are not pairings | + +The figure is [`docs/assets/figures/mage-007/kernel-lane-fixes.svg`](../assets/figures/mage-007/kernel-lane-fixes.svg), +drawn by `scripts/plot-kernel-lane-fixes.py` from +[`docs/assets/results/mage-007/kernel-lane-fixes.json`](../assets/results/mage-007/kernel-lane-fixes.json). + +## Values + +Kernel time in microseconds, mean of 100 launches. + +| Operation | Before | After | Change | Reference (other session) | +| --- | ---: | ---: | ---: | --- | +| LayerNorm 4096×4096 | 255.41 (`layer_norm_warp`) | **186.80** (`layer_norm_pair`) | 1.37× faster | Triton 181.59 (mage-006) | +| Neighbor aggregation 4096×64×65536 | 9.45 (scalar) | **7.06** (feature quad) | 1.34× faster | Triton 5.97, PyTorch 120.93 (mage-006) | +| Bias + GELU 4096×768 | 10.29 (scalar, kept) | 27.00 (feature quad, **rejected**) | 2.6× slower | Triton 7.76, cuTile 7.97 (mage-004) | + +Worst full-output error: 4.77e-07 (LayerNorm), 3.58e-07 (neighbor), 5.31e-06 (GELU). + +## What each change was + +**LayerNorm at wide rows** (issue #54, PR #69). The two-warp row kernel was gated at +`width <= 2048`, so a 4096-wide row fell back to `layer_norm_warp` — the capture named +the kernel, which is how the fallback was confirmed rather than assumed. The pairwise +kernel's own soundness condition, each warp's span a multiple of 32 lanes, already +held at 4096: the cap was a stale heuristic. Raising it to 4096 is the whole change. + +**Neighbor aggregation** (issue #59, PR #71). The scalar kernel gives each thread one +feature and a serial edge loop, so one gather is in flight per thread. The quad form +gives each thread four features — one 128-bit load per edge — and unrolls the edge +walk by two, so two independent gathers overlap. Triton's 5.97 µs remains ahead, so +the gap narrows from 1.7× to 1.2×; the kernel moves about 17 MB of gathered rows in +7 µs, which is L2-bandwidth work on this part, so more edges in flight per thread is +the next lever. + +**Bias + GELU** (issue #59, PR #71). The same vectorization was tried and rejected: +four features per thread measured 27.00 µs against the scalar kernel's 10.29 µs in the +same session. The LayerNorm treatment does not transfer to this elementwise shape — +one element per thread wins, most likely on occupancy. Kept here because it is the +counterexample: "vectorize the elementwise kernel" is not a rule. + +## What the numbers do not establish + +- No hardware counters are available, so the L2-bandwidth reading of the neighbor + kernel is inferred from traffic arithmetic, not measured. +- One capture per point: a kernel time here carries no interval of its own. +- The references are quoted from other sessions. A consolidated pass + (`scripts/evolve_capture.py --all`) is built for same-session comparisons and would + make these four arms one measurement whenever the lane wants it. +- The LayerNorm change is validated at 4096×4096, 4096×512, 3072×1024 and 2048×2048; + widths above 4096 still fall back, and were not measured. +- GELU's rejection is a single before/after pair, not a sweep: other vector widths + (two features, or quads with a different block size) were not tried. + +## Reproduction + +```bash +source scripts/oxide-env.sh +cd examples/oxide && CARGO_BUILD_JOBS=4 cargo oxide build --arch sm_89 && cd ../.. +# one operation, one capture +.venv/bin/python -m mage profile-exec ... # see docs/profiling.md for the native workflow +uv run --script scripts/plot-kernel-lane-fixes.py +``` diff --git a/scripts/plot-kernel-lane-fixes.py b/scripts/plot-kernel-lane-fixes.py new file mode 100644 index 0000000..77f73e5 --- /dev/null +++ b/scripts/plot-kernel-lane-fixes.py @@ -0,0 +1,111 @@ +"""Draw what each kernel-lane change measured: before, after, and the references. + +Reads docs/assets/results/mage-007/kernel-lane-fixes.json and writes the figure to +docs/assets/figures/mage-007/. The pair in each row was measured in one session; +the Triton and cuTile bars are references from their own sessions and are drawn +differently so the two kinds cannot be confused. + +Run: uv run --script scripts/plot-kernel-lane-fixes.py +""" +import json +import math +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.ticker import MaxNLocator # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +EXPERIMENT = "mage-007" +SOURCE = ROOT / "docs/assets/results" / EXPERIMENT / "kernel-lane-fixes.json" +OUT = ROOT / "docs/assets/figures" / EXPERIMENT + +BG, INK, MUTED, RULE = "#101217", "#edf0f5", "#a0a9b9", "#303641" +KIND_COLORS = {"before": "#8b93a7", "after": "#91dbba", "rejected": "#e08a8a", "reference": "#3d4759"} +KIND_EDGE = {"before": "#a0a9b9", "after": "#91dbba", "rejected": "#e08a8a", "reference": "#93caff"} + + +def plot(data, mobile): + changes = data["changes"] + width, height = (3.65, 6.6) if mobile else (7.4, 6.2) + fig, axes = plt.subplots(len(changes), 1, figsize=(width, height)) + if len(changes) == 1: + axes = [axes] + fig.subplots_adjust(top=.845 if not mobile else .86, bottom=.135 if not mobile else .16, + left=.30 if not mobile else .40, right=.985, hspace=.72) + + fig.text(.01, .978, "What three kernel changes measured", color=INK, + fontsize=13 if mobile else 15, weight="bold", va="top") + fig.text(.01, .945, "GPU kernel time, before and after; references from other sessions", + color=MUTED, fontsize=8.4 if mobile else 10, va="top") + fig.text(.01, .012, "RTX 4090 · FP32 · each row has its own scale · pairs measured in one session", + color=MUTED, fontsize=7.4 if mobile else 9, va="bottom") + + for ax, change in zip(axes, changes): + bars = change["bars"] + values = [b["us"] for b in bars] + extent = max(values) + ax.set_xlim(0, extent * (1.34 if mobile else 1.22)) + ax.set_ylim(len(bars) - .45, -.6) + for y, bar in enumerate(bars): + color = KIND_COLORS[bar["kind"]] + ax.barh(y, bar["us"], height=.54, color=color, zorder=3, + edgecolor=KIND_EDGE[bar["kind"]], linewidth=1.1 if bar["kind"] == "reference" else 0) + if bar["kind"] == "reference": + ax.barh(y, bar["us"], height=.54, fill=False, hatch="////", + edgecolor=KIND_EDGE["reference"], linewidth=0, zorder=4) + label = f"{bar['us']:.2f}" if bar["us"] < 10 else f"{bar['us']:.1f}" + ax.annotate(label, (bar["us"], y), xytext=(5, 0), textcoords="offset points", + ha="left", va="center", color=INK, fontsize=8.4 if mobile else 10) + names = [f"{b['name']}\n(before)" if b["kind"] == "before" else b["name"] for b in bars] + ax.set_yticks(range(len(bars)), names, color=MUTED, fontsize=7.6 if mobile else 9.4) + for tick, bar in zip(ax.get_yticklabels(), bars): + tick.set_color(KIND_EDGE[bar["kind"]] if bar["kind"] in ("after", "rejected") else MUTED) + ax.set_title(change["label"], loc="left", color=INK, fontsize=9.8 if mobile else 11.5, + pad=14, weight="medium") + caption = change["note"] + if change.get("kind_note"): + caption = f"{change['kind_note']} — {caption}" + ax.text(0, 1.02, caption, transform=ax.transAxes, color=MUTED, + fontsize=7 if mobile else 8.4, va="bottom") + ax.xaxis.set_major_locator(MaxNLocator(nbins=3 if mobile else 5)) + ax.tick_params(axis="both", length=0, colors=MUTED, labelsize=7.4 if mobile else 8.6) + ax.grid(axis="x", color=RULE, linewidth=.6, zorder=0) + for spine in ax.spines.values(): + spine.set_visible(False) + ax.set_facecolor(BG) + + fig.patch.set_facecolor(BG) + name = f"kernel-lane-fixes{'-mobile' if mobile else ''}" + metadata = {"Date": None, "Description": + f"RTX 4090; {data['instrument']}. {data['session']} " + "Hatched bars are references from other sessions. " + "See docs/assets/results/mage-007/kernel-lane-fixes.json for the values."} + fig.savefig(OUT / f"{name}.svg", metadata=metadata) + svg = OUT / f"{name}.svg" + svg.write_bytes(b"\n".join(line.rstrip() for line in svg.read_bytes().splitlines()) + b"\n") + if not mobile: + fig.savefig(OUT / f"{name}.png", dpi=200, metadata=metadata) + plt.close(fig) + + +def main(): + data = json.loads(SOURCE.read_text()) + for change in data["changes"]: + change["kind_note"] = "" + if any(b["kind"] == "rejected" for b in change["bars"]): + change["kind_note"] = "an attempt that measured worse and was reverted" + elif all(b["us"] for b in change["bars"]) and not math.isnan(change["bars"][0]["us"]) \ + and change["bars"][1]["us"] < change["bars"][0]["us"]: + ratio = change["bars"][0]["us"] / change["bars"][1]["us"] + change["kind_note"] = f"{ratio:.2f}x faster" + OUT.mkdir(parents=True, exist_ok=True) + plot(data, mobile=False) + plot(data, mobile=True) + print("wrote", OUT / "kernel-lane-fixes.svg") + + +if __name__ == "__main__": + main()