From 44e907d31c68e61a2244deb5c0c65c7eeeb2730a Mon Sep 17 00:00:00 2001 From: J-P Nurmi Date: Thu, 10 Sep 2026 14:53:56 +0200 Subject: [PATCH 1/3] test(benchmark): add metrics and improve logs benchmark Add benchmark_metrics for multi-threaded metrics throughput. The logs benchmark previously attempted 32x100 logs at peak concurrency, exceeding far the benchmark's configured 10x100 buffer capacity. Overflows caused quick log discards to dominate, making the results look artificially good. Reduce iterations per thread to 32, so the peak of 1,024 logs only slightly exceeds the 1000-log buffer capacity. Track enqueue failures and count only successful enqueues as processed items to make drops visible in the results. Use real time to measure elapsed time across concurrent threads: https://google.github.io/benchmark/user_guide.html#multithreaded-benchmarks --- tests/benchmark.py | 12 +++++ tests/benchmark/CMakeLists.txt | 1 + tests/benchmark/benchmark_logs.cpp | 17 +++++-- tests/benchmark/benchmark_metrics.cpp | 65 +++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 tests/benchmark/benchmark_metrics.cpp diff --git a/tests/benchmark.py b/tests/benchmark.py index 6132198615..fa384b8b9d 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -84,6 +84,18 @@ def test_benchmark_logs(threads, cmake, httpserver, gbenchmark): ) +@pytest.mark.parametrize("threads", [1, 8, 16, 32]) +def test_benchmark_metrics(threads, cmake, httpserver, gbenchmark): + run_benchmark( + f"^benchmark_metrics.*threads:{threads}$", + "none", + cmake, + httpserver, + gbenchmark, + f"Metrics ({threads} thread{'s' if threads > 1 else ''})", + ) + + @pytest.mark.parametrize("backend", ["inproc", "breakpad", "crashpad", "native"]) def test_benchmark_libsize(backend, cmake, gmeasurement): tmp_path = cmake( diff --git a/tests/benchmark/CMakeLists.txt b/tests/benchmark/CMakeLists.txt index aef7cc99f9..0b8df26230 100644 --- a/tests/benchmark/CMakeLists.txt +++ b/tests/benchmark/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(sentry_benchmark benchmark_init.cpp benchmark_backend.cpp benchmark_logs.cpp + benchmark_metrics.cpp benchmark_scope.cpp ) diff --git a/tests/benchmark/benchmark_logs.cpp b/tests/benchmark/benchmark_logs.cpp index 47e5a60db6..f659667478 100644 --- a/tests/benchmark/benchmark_logs.cpp +++ b/tests/benchmark/benchmark_logs.cpp @@ -5,9 +5,8 @@ extern "C" { } static void -discard_envelope(sentry_envelope_t *envelope, void *state) +discard_envelope(sentry_envelope_t *envelope, void *) { - (void)state; sentry_envelope_free(envelope); } @@ -16,6 +15,8 @@ setup_logs(const benchmark::State &) { sentry_options_t *options = sentry_options_new(); sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_release(options, "benchmark@1.0"); + sentry_options_set_environment(options, "test"); sentry_options_set_logs_with_attributes(options, true); sentry_options_set_auto_session_tracking(options, 0); sentry_options_set_transport( @@ -40,9 +41,16 @@ benchmark_logs(benchmark::State &state) sentry_value_new_attribute(sentry_value_new_string("attribute"), NULL)); int i = 0; + int failed = 0; for (auto _ : state) { - sentry_log_info("log %d", sentry_value_incref(attributes), i++, NULL); + if (sentry_log_info( + "log %d", sentry_value_incref(attributes), i++, NULL) + != SENTRY_LOG_RETURN_SUCCESS) { + failed++; + } } + state.SetItemsProcessed(state.iterations() - failed); + state.counters["enqueue_failures"] = failed; sentry_value_decref(attributes); } @@ -52,7 +60,8 @@ BENCHMARK(benchmark_logs) ->Threads(8) ->Threads(16) ->Threads(32) - ->Iterations(100) + ->Iterations(32) // 32x32=1024 (peak) > 10x100=1000 (capacity) + ->UseRealTime() ->Unit(benchmark::kMillisecond) ->Setup(setup_logs) ->Teardown(teardown_logs); diff --git a/tests/benchmark/benchmark_metrics.cpp b/tests/benchmark/benchmark_metrics.cpp new file mode 100644 index 0000000000..7e8d2741fd --- /dev/null +++ b/tests/benchmark/benchmark_metrics.cpp @@ -0,0 +1,65 @@ +#include + +extern "C" { +#include "sentry.h" +} + +static void +discard_envelope(sentry_envelope_t *envelope, void *) +{ + sentry_envelope_free(envelope); +} + +static void +setup_metrics(const benchmark::State &) +{ + sentry_options_t *options = sentry_options_new(); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_release(options, "benchmark@1.0"); + sentry_options_set_environment(options, "test"); + sentry_options_set_auto_session_tracking(options, 0); + sentry_options_set_transport( + options, sentry_transport_new(discard_envelope)); + sentry_init(options); + + sentry_set_attribute("global", + sentry_value_new_attribute(sentry_value_new_string("attribute"), NULL)); +} + +static void +teardown_metrics(const benchmark::State &) +{ + sentry_close(); +} + +static void +benchmark_metrics(benchmark::State &state) +{ + sentry_value_t attributes = sentry_value_new_object(); + sentry_value_set_by_key(attributes, "asset.type", + sentry_value_new_attribute(sentry_value_new_string("texture"), NULL)); + + int failed = 0; + for (auto _ : state) { + if (sentry_metrics_distribution("asset.load.duration", 12.5, + SENTRY_UNIT_MILLISECOND, sentry_value_incref(attributes)) + != SENTRY_METRICS_RESULT_SUCCESS) { + failed++; + } + } + state.SetItemsProcessed(state.iterations() - failed); + state.counters["enqueue_failures"] = failed; + + sentry_value_decref(attributes); +} + +BENCHMARK(benchmark_metrics) + ->Threads(1) + ->Threads(8) + ->Threads(16) + ->Threads(32) + ->Iterations(32) // 32x32=1024 (peak) > 10x100=1000 (capacity) + ->UseRealTime() + ->Unit(benchmark::kMillisecond) + ->Setup(setup_metrics) + ->Teardown(teardown_metrics); From 9c63f7c251643a8dec078c14598522ffa10a886e Mon Sep 17 00:00:00 2001 From: J-P Nurmi Date: Thu, 10 Sep 2026 15:41:13 +0200 Subject: [PATCH 2/3] test(benchmark): Split scope benchmarks into tags and breadcrumbs --- tests/benchmark.py | 19 +++++++++--- tests/benchmark/CMakeLists.txt | 3 +- tests/benchmark/benchmark_breadcrumbs.cpp | 29 +++++++++++++++++++ ...benchmark_scope.cpp => benchmark_tags.cpp} | 26 ++--------------- 4 files changed, 48 insertions(+), 29 deletions(-) create mode 100644 tests/benchmark/benchmark_breadcrumbs.cpp rename tests/benchmark/{benchmark_scope.cpp => benchmark_tags.cpp} (54%) diff --git a/tests/benchmark.py b/tests/benchmark.py index fa384b8b9d..ecb7792599 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -59,16 +59,27 @@ def test_benchmark_backend(backend, cmake, httpserver, gbenchmark): ) -@pytest.mark.parametrize("test_name", ["set_tag", "add_breadcrumb"]) @pytest.mark.parametrize("backend", ["inproc", "breakpad", "crashpad", "native"]) -def test_benchmark_scope(test_name, backend, cmake, httpserver, gbenchmark): +def test_benchmark_tags(backend, cmake, httpserver, gbenchmark): run_benchmark( - f"benchmark_scope_{test_name}", + "benchmark_tags", backend, cmake, httpserver, gbenchmark, - f"Scope {test_name} ({backend})", + f"Tags ({backend})", + ) + + +@pytest.mark.parametrize("backend", ["inproc", "breakpad", "crashpad", "native"]) +def test_benchmark_breadcrumbs(backend, cmake, httpserver, gbenchmark): + run_benchmark( + "benchmark_breadcrumbs", + backend, + cmake, + httpserver, + gbenchmark, + f"Breadcrumbs ({backend})", ) diff --git a/tests/benchmark/CMakeLists.txt b/tests/benchmark/CMakeLists.txt index 0b8df26230..5e37ea4db9 100644 --- a/tests/benchmark/CMakeLists.txt +++ b/tests/benchmark/CMakeLists.txt @@ -11,9 +11,10 @@ add_executable(sentry_benchmark ${SENTRY_SOURCES} benchmark_init.cpp benchmark_backend.cpp + benchmark_breadcrumbs.cpp benchmark_logs.cpp benchmark_metrics.cpp - benchmark_scope.cpp + benchmark_tags.cpp ) if(SENTRY_BACKEND_CRASHPAD) diff --git a/tests/benchmark/benchmark_breadcrumbs.cpp b/tests/benchmark/benchmark_breadcrumbs.cpp new file mode 100644 index 0000000000..b60be416a5 --- /dev/null +++ b/tests/benchmark/benchmark_breadcrumbs.cpp @@ -0,0 +1,29 @@ +#include + +extern "C" { +#include "sentry_core.h" +#include "sentry_options.h" +#include "sentry_scope.h" +} + +static void +benchmark_breadcrumbs(benchmark::State &state) +{ + sentry_options_t *options = sentry_options_new(); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_init(options); + + int i = 0; + for (auto _ : state) { + char msg[32]; + snprintf(msg, sizeof(msg), "message%d", i); + sentry_add_breadcrumb(sentry_value_new_breadcrumb(NULL, msg)); + i++; + } + + sentry_close(); +} + +BENCHMARK(benchmark_breadcrumbs) + ->Iterations(1000) + ->Unit(benchmark::kMillisecond); diff --git a/tests/benchmark/benchmark_scope.cpp b/tests/benchmark/benchmark_tags.cpp similarity index 54% rename from tests/benchmark/benchmark_scope.cpp rename to tests/benchmark/benchmark_tags.cpp index 212f91bfa0..dee79a3c70 100644 --- a/tests/benchmark/benchmark_scope.cpp +++ b/tests/benchmark/benchmark_tags.cpp @@ -7,7 +7,7 @@ extern "C" { } static void -benchmark_scope_set_tag(benchmark::State &state) +benchmark_tags(benchmark::State &state) { sentry_options_t *options = sentry_options_new(); sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); @@ -28,28 +28,6 @@ benchmark_scope_set_tag(benchmark::State &state) sentry_close(); } -BENCHMARK(benchmark_scope_set_tag) - ->Iterations(1000) - ->Unit(benchmark::kMillisecond); - -static void -benchmark_scope_add_breadcrumb(benchmark::State &state) -{ - sentry_options_t *options = sentry_options_new(); - sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); - sentry_init(options); - - int i = 0; - for (auto _ : state) { - char msg[32]; - snprintf(msg, sizeof(msg), "message%d", i); - sentry_add_breadcrumb(sentry_value_new_breadcrumb(NULL, msg)); - i++; - } - - sentry_close(); -} - -BENCHMARK(benchmark_scope_add_breadcrumb) +BENCHMARK(benchmark_tags) ->Iterations(1000) ->Unit(benchmark::kMillisecond); From 96a20da9f474a7df81cda72d0a32982a538f9b5f Mon Sep 17 00:00:00 2001 From: J-P Nurmi Date: Thu, 10 Sep 2026 16:04:21 +0200 Subject: [PATCH 3/3] test(benchmark): Add contexts benchmark --- tests/benchmark.py | 12 +++++++++ tests/benchmark/CMakeLists.txt | 1 + tests/benchmark/benchmark_contexts.cpp | 35 ++++++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 tests/benchmark/benchmark_contexts.cpp diff --git a/tests/benchmark.py b/tests/benchmark.py index ecb7792599..ba05c29cb8 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -192,3 +192,15 @@ def test_benchmark_stack_usage(backend, cmake, gmeasurement): "bytes", f"Peak {peak}b, Segments {len(measurements)}", ) + + +@pytest.mark.parametrize("backend", ["inproc", "breakpad", "crashpad", "native"]) +def test_benchmark_contexts(backend, cmake, httpserver, gbenchmark): + run_benchmark( + "benchmark_contexts", + backend, + cmake, + httpserver, + gbenchmark, + f"Contexts ({backend})", + ) diff --git a/tests/benchmark/CMakeLists.txt b/tests/benchmark/CMakeLists.txt index 5e37ea4db9..1056160581 100644 --- a/tests/benchmark/CMakeLists.txt +++ b/tests/benchmark/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(sentry_benchmark benchmark_init.cpp benchmark_backend.cpp benchmark_breadcrumbs.cpp + benchmark_contexts.cpp benchmark_logs.cpp benchmark_metrics.cpp benchmark_tags.cpp diff --git a/tests/benchmark/benchmark_contexts.cpp b/tests/benchmark/benchmark_contexts.cpp new file mode 100644 index 0000000000..16cda6d089 --- /dev/null +++ b/tests/benchmark/benchmark_contexts.cpp @@ -0,0 +1,35 @@ +#include + +extern "C" { +#include "sentry_core.h" +#include "sentry_options.h" +#include "sentry_scope.h" +} + +static void +benchmark_contexts(benchmark::State &state) +{ + sentry_options_t *options = sentry_options_new(); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + // flush both __sentry-event and the external crash report + sentry_options_set_external_crash_reporter_path(options, "."); + sentry_options_set_debug(options, true); + sentry_init(options); + + int i = 0; + for (auto _ : state) { + char key[32], val[32]; + snprintf(key, sizeof(key), "context%d", i); + snprintf(val, sizeof(val), "value%d", i); + sentry_value_t context = sentry_value_new_object(); + sentry_value_set_by_key(context, "value", sentry_value_new_string(val)); + sentry_set_context(key, context); + i++; + } + + sentry_close(); +} + +BENCHMARK(benchmark_contexts) + ->Iterations(1000) + ->Unit(benchmark::kMillisecond);