test: fix and extend benchmarks

tests/benchmark.py

@@ -59,16 +59,27 @@ def test_benchmark_backend(backend, cmake, httpserver, gbenchmark):
     )
 
 
-@pytest.mark.parametrize("test_name", ["set_tag", "add_breadcrumb"])
 @pytest.mark.parametrize("backend", ["inproc", "breakpad", "crashpad", "native"])
-def test_benchmark_scope(test_name, backend, cmake, httpserver, gbenchmark):
+def test_benchmark_tags(backend, cmake, httpserver, gbenchmark):
     run_benchmark(
-        f"benchmark_scope_{test_name}",
+        "benchmark_tags",
         backend,
         cmake,
         httpserver,
         gbenchmark,
-        f"Scope {test_name} ({backend})",
+        f"Tags ({backend})",
+    )
+
+
+@pytest.mark.parametrize("backend", ["inproc", "breakpad", "crashpad", "native"])
+def test_benchmark_breadcrumbs(backend, cmake, httpserver, gbenchmark):
+    run_benchmark(
+        "benchmark_breadcrumbs",
+        backend,
+        cmake,
+        httpserver,
+        gbenchmark,
+        f"Breadcrumbs ({backend})",
     )
 
 
@@ -84,6 +95,18 @@ def test_benchmark_logs(threads, cmake, httpserver, gbenchmark):
     )
 
 
+@pytest.mark.parametrize("threads", [1, 8, 16, 32])
+def test_benchmark_metrics(threads, cmake, httpserver, gbenchmark):
+    run_benchmark(
+        f"^benchmark_metrics.*threads:{threads}$",
+        "none",
+        cmake,
+        httpserver,
+        gbenchmark,
+        f"Metrics ({threads} thread{'s' if threads > 1 else ''})",
+    )
+
+
 @pytest.mark.parametrize("backend", ["inproc", "breakpad", "crashpad", "native"])
 def test_benchmark_libsize(backend, cmake, gmeasurement):
     tmp_path = cmake(
@@ -169,3 +192,15 @@ def test_benchmark_stack_usage(backend, cmake, gmeasurement):
         "bytes",
         f"Peak {peak}b, Segments {len(measurements)}",
     )
+
+
+@pytest.mark.parametrize("backend", ["inproc", "breakpad", "crashpad", "native"])
+def test_benchmark_contexts(backend, cmake, httpserver, gbenchmark):
+    run_benchmark(
+        "benchmark_contexts",
+        backend,
+        cmake,
+        httpserver,
+        gbenchmark,
+        f"Contexts ({backend})",
+    )

tests/benchmark/CMakeLists.txt

@@ -11,8 +11,11 @@ add_executable(sentry_benchmark
 	${SENTRY_SOURCES}
 	benchmark_init.cpp
 	benchmark_backend.cpp
+	benchmark_breadcrumbs.cpp
+	benchmark_contexts.cpp
 	benchmark_logs.cpp
-	benchmark_scope.cpp
+	benchmark_metrics.cpp
+	benchmark_tags.cpp
 )
 
 if(SENTRY_BACKEND_CRASHPAD)

tests/benchmark/benchmark_breadcrumbs.cpp

@@ -0,0 +1,29 @@
+#include <benchmark/benchmark.h>
+
+extern "C" {
+#include "sentry_core.h"
+#include "sentry_options.h"
+#include "sentry_scope.h"
+}
+
+static void
+benchmark_breadcrumbs(benchmark::State &state)
+{
+    sentry_options_t *options = sentry_options_new();
+    sentry_options_set_dsn(options, "https://foo@sentry.invalid/42");
+    sentry_init(options);
+
+    int i = 0;
+    for (auto _ : state) {
+        char msg[32];
+        snprintf(msg, sizeof(msg), "message%d", i);
+        sentry_add_breadcrumb(sentry_value_new_breadcrumb(NULL, msg));
+        i++;
+    }
+
+    sentry_close();
+}
+
+BENCHMARK(benchmark_breadcrumbs)
+    ->Iterations(1000)
+    ->Unit(benchmark::kMillisecond);

tests/benchmark/benchmark_contexts.cpp

@@ -0,0 +1,35 @@
+#include <benchmark/benchmark.h>
+
+extern "C" {
+#include "sentry_core.h"
+#include "sentry_options.h"
+#include "sentry_scope.h"
+}
+
+static void
+benchmark_contexts(benchmark::State &state)
+{
+    sentry_options_t *options = sentry_options_new();
+    sentry_options_set_dsn(options, "https://foo@sentry.invalid/42");
+    // flush both __sentry-event and the external crash report
+    sentry_options_set_external_crash_reporter_path(options, ".");
+    sentry_options_set_debug(options, true);
+    sentry_init(options);
+
+    int i = 0;
+    for (auto _ : state) {
+        char key[32], val[32];
+        snprintf(key, sizeof(key), "context%d", i);
+        snprintf(val, sizeof(val), "value%d", i);
+        sentry_value_t context = sentry_value_new_object();
+        sentry_value_set_by_key(context, "value", sentry_value_new_string(val));
+        sentry_set_context(key, context);
+        i++;
+    }
+
+    sentry_close();
+}
+
+BENCHMARK(benchmark_contexts)
+    ->Iterations(1000)
+    ->Unit(benchmark::kMillisecond);

tests/benchmark/benchmark_logs.cpp

@@ -5,9 +5,8 @@ extern "C" {
 }
 
 static void
-discard_envelope(sentry_envelope_t *envelope, void *state)
+discard_envelope(sentry_envelope_t *envelope, void *)
 {
-    (void)state;
     sentry_envelope_free(envelope);
 }
 
@@ -16,6 +15,8 @@ setup_logs(const benchmark::State &)
 {
     sentry_options_t *options = sentry_options_new();
     sentry_options_set_dsn(options, "https://foo@sentry.invalid/42");
+    sentry_options_set_release(options, "benchmark@1.0");
+    sentry_options_set_environment(options, "test");
     sentry_options_set_logs_with_attributes(options, true);
     sentry_options_set_auto_session_tracking(options, 0);
     sentry_options_set_transport(
@@ -40,9 +41,16 @@ benchmark_logs(benchmark::State &state)
         sentry_value_new_attribute(sentry_value_new_string("attribute"), NULL));
 
     int i = 0;
+    int failed = 0;
     for (auto _ : state) {
-        sentry_log_info("log %d", sentry_value_incref(attributes), i++, NULL);
+        if (sentry_log_info(
+                "log %d", sentry_value_incref(attributes), i++, NULL)
+            != SENTRY_LOG_RETURN_SUCCESS) {
+            failed++;
+        }
     }
+    state.SetItemsProcessed(state.iterations() - failed);
+    state.counters["enqueue_failures"] = failed;
 
     sentry_value_decref(attributes);
 }
@@ -52,7 +60,8 @@ BENCHMARK(benchmark_logs)
     ->Threads(8)
     ->Threads(16)
     ->Threads(32)
-    ->Iterations(100)
+    ->Iterations(32) // 32x32=1024 (peak) > 10x100=1000 (capacity)
+    ->UseRealTime()
     ->Unit(benchmark::kMillisecond)
     ->Setup(setup_logs)
     ->Teardown(teardown_logs);

tests/benchmark/benchmark_metrics.cpp

@@ -0,0 +1,65 @@
+#include <benchmark/benchmark.h>
+
+extern "C" {
+#include "sentry.h"
+}
+
+static void
+discard_envelope(sentry_envelope_t *envelope, void *)
+{
+    sentry_envelope_free(envelope);
+}
+
+static void
+setup_metrics(const benchmark::State &)
+{
+    sentry_options_t *options = sentry_options_new();
+    sentry_options_set_dsn(options, "https://foo@sentry.invalid/42");
+    sentry_options_set_release(options, "benchmark@1.0");
+    sentry_options_set_environment(options, "test");
+    sentry_options_set_auto_session_tracking(options, 0);
+    sentry_options_set_transport(
+        options, sentry_transport_new(discard_envelope));
+    sentry_init(options);
+
+    sentry_set_attribute("global",
+        sentry_value_new_attribute(sentry_value_new_string("attribute"), NULL));
+}
+
+static void
+teardown_metrics(const benchmark::State &)
+{
+    sentry_close();
+}
+
+static void
+benchmark_metrics(benchmark::State &state)
+{
+    sentry_value_t attributes = sentry_value_new_object();
+    sentry_value_set_by_key(attributes, "asset.type",
+        sentry_value_new_attribute(sentry_value_new_string("texture"), NULL));
+
+    int failed = 0;
+    for (auto _ : state) {
+        if (sentry_metrics_distribution("asset.load.duration", 12.5,
+                SENTRY_UNIT_MILLISECOND, sentry_value_incref(attributes))
+            != SENTRY_METRICS_RESULT_SUCCESS) {
+            failed++;
+        }
+    }
+    state.SetItemsProcessed(state.iterations() - failed);
+    state.counters["enqueue_failures"] = failed;
+
+    sentry_value_decref(attributes);
+}
+
+BENCHMARK(benchmark_metrics)
+    ->Threads(1)
+    ->Threads(8)
+    ->Threads(16)
+    ->Threads(32)
+    ->Iterations(32) // 32x32=1024 (peak) > 10x100=1000 (capacity)
+    ->UseRealTime()
+    ->Unit(benchmark::kMillisecond)
+    ->Setup(setup_metrics)
+    ->Teardown(teardown_metrics);

tests/benchmark/benchmark_scope.cpp → tests/benchmark/benchmark_tags.cpp

@@ -7,7 +7,7 @@ extern "C" {
 }
 
 static void
-benchmark_scope_set_tag(benchmark::State &state)
+benchmark_tags(benchmark::State &state)
 {
     sentry_options_t *options = sentry_options_new();
     sentry_options_set_dsn(options, "https://foo@sentry.invalid/42");
@@ -28,28 +28,6 @@ benchmark_scope_set_tag(benchmark::State &state)
     sentry_close();
 }
 
-BENCHMARK(benchmark_scope_set_tag)
-    ->Iterations(1000)
-    ->Unit(benchmark::kMillisecond);
-
-static void
-benchmark_scope_add_breadcrumb(benchmark::State &state)
-{
-    sentry_options_t *options = sentry_options_new();
-    sentry_options_set_dsn(options, "https://foo@sentry.invalid/42");
-    sentry_init(options);
-
-    int i = 0;
-    for (auto _ : state) {
-        char msg[32];
-        snprintf(msg, sizeof(msg), "message%d", i);
-        sentry_add_breadcrumb(sentry_value_new_breadcrumb(NULL, msg));
-        i++;
-    }
-
-    sentry_close();
-}
-
-BENCHMARK(benchmark_scope_add_breadcrumb)
+BENCHMARK(benchmark_tags)
     ->Iterations(1000)
     ->Unit(benchmark::kMillisecond);