diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 74c1ac6..a3c0931 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -2,30 +2,197 @@ name: CI on: push: + branches: [main] paths: - 'csharp/**' + - 'scripts/**' + - '.github/workflows/CI.yml' + pull_request: + branches: [main] + paths: + - 'csharp/**' + - 'scripts/**' - '.github/workflows/CI.yml' defaults: run: working-directory: csharp +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: - build-linux: + test: + name: Test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + - name: Build + run: dotnet build Comparisons.SQLiteVSDoublets.sln -c Release --warnaserror + - name: Verify every storage variant + run: dotnet run -c Release --no-build -- --self-test + + results-pipeline: + name: Results pipeline tests runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - uses: actions/checkout@v1 - - name: Run - run: dotnet run -c Release - build-windows: - runs-on: windows-latest + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install chart dependencies + run: python -m pip install matplotlib numpy + working-directory: . + - name: Test shared and C# reporting + run: | + python -m unittest discover -s scripts -p 'test_*.py' -v + python -m unittest discover -s csharp -p 'test_*.py' -v + working-directory: . + + benchmark-pr: + name: Benchmark (PR validation) + if: github.event_name == 'pull_request' + needs: [test, results-pipeline] + runs-on: ubuntu-latest + timeout-minutes: 45 steps: - - uses: actions/checkout@v1 - - name: Run - run: dotnet run -c Release - build-macos: - runs-on: macos-latest + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install chart dependencies + run: python -m pip install matplotlib numpy + working-directory: . + - name: Build benchmark + run: dotnet build Comparisons.SQLiteVSDoublets.sln -c Release + - name: Run reduced-scale benchmark + env: + BENCHMARK_LINK_COUNT: 10 + BACKGROUND_LINK_COUNT: 30 + BENCHMARK_WARMUP_COUNT: 1 + BENCHMARK_ITERATION_COUNT: 1 + run: >- + dotnet run -c Release --no-build -- + --filter '*LinksBenchmarks*' > benchmark.log 2>&1 + - name: Generate table and charts + env: + BENCHMARK_LINK_COUNT: 10 + BACKGROUND_LINK_COUNT: 30 + run: | + mkdir -p artifacts + report_path=$(find BenchmarkDotNet.Artifacts/results -name '*LinksBenchmarks-report-full.json' -print -quit) + test -n "$report_path" + python out.py "$report_path" \ + --results artifacts/results.md \ + --output-dir artifacts \ + --docs-dir artifacts/docs + - name: Add results to job summary + run: | + { + echo '## C# benchmark (reduced-scale validation)' + echo + cat artifacts/results.md + echo + echo '_These numbers validate the pipeline; full results are published from `main`._' + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload diagnostic and report artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: csharp-benchmark-pr + path: | + csharp/benchmark.log + csharp/BenchmarkDotNet.Artifacts/results/ + csharp/artifacts/ + + benchmark: + name: Benchmark (full) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: [test, results-pipeline] + runs-on: ubuntu-latest + timeout-minutes: 180 + concurrency: + group: benchmark-publication + cancel-in-progress: false + permissions: + contents: write steps: - - uses: actions/checkout@v1 - - name: Run - run: dotnet run -c Release + - uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install chart dependencies + run: python -m pip install matplotlib numpy + working-directory: . + - name: Build benchmark + run: dotnet build Comparisons.SQLiteVSDoublets.sln -c Release + - name: Run full benchmark + env: + BENCHMARK_LINK_COUNT: 1000 + BACKGROUND_LINK_COUNT: 3000 + BENCHMARK_WARMUP_COUNT: 2 + BENCHMARK_ITERATION_COUNT: 5 + run: >- + dotnet run -c Release --no-build -- + --filter '*LinksBenchmarks*' > benchmark.log 2>&1 + - name: Generate and publish documentation + env: + BENCHMARK_LINK_COUNT: 1000 + BACKGROUND_LINK_COUNT: 3000 + run: | + report_path=$(find BenchmarkDotNet.Artifacts/results -name '*LinksBenchmarks-report-full.json' -print -quit) + test -n "$report_path" + python out.py "$report_path" \ + --results results.md \ + --readme ../README.md \ + --readme ../README.ru.md \ + --docs-dir ../docs/benchmarks + - name: Add results to job summary + run: | + { + echo '## C# benchmark (full scale)' + echo + cat results.md + } >> "$GITHUB_STEP_SUMMARY" + - name: Commit published results + working-directory: . + run: | + git config user.email 'linksplatform@gmail.com' + git config user.name 'LinksPlatformBencher' + git add README.md README.ru.md csharp/results.md docs/benchmarks/ + if git diff --staged --quiet; then + echo 'No benchmark documentation changed.' + else + git commit -m 'Update C# benchmark results [skip ci]' + git push origin HEAD:main + fi + - name: Upload diagnostic and report artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: csharp-benchmark-full + path: | + csharp/benchmark.log + csharp/BenchmarkDotNet.Artifacts/results/ + csharp/results.md + csharp/bench_csharp.png + csharp/bench_csharp_log_scale.png diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index aaaff79..0d5d0bb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -2,27 +2,36 @@ name: Rust on: push: - branches: main + branches: [main] paths: - 'rust/**' + - 'scripts/**' - '.github/workflows/rust.yml' pull_request: + branches: [main] paths: - 'rust/**' + - 'scripts/**' - '.github/workflows/rust.yml' env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + RUST_TOOLCHAIN: nightly-2022-08-22 defaults: run: working-directory: rust +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: test: name: Test (${{ matrix.os }}) runs-on: ${{ matrix.os }} + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -30,21 +39,196 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup Rust + - name: Setup pinned Rust nightly uses: dtolnay/rust-toolchain@master with: - toolchain: nightly-2022-08-22 + toolchain: ${{ env.RUST_TOOLCHAIN }} + components: rustfmt, clippy + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: rust -> target + cache-on-failure: 'true' + + - name: Check formatting + run: cargo fmt --all -- --check + + - name: Run Clippy + run: cargo clippy --lib --tests --benches -- -D warnings + + - name: Run unit and integration tests + run: cargo test --release --lib --tests + + # `cargo test --all-targets` executes Criterion binaries. Compile them + # explicitly so file-backed measurements are only run in bounded jobs. + - name: Compile benchmarks + run: cargo check --release --benches + + results-pipeline: + name: Results pipeline tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install chart dependencies + run: python -m pip install matplotlib numpy + working-directory: . + - name: Test shared and Rust reporting + run: | + python -m unittest discover -s scripts -p 'test_*.py' -v + python -m unittest discover -s rust -p 'test_*.py' -v + working-directory: . - - name: Cache cargo registry - uses: actions/cache@v4 + benchmark-pr: + name: Benchmark (PR validation) + if: github.event_name == 'pull_request' + needs: [test, results-pipeline] + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - name: Setup pinned Rust nightly + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + components: rustfmt, clippy + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install chart dependencies + run: python -m pip install matplotlib numpy + working-directory: . + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: rust -> target + cache-on-failure: 'true' + - name: Run reduced-scale benchmark + env: + BENCHMARK_LINK_COUNT: 10 + BACKGROUND_LINK_COUNT: 30 + BENCHMARK_OBJECT_COUNT: 5 + BENCHMARK_SAMPLE_SIZE: 10 + BENCHMARK_MEASUREMENT_SECONDS: 1 + BENCHMARK_WARM_UP_MILLISECONDS: 100 + run: | + set -o pipefail + rm -rf target/criterion + cargo bench --bench bench -- --output-format bencher | tee out.txt + - name: Generate table and charts + env: + BENCHMARK_LINK_COUNT: 10 + BACKGROUND_LINK_COUNT: 30 + BENCHMARK_OBJECT_COUNT: 5 + run: | + mkdir -p artifacts + python out.py out.txt \ + --results artifacts/results.md \ + --output-dir artifacts \ + --docs-dir artifacts/docs + - name: Add results to job summary + run: | + { + echo '## Rust benchmark (reduced-scale validation)' + echo + cat artifacts/results.md + echo + echo '_These numbers validate the pipeline; full results are published from `main`._' + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload diagnostic and report artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: rust-benchmark-pr + path: | + rust/out.txt + rust/artifacts/ + + benchmark: + name: Benchmark (full) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: [test, results-pipeline] + runs-on: ubuntu-latest + timeout-minutes: 180 + concurrency: + group: benchmark-publication + cancel-in-progress: false + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + - name: Setup pinned Rust nightly + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + components: rustfmt, clippy + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install chart dependencies + run: python -m pip install matplotlib numpy + working-directory: . + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: rust -> target + cache-on-failure: 'true' + - name: Run full benchmark + env: + BENCHMARK_LINK_COUNT: 1000 + BACKGROUND_LINK_COUNT: 3000 + BENCHMARK_OBJECT_COUNT: 1000 + BENCHMARK_SAMPLE_SIZE: 20 + BENCHMARK_MEASUREMENT_SECONDS: 3 + BENCHMARK_WARM_UP_MILLISECONDS: 1000 + run: | + set -o pipefail + rm -rf target/criterion + cargo bench --bench bench -- --output-format bencher | tee out.txt + - name: Generate and publish documentation + env: + BENCHMARK_LINK_COUNT: 1000 + BACKGROUND_LINK_COUNT: 3000 + BENCHMARK_OBJECT_COUNT: 1000 + run: | + python out.py out.txt \ + --results results.md \ + --readme ../README.md \ + --readme ../README.ru.md \ + --docs-dir ../docs/benchmarks + - name: Add results to job summary + run: | + { + echo '## Rust benchmark (full scale)' + echo + cat results.md + } >> "$GITHUB_STEP_SUMMARY" + - name: Commit published results + working-directory: . + run: | + git config user.email 'linksplatform@gmail.com' + git config user.name 'LinksPlatformBencher' + git add README.md README.ru.md rust/results.md docs/benchmarks/ + if git diff --staged --quiet; then + echo 'No benchmark documentation changed.' + else + git commit -m 'Update Rust benchmark results [skip ci]' + git push origin HEAD:main + fi + - name: Upload diagnostic and report artifacts + if: always() + uses: actions/upload-artifact@v4 with: + name: rust-benchmark-full path: | - ~/.cargo/registry - ~/.cargo/git - rust/target - key: ${{ runner.os }}-cargo-${{ hashFiles('rust/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - - name: Run tests - run: cargo test --release + rust/out.txt + rust/results.md + rust/bench_rust.png + rust/bench_rust_log_scale.png diff --git a/.gitignore b/.gitignore index 86402cb..6377d22 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ cpp/*.so cpp/*.exe # Common temporary files +__pycache__/ +*.py[cod] *.tmp *.log *.swp diff --git a/README.md b/README.md index bebc4e6..b9236af 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,56 @@ ![Comparison of models](https://github.com/LinksPlatform/Documentation/raw/master/doc/ModelsComparison/relational_model_vs_associative_model_vs_links.png) -Comparison of SQLite and LinksPlatform's Doublets (links) on basic embeded database operations with objects (create list, read list, delete list). +Comparison of SQLite and LinksPlatform's Doublets (links) on basic embedded database operations with links and object-like structures. Based on examples from https://github.com/FahaoTang/dotnetcore-examples and https://github.com/Konard/LinksPlatform +## Automated benchmark suite + +The comparison runs the same link workload against SQLite in-memory and file +databases and four Doublets layouts: united/split and volatile/non-volatile. +It measures create, update, delete, enumerate all, and queries by identity, +concrete `(source, target)`, outgoing source, and incoming target. The Rust +suite also measures creation, reading, and deletion of object-like blog posts +on every storage variant. + +Preparation and cleanup are outside the measured region. Pull requests run a +reduced-scale validation and preserve raw output, tables, and charts as +workflow artifacts. Full runs on `main` publish the tables and charts below +with a link to the producing workflow run. + +Run the checks locally: + +```bash +cd rust +cargo test --lib --tests +cargo check --benches +BENCHMARK_LINK_COUNT=10 BACKGROUND_LINK_COUNT=30 \ + BENCHMARK_OBJECT_COUNT=5 cargo bench --bench bench -- \ + --output-format bencher + +cd ../csharp +dotnet run -c Release -- --self-test +BENCHMARK_LINK_COUNT=10 BACKGROUND_LINK_COUNT=30 \ + dotnet run -c Release -- --filter '*LinksBenchmarks*' +``` + +### C# link results + + +> Results will be generated by the full benchmark workflow after this change +> reaches `main`. + + +### Rust link and object results + + +> Results will be generated by the full benchmark workflow after this change +> reaches `main`. + + +The original C# object comparison and its historical results remain below. + ## SQLite ```C# using System.Linq; diff --git a/README.ru.md b/README.ru.md index 68be01c..931e2e9 100644 --- a/README.ru.md +++ b/README.ru.md @@ -4,10 +4,56 @@ ![Сравнение моделей данных](https://github.com/LinksPlatform/Documentation/raw/master/doc/ModelsComparison/relational_model_vs_associative_model_vs_links_ru.png) -Сравнение SQLite и Дуплетов ПлатформыСвязей на базовых операциях в качестве встариваемых баз данных с объектами (создание списка, чтение списка, удаление списка). +Сравнение SQLite и Дуплетов ПлатформыСвязей на базовых операциях встроенных баз данных со связями и объектоподобными структурами. Основано на примерах из https://github.com/FahaoTang/dotnetcore-examples и https://github.com/Konard/LinksPlatform +## Автоматизированный набор тестов производительности + +Одинаковая нагрузка со связями выполняется для SQLite в памяти и в файле, а +также для четырёх вариантов Дуплетов: объединённого/разделённого и +энергозависимого/энергонезависимого. Измеряются создание, обновление, удаление, +перечисление всех связей и запросы по идентификатору, конкретной паре +`(начало, конец)`, началу и концу. Набор на Rust также измеряет создание, +чтение и удаление объектоподобных записей блога во всех вариантах хранилищ. + +Подготовка и очистка данных не входят в измеряемый интервал. Для pull request +запускается сокращённая проверка, а исходный вывод, таблицы и диаграммы +сохраняются как артефакты workflow. Полные запуски в `main` публикуют таблицы и +диаграммы ниже вместе со ссылкой на создавший их запуск. + +Локальный запуск проверок: + +```bash +cd rust +cargo test --lib --tests +cargo check --benches +BENCHMARK_LINK_COUNT=10 BACKGROUND_LINK_COUNT=30 \ + BENCHMARK_OBJECT_COUNT=5 cargo bench --bench bench -- \ + --output-format bencher + +cd ../csharp +dotnet run -c Release -- --self-test +BENCHMARK_LINK_COUNT=10 BACKGROUND_LINK_COUNT=30 \ + dotnet run -c Release -- --filter '*LinksBenchmarks*' +``` + +### Результаты операций со связями в C#-версии + + +> Результаты будут созданы полным workflow после попадания изменений в +> `main`. + + +### Результаты операций со связями и объектами на Rust + + +> Результаты будут созданы полным workflow после попадания изменений в +> `main`. + + +Исходное сравнение объектов на C# и его исторические результаты сохранены ниже. + ## SQLite ```C# using System.Linq; diff --git a/csharp/Benchmarks.cs b/csharp/Benchmarks.cs index 253dff7..ee78467 100644 --- a/csharp/Benchmarks.cs +++ b/csharp/Benchmarks.cs @@ -16,13 +16,13 @@ public class Benchmarks { private class Config : ManualConfig { - public Config() => Add(new SizeAfterCreationColumn()); + public Config() => AddColumn(new SizeAfterCreationColumn()); } [Params(1000, 10000, 100000)] public int N; - private SQLiteTestRun _sqliteTestRun; - private DoubletsTestRun _doubletsTestRun; + private SQLiteTestRun _sqliteTestRun = null!; + private DoubletsTestRun _doubletsTestRun = null!; [GlobalSetup] public void Setup() diff --git a/csharp/Comparisons.SQLiteVSDoublets.csproj b/csharp/Comparisons.SQLiteVSDoublets.csproj index 1fde215..58d6b82 100644 --- a/csharp/Comparisons.SQLiteVSDoublets.csproj +++ b/csharp/Comparisons.SQLiteVSDoublets.csproj @@ -4,6 +4,11 @@ Exe net8.0 enable + + $(NoWarn);NETSDK1206 diff --git a/csharp/Doublets/DoubletsDbContext.cs b/csharp/Doublets/DoubletsDbContext.cs index d163713..b1b7ecd 100644 --- a/csharp/Doublets/DoubletsDbContext.cs +++ b/csharp/Doublets/DoubletsDbContext.cs @@ -293,7 +293,7 @@ public IList GetBlogPosts() // All links that match this query are BlogPosts. var any = _links.Constants.Any; var query = new Link(any, _blogPostMarker, any); - _links.Each(listFiller.AddAndReturnConstant, query); + _links.Each(element => listFiller.AddAndReturnConstant(element!), query); return list.Select(LoadBlogPost).ToList(); } diff --git a/csharp/Links/DoubletsLinksStorage.cs b/csharp/Links/DoubletsLinksStorage.cs new file mode 100644 index 0000000..e899390 --- /dev/null +++ b/csharp/Links/DoubletsLinksStorage.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Platform.Data.Doublets; +using Platform.Disposables; + +namespace Comparisons.SQLiteVSDoublets.Links; + +public sealed class DoubletsLinksStorage : ILinksStorage +{ + private readonly ILinks _links; + private readonly IReadOnlyList _paths; + + public DoubletsLinksStorage(ILinks links, params string[] paths) + { + _links = links; + _paths = paths; + } + + public uint Create(uint source, uint target) => _links.CreateAndUpdate(source, target); + + public uint CreatePoint() => _links.CreatePoint(); + + public void Update(uint id, uint source, uint target) => _links.Update(id, source, target); + + public void Delete(uint id) => Platform.Data.ILinksExtensions.Delete(_links, id); + + public IReadOnlyList QueryAll() + { + var any = _links.Constants.Any; + return Convert(_links.All(any, any, any)); + } + + public LinkRecord? QueryById(uint id) + { + var any = _links.Constants.Any; + var result = _links.All(id, any, any); + return result.Count == 0 || result[0] is null ? null : Convert(result[0]!); + } + + public IReadOnlyList QueryBySourceTarget(uint source, uint target) + { + var any = _links.Constants.Any; + return Convert(_links.All(any, source, target)); + } + + public IReadOnlyList QueryBySource(uint source) + { + var any = _links.Constants.Any; + return Convert(_links.All(any, source, any)); + } + + public IReadOnlyList QueryByTarget(uint target) + { + var any = _links.Constants.Any; + return Convert(_links.All(any, any, target)); + } + + public int Count => QueryAll().Count; + + public void Dispose() + { + _links.DisposeIfPossible(); + foreach (var path in _paths.Where(File.Exists)) + { + File.Delete(path); + } + } + + private LinkRecord Convert(IList link) => new( + _links.GetIndex(link), + _links.GetSource(link), + _links.GetTarget(link)); + + private List Convert(IList?> links) => + links.Where(link => link is not null).Select(link => Convert(link!)).ToList(); +} diff --git a/csharp/Links/ILinksStorage.cs b/csharp/Links/ILinksStorage.cs new file mode 100644 index 0000000..606f5a7 --- /dev/null +++ b/csharp/Links/ILinksStorage.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; + +namespace Comparisons.SQLiteVSDoublets.Links; + +public interface ILinksStorage : IDisposable +{ + uint Create(uint source, uint target); + + uint CreatePoint(); + + void Update(uint id, uint source, uint target); + + void Delete(uint id); + + IReadOnlyList QueryAll(); + + LinkRecord? QueryById(uint id); + + IReadOnlyList QueryBySourceTarget(uint source, uint target); + + IReadOnlyList QueryBySource(uint source); + + IReadOnlyList QueryByTarget(uint target); + + int Count { get; } +} diff --git a/csharp/Links/LinkRecord.cs b/csharp/Links/LinkRecord.cs new file mode 100644 index 0000000..3b75d83 --- /dev/null +++ b/csharp/Links/LinkRecord.cs @@ -0,0 +1,3 @@ +namespace Comparisons.SQLiteVSDoublets.Links; + +public readonly record struct LinkRecord(uint Id, uint Source, uint Target); diff --git a/csharp/Links/LinksData.cs b/csharp/Links/LinksData.cs new file mode 100644 index 0000000..4bf84d2 --- /dev/null +++ b/csharp/Links/LinksData.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; + +namespace Comparisons.SQLiteVSDoublets.Links; + +public static class LinksData +{ + public static uint[] FillBackground(ILinksStorage storage, int count) + { + var ids = new uint[count]; + for (var index = 0; index < count; index++) + { + ids[index] = storage.CreatePoint(); + } + return ids; + } + + public static LinkRecord[] FillBenchmarked( + ILinksStorage storage, + IReadOnlyList background, + int count) + { + if (background.Count == 0) + { + throw new ArgumentException("Background links are required.", nameof(background)); + } + + var links = new LinkRecord[count]; + for (var index = 0; index < count; index++) + { + var source = background[index % background.Count]; + var target = background[((index % background.Count) + 1 + (index / background.Count)) + % background.Count]; + links[index] = new LinkRecord(storage.Create(source, target), source, target); + } + return links; + } +} diff --git a/csharp/Links/LinksStorageFactory.cs b/csharp/Links/LinksStorageFactory.cs new file mode 100644 index 0000000..d6bad6a --- /dev/null +++ b/csharp/Links/LinksStorageFactory.cs @@ -0,0 +1,55 @@ +using System; +using System.IO; +using Platform.Data.Doublets; +using Platform.Data.Doublets.Memory.Split.Generic; +using Platform.Data.Doublets.Memory.United.Generic; +using Platform.Memory; + +namespace Comparisons.SQLiteVSDoublets.Links; + +public enum LinksVariant +{ + SQLite_Memory, + SQLite_File, + Doublets_United_Volatile, + Doublets_United_NonVolatile, + Doublets_Split_Volatile, + Doublets_Split_NonVolatile, +} + +public static class LinksStorageFactory +{ + public static ILinksStorage Create(LinksVariant variant) => variant switch + { + LinksVariant.SQLite_Memory => new SQLiteLinksStorage(inMemory: true), + LinksVariant.SQLite_File => new SQLiteLinksStorage(inMemory: false), + LinksVariant.Doublets_United_Volatile => new DoubletsLinksStorage( + new UnitedMemoryLinks(new HeapResizableDirectMemory())), + LinksVariant.Doublets_United_NonVolatile => CreateUnitedFile(), + LinksVariant.Doublets_Split_Volatile => new DoubletsLinksStorage( + new SplitMemoryLinks( + new HeapResizableDirectMemory(), + new HeapResizableDirectMemory())), + LinksVariant.Doublets_Split_NonVolatile => CreateSplitFile(), + _ => throw new ArgumentOutOfRangeException(nameof(variant), variant, null), + }; + + private static DoubletsLinksStorage CreateUnitedFile() + { + var path = TemporaryPath("united.links"); + ILinks links = new UnitedMemoryLinks(path); + return new DoubletsLinksStorage(links, path); + } + + private static DoubletsLinksStorage CreateSplitFile() + { + var dataPath = TemporaryPath("split.data.links"); + var indexPath = TemporaryPath("split.index.links"); + ILinks links = new SplitMemoryLinks(dataPath, indexPath); + return new DoubletsLinksStorage(links, dataPath, indexPath); + } + + private static string TemporaryPath(string suffix) => Path.Combine( + Path.GetTempPath(), + $"sqlite-vs-doublets-{Guid.NewGuid():N}-{suffix}"); +} diff --git a/csharp/Links/SQLiteLinksStorage.cs b/csharp/Links/SQLiteLinksStorage.cs new file mode 100644 index 0000000..f46a53b --- /dev/null +++ b/csharp/Links/SQLiteLinksStorage.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Data.Sqlite; + +namespace Comparisons.SQLiteVSDoublets.Links; + +public sealed class SQLiteLinksStorage : ILinksStorage +{ + private readonly SqliteConnection _connection; + private readonly string? _path; + private uint _nextId = 1; + + public SQLiteLinksStorage(bool inMemory) + { + _path = inMemory + ? null + : Path.Combine(Path.GetTempPath(), $"sqlite-vs-doublets-{Guid.NewGuid():N}.db"); + var dataSource = _path ?? ":memory:"; + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = dataSource, + // A pooled native connection keeps temporary database files open + // after Dispose, which prevents their immediate removal on Windows. + Pooling = false, + }.ToString(); + _connection = new SqliteConnection(connectionString); + _connection.Open(); + using var command = _connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE links ( + id INTEGER PRIMARY KEY, + source INTEGER NOT NULL, + target INTEGER NOT NULL + ); + CREATE INDEX idx_source ON links(source); + CREATE INDEX idx_target ON links(target); + CREATE INDEX idx_source_target ON links(source, target); + """; + command.ExecuteNonQuery(); + } + + public uint Create(uint source, uint target) + { + var id = _nextId++; + using var command = Command( + "INSERT INTO links (id, source, target) VALUES ($id, $source, $target)", + ("$id", id), + ("$source", source), + ("$target", target)); + command.ExecuteNonQuery(); + return id; + } + + public uint CreatePoint() + { + var id = Create(0, 0); + Update(id, id, id); + return id; + } + + public void Update(uint id, uint source, uint target) + { + using var command = Command( + "UPDATE links SET source = $source, target = $target WHERE id = $id", + ("$source", source), + ("$target", target), + ("$id", id)); + command.ExecuteNonQuery(); + } + + public void Delete(uint id) + { + using var command = Command("DELETE FROM links WHERE id = $id", ("$id", id)); + command.ExecuteNonQuery(); + } + + public IReadOnlyList QueryAll() => Query( + "SELECT id, source, target FROM links"); + + public LinkRecord? QueryById(uint id) + { + var links = Query( + "SELECT id, source, target FROM links WHERE id = $id", + ("$id", id)); + return links.Count == 0 ? null : links[0]; + } + + public IReadOnlyList QueryBySourceTarget(uint source, uint target) => Query( + "SELECT id, source, target FROM links WHERE source = $source AND target = $target", + ("$source", source), + ("$target", target)); + + public IReadOnlyList QueryBySource(uint source) => Query( + "SELECT id, source, target FROM links WHERE source = $source", + ("$source", source)); + + public IReadOnlyList QueryByTarget(uint target) => Query( + "SELECT id, source, target FROM links WHERE target = $target", + ("$target", target)); + + public int Count + { + get + { + using var command = _connection.CreateCommand(); + command.CommandText = "SELECT COUNT(*) FROM links"; + return Convert.ToInt32(command.ExecuteScalar()); + } + } + + public void Dispose() + { + _connection.Dispose(); + if (_path is not null) + { + File.Delete(_path); + } + } + + private SqliteCommand Command( + string sql, + params (string Name, uint Value)[] parameters) + { + var command = _connection.CreateCommand(); + command.CommandText = sql; + foreach (var parameter in parameters) + { + command.Parameters.AddWithValue(parameter.Name, parameter.Value); + } + return command; + } + + private List Query( + string sql, + params (string Name, uint Value)[] parameters) + { + using var command = Command(sql, parameters); + using var reader = command.ExecuteReader(); + var result = new List(); + while (reader.Read()) + { + result.Add(new LinkRecord( + checked((uint)reader.GetInt64(0)), + checked((uint)reader.GetInt64(1)), + checked((uint)reader.GetInt64(2)))); + } + return result; + } +} diff --git a/csharp/LinksBenchmarks.cs b/csharp/LinksBenchmarks.cs new file mode 100644 index 0000000..bf3545f --- /dev/null +++ b/csharp/LinksBenchmarks.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Attributes; +using Comparisons.SQLiteVSDoublets.Links; + +namespace Comparisons.SQLiteVSDoublets; + +[MemoryDiagnoser] +public class LinksBenchmarks +{ + private ILinksStorage _storage = null!; + private uint[] _background = null!; + private uint[] _updateSources = null!; + private LinkRecord[] _links = null!; + + [ParamsAllValues] + public LinksVariant Variant { get; set; } + + [ParamsSource(nameof(LinkCounts))] + public int N { get; set; } + + public IEnumerable LinkCounts => new[] { EnvironmentValue("BENCHMARK_LINK_COUNT", 1000) }; + + [IterationSetup(Target = nameof(Create))] + public void SetupCreate() => Setup(createBenchmarked: false); + + [IterationSetup(Target = nameof(Update))] + public void SetupUpdate() + { + Setup(createBenchmarked: true); + _updateSources = LinksData.FillBackground(_storage, N); + } + + [IterationSetup(Target = nameof(Delete))] + public void SetupDelete() => Setup(createBenchmarked: true); + + [IterationSetup(Target = nameof(EachAll))] + public void SetupEachAll() => Setup(createBenchmarked: true); + + [IterationSetup(Target = nameof(EachIdentity))] + public void SetupEachIdentity() => Setup(createBenchmarked: true); + + [IterationSetup(Target = nameof(EachConcrete))] + public void SetupEachConcrete() => Setup(createBenchmarked: true); + + [IterationSetup(Target = nameof(EachOutgoing))] + public void SetupEachOutgoing() => Setup(createBenchmarked: true); + + [IterationSetup(Target = nameof(EachIncoming))] + public void SetupEachIncoming() => Setup(createBenchmarked: true); + + [IterationCleanup] + public void Cleanup() => _storage.Dispose(); + + [Benchmark] + public uint Create() + { + var created = LinksData.FillBenchmarked(_storage, _background, N); + return created[^1].Id; + } + + [Benchmark] + public uint Update() + { + var last = 0U; + for (var index = 0; index < _links.Length; index++) + { + _storage.Update(_links[index].Id, _updateSources[index], _background[0]); + last = _links[index].Id; + } + return last; + } + + [Benchmark] + public uint Delete() + { + var last = 0U; + foreach (var link in _links) + { + _storage.Delete(link.Id); + last = link.Id; + } + return last; + } + + [Benchmark] + public int EachAll() => _storage.QueryAll().Count; + + [Benchmark] + public ulong EachIdentity() + { + var checksum = 0UL; + foreach (var link in _links) + { + checksum += _storage.QueryById(link.Id)?.Id ?? 0; + } + return checksum; + } + + [Benchmark] + public int EachConcrete() + { + var count = 0; + foreach (var link in _links) + { + count += _storage.QueryBySourceTarget(link.Source, link.Target).Count; + } + return count; + } + + [Benchmark] + public int EachOutgoing() + { + var count = 0; + foreach (var link in _links) + { + count += _storage.QueryBySource(link.Source).Count; + } + return count; + } + + [Benchmark] + public int EachIncoming() + { + var count = 0; + foreach (var link in _links) + { + count += _storage.QueryByTarget(link.Target).Count; + } + return count; + } + + private void Setup(bool createBenchmarked) + { + _storage = LinksStorageFactory.Create(Variant); + _background = LinksData.FillBackground( + _storage, + EnvironmentValue("BACKGROUND_LINK_COUNT", 3000)); + _links = createBenchmarked + ? LinksData.FillBenchmarked(_storage, _background, N) + : Array.Empty(); + _updateSources = Array.Empty(); + } + + private static int EnvironmentValue(string name, int fallback) => + int.TryParse(Environment.GetEnvironmentVariable(name), out var value) && value > 0 + ? value + : fallback; +} diff --git a/csharp/LinksSelfTest.cs b/csharp/LinksSelfTest.cs new file mode 100644 index 0000000..4ef627d --- /dev/null +++ b/csharp/LinksSelfTest.cs @@ -0,0 +1,58 @@ +using System; +using System.Linq; +using Comparisons.SQLiteVSDoublets.Links; + +namespace Comparisons.SQLiteVSDoublets; + +public static class LinksSelfTest +{ + public static void Run() + { + foreach (var variant in Enum.GetValues()) + { + Check(variant); + Console.WriteLine($"{variant}: passed"); + } + } + + private static void Check(LinksVariant variant) + { + using var storage = LinksStorageFactory.Create(variant); + var background = LinksData.FillBackground(storage, 8); + var created = LinksData.FillBenchmarked(storage, background, 16); + Ensure(storage.Count == 24, $"{variant}: unexpected link count"); + Ensure(storage.QueryAll().Count == 24, $"{variant}: query all failed"); + + var sample = created[created.Length / 2]; + Ensure(storage.QueryById(sample.Id) == sample, $"{variant}: identity query failed"); + Ensure( + storage.QueryBySourceTarget(sample.Source, sample.Target).Contains(sample), + $"{variant}: concrete query failed"); + Ensure( + storage.QueryBySource(sample.Source).Contains(sample), + $"{variant}: outgoing query failed"); + Ensure( + storage.QueryByTarget(sample.Target).Contains(sample), + $"{variant}: incoming query failed"); + + var unused = storage.CreatePoint(); + storage.Update(sample.Id, unused, background[0]); + var updated = storage.QueryById(sample.Id); + Ensure( + updated is { Source: var source, Target: var target } + && source == unused + && target == background[0], + $"{variant}: update failed"); + + storage.Delete(sample.Id); + Ensure(storage.QueryById(sample.Id) is null, $"{variant}: delete failed"); + } + + private static void Ensure(bool condition, string message) + { + if (!condition) + { + throw new InvalidOperationException(message); + } + } +} diff --git a/csharp/Model/BlogPost.cs b/csharp/Model/BlogPost.cs index c4e45bf..16ad446 100644 --- a/csharp/Model/BlogPost.cs +++ b/csharp/Model/BlogPost.cs @@ -27,7 +27,7 @@ public class BlogPost /// /// [Required] - public string Title { get; set; } + public string Title { get; set; } = string.Empty; /// /// @@ -36,7 +36,7 @@ public class BlogPost /// /// [Required] - public string Content { get; set; } + public string Content { get; set; } = string.Empty; /// /// diff --git a/csharp/Program.cs b/csharp/Program.cs index 36418b6..7b376eb 100644 --- a/csharp/Program.cs +++ b/csharp/Program.cs @@ -1,63 +1,101 @@ using System; -using System.Linq; using System.Collections.Generic; -using Comparisons.SQLiteVSDoublets.SQLite; +using System.Linq; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Exporters.Json; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; using Comparisons.SQLiteVSDoublets.Doublets; using Comparisons.SQLiteVSDoublets.Model; -using BenchmarkDotNet.Running; +using Comparisons.SQLiteVSDoublets.SQLite; -namespace Comparisons.SQLiteVSDoublets +namespace Comparisons.SQLiteVSDoublets; + +internal static class Program { - class Program + private static void Main(string[] args) { - /// - /// - /// Main. - /// - /// - /// - static void Main() + if (args.Length == 1 && args[0] == "--self-test") { - BenchmarkRunner.Run(); - // Use this method if you need full control over the execution. - //Run(); + LinksSelfTest.Run(); + return; } - private static void Run() + + if (args.Length == 1 && args[0] == "--manual-object-test") { - const int numberOfTestRuns = 1; - const int numberOfRecordsPerTestRun = 1; - BlogPosts.GenerateData(numberOfRecordsPerTestRun); - var sqliteTestRuns = new List(); - var doubletsTestRuns = new List(); - for (int i = 0; i < numberOfTestRuns; i++) - { - var sqliteTestRun = new SQLiteTestRun("test.db"); - sqliteTestRun.Run(); - sqliteTestRuns.Add(sqliteTestRun); - var doubletsTestRun = new DoubletsTestRun("test.links"); - doubletsTestRun.Run(); - doubletsTestRuns.Add(doubletsTestRun); - } - Console.WriteLine("SQLite results:"); - var averageSqliteResults = GetResultsAverage(sqliteTestRuns); - Console.WriteLine(averageSqliteResults.ToString()); - Console.WriteLine("Doublets results:"); - var averageDoubletsResults = GetResultsAverage(doubletsTestRuns); - Console.WriteLine(averageDoubletsResults.ToString()); + RunManualObjectTest(); + return; } - private static TestRunResults GetResultsAverage(IEnumerable testRuns) + + if (args.Length == 0) { - return new TestRunResults() - { - PrepareTime = new TimeSpan((long)testRuns.Select(x => x.Results.PrepareTime.Ticks).Average()), - DbSizeAfterPrepare = (long)testRuns.Select(x => x.Results.DbSizeAfterPrepare).Average(), - ListCreationTime = new TimeSpan((long)testRuns.Select(x => x.Results.ListCreationTime.Ticks).Average()), - DbSizeAfterCreation = (long)testRuns.Select(x => x.Results.DbSizeAfterCreation).Average(), - ListReadingTime = new TimeSpan((long)testRuns.Select(x => x.Results.ListReadingTime.Ticks).Average()), - DbSizeAfterReading = (long)testRuns.Select(x => x.Results.DbSizeAfterReading).Average(), - ListDeletionTime = new TimeSpan((long)testRuns.Select(x => x.Results.ListDeletionTime.Ticks).Average()), - DbSizeAfterDeletion = (long)testRuns.Select(x => x.Results.DbSizeAfterDeletion).Average(), - }; + // Preserve the original entry point for the object-like benchmark. + BenchmarkRunner.Run(); + return; } + + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, BenchmarkConfig()); } + + private static void RunManualObjectTest() + { + const int numberOfTestRuns = 1; + const int numberOfRecordsPerTestRun = 1; + BlogPosts.GenerateData(numberOfRecordsPerTestRun); + var sqliteTestRuns = new List(); + var doubletsTestRuns = new List(); + for (var index = 0; index < numberOfTestRuns; index++) + { + var sqliteTestRun = new SQLiteTestRun("test.db"); + sqliteTestRun.Run(); + sqliteTestRuns.Add(sqliteTestRun); + var doubletsTestRun = new DoubletsTestRun("test.links"); + doubletsTestRun.Run(); + doubletsTestRuns.Add(doubletsTestRun); + } + Console.WriteLine("SQLite results:"); + Console.WriteLine(GetResultsAverage(sqliteTestRuns)); + Console.WriteLine("Doublets results:"); + Console.WriteLine(GetResultsAverage(doubletsTestRuns)); + } + + private static TestRunResults GetResultsAverage(IEnumerable testRuns) => new() + { + PrepareTime = new TimeSpan( + (long)testRuns.Select(run => run.Results.PrepareTime.Ticks).Average()), + DbSizeAfterPrepare = (long)testRuns.Select( + run => run.Results.DbSizeAfterPrepare).Average(), + ListCreationTime = new TimeSpan( + (long)testRuns.Select(run => run.Results.ListCreationTime.Ticks).Average()), + DbSizeAfterCreation = (long)testRuns.Select( + run => run.Results.DbSizeAfterCreation).Average(), + ListReadingTime = new TimeSpan( + (long)testRuns.Select(run => run.Results.ListReadingTime.Ticks).Average()), + DbSizeAfterReading = (long)testRuns.Select( + run => run.Results.DbSizeAfterReading).Average(), + ListDeletionTime = new TimeSpan( + (long)testRuns.Select(run => run.Results.ListDeletionTime.Ticks).Average()), + DbSizeAfterDeletion = (long)testRuns.Select( + run => run.Results.DbSizeAfterDeletion).Average(), + }; + + private static IConfig BenchmarkConfig() + { + var warmups = EnvironmentValue("BENCHMARK_WARMUP_COUNT", 1); + var iterations = EnvironmentValue("BENCHMARK_ITERATION_COUNT", 3); + var job = Job.Default + .WithWarmupCount(warmups) + .WithIterationCount(iterations); + return ManualConfig.Create(DefaultConfig.Instance) + .AddJob(job) + .AddExporter(JsonExporter.Full) + .WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(40)); + } + + private static int EnvironmentValue(string name, int fallback) => + int.TryParse(Environment.GetEnvironmentVariable(name), out var value) && value > 0 + ? value + : fallback; } diff --git a/csharp/global.json b/csharp/global.json new file mode 100644 index 0000000..4c4c3ae --- /dev/null +++ b/csharp/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "8.0.100", + "rollForward": "latestFeature", + "allowPrerelease": false + } +} diff --git a/csharp/out.py b/csharp/out.py new file mode 100644 index 0000000..2a9eac4 --- /dev/null +++ b/csharp/out.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Parse BenchmarkDotNet JSON and publish the C# link benchmark report.""" + +import argparse +import json +import os +import sys +from urllib.parse import parse_qs + +REPOSITORY_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPOSITORY_ROOT, "scripts")) + +import benchmark_report as report # noqa: E402 + + +METHODS = { + "Create": "create", + "Update": "update", + "Delete": "delete", + "EachAll": "query_all", + "EachIdentity": "query_by_id", + "EachConcrete": "query_by_source_target", + "EachOutgoing": "query_by_source", + "EachIncoming": "query_by_target", +} + + +def parse_document(document): + """Parse a BenchmarkDotNet full JSON export into shared report results.""" + results = report.empty_results(report.LINK_OPERATIONS) + for benchmark in document.get("Benchmarks", []): + operation = METHODS.get(benchmark.get("Method")) + parameters = parse_qs(benchmark.get("Parameters", "")) + variant = parameters.get("Variant", [None])[0] + mean = benchmark.get("Statistics", {}).get("Mean") + if operation in results and variant and isinstance(mean, (int, float)) and mean > 0: + results[operation][variant] = round(mean) + return results + + +def parse_results(path): + if not os.path.exists(path): + return report.empty_results(report.LINK_OPERATIONS) + with open(path, "r", encoding="utf-8") as handle: + return parse_document(json.load(handle)) + + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", help="BenchmarkDotNet *-report-full.json file") + parser.add_argument("--results", default="results.md") + parser.add_argument("--readme", action="append", default=[]) + parser.add_argument("--docs-dir") + parser.add_argument("--output-dir", default="") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv if argv is not None else sys.argv[1:]) + try: + results = parse_results(args.input) + except (json.JSONDecodeError, OSError) as error: + print(f"Cannot parse {args.input}: {error}") + print(report.report_input_excerpt(args.input)) + return 1 + + if not report.has_any_results(results): + print(f"No benchmark data found in {args.input}") + print(report.report_input_excerpt(args.input)) + return 1 + + missing = report.missing_measurements( + results, + report.LINK_OPERATIONS, + report.VARIANTS, + ) + if missing: + print("Benchmark output is incomplete; missing:") + print("\n".join(missing)) + return 1 + + provenance = report.build_provenance(language="C#", object_count=False) + section = report.render_results_section( + results, + provenance, + operations=report.LINK_OPERATIONS, + ) + section += ( + "\n\n![C# benchmark comparison](docs/benchmarks/bench_csharp.png)" + "\n\n![C# benchmark comparison, logarithmic scale]" + "(docs/benchmarks/bench_csharp_log_scale.png)" + ) + with open(args.results, "w", encoding="utf-8") as handle: + handle.write(section + "\n") + print(report.format_results_table(results, operations=report.LINK_OPERATIONS)) + print(f"Generated {args.results}") + + charts = report.generate_charts( + results, + "bench_csharp", + "Benchmark Comparison: SQLite vs Doublets (C#)", + args.output_dir, + operations=report.LINK_OPERATIONS, + ) + report.copy_charts(charts, args.docs_dir) + + for readme in args.readme: + changed = report.update_markers( + readme, + section, + report.CSHARP_START_MARKER, + report.CSHARP_END_MARKER, + ) + print(f"{'Updated' if changed else 'No changes needed in'} {readme}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/csharp/test_out.py b/csharp/test_out.py new file mode 100644 index 0000000..0311234 --- /dev/null +++ b/csharp/test_out.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Regression tests for the BenchmarkDotNet JSON report pipeline.""" + +import json +import os +import tempfile +import unittest + +import out + + +def complete_document(): + benchmarks = [] + mean = 1_000.4 + for method in out.METHODS: + for variant, _label, _color in out.report.VARIANTS: + benchmarks.append( + { + "Method": method, + "Parameters": f"N=10&Variant={variant}", + "Statistics": {"Mean": mean}, + } + ) + mean += 100 + return {"Title": "test", "Benchmarks": benchmarks} + + +class ParserTests(unittest.TestCase): + def test_parses_every_operation_and_variant(self): + results = out.parse_document(complete_document()) + + self.assertFalse( + out.report.missing_measurements( + results, + out.report.LINK_OPERATIONS, + out.report.VARIANTS, + ) + ) + self.assertEqual(results["create"]["Doublets_United_Volatile"], 1_000) + + def test_ignores_legacy_object_benchmarks(self): + document = complete_document() + document["Benchmarks"].append( + { + "Method": "SQLite", + "Parameters": "N=10", + "Statistics": {"Mean": 42}, + } + ) + + self.assertEqual(out.parse_document(document), out.parse_document(complete_document())) + + def test_ignores_failed_benchmarks_without_statistics(self): + results = out.parse_document( + { + "Benchmarks": [ + { + "Method": "Create", + "Parameters": "N=10&Variant=SQLite_Memory", + } + ] + } + ) + + self.assertFalse(out.report.has_any_results(results)) + + +class MainTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.input = os.path.join(self.temporary.name, "results.json") + self.results = os.path.join(self.temporary.name, "results.md") + self.readme = os.path.join(self.temporary.name, "README.md") + + def write_json(self, document): + with open(self.input, "w", encoding="utf-8") as handle: + json.dump(document, handle) + + def test_generates_results_and_updates_readme(self): + self.write_json(complete_document()) + with open(self.readme, "w", encoding="utf-8") as handle: + handle.write( + f"{out.report.CSHARP_START_MARKER}\nold\n" + f"{out.report.CSHARP_END_MARKER}\n" + ) + + result = out.main( + [ + self.input, + "--results", + self.results, + "--readme", + self.readme, + "--output-dir", + self.temporary.name, + ] + ) + + self.assertEqual(result, 0) + with open(self.results, "r", encoding="utf-8") as handle: + self.assertIn("Each Concrete", handle.read()) + with open(self.readme, "r", encoding="utf-8") as handle: + self.assertIn("SQLite File", handle.read()) + + def test_rejects_incomplete_results(self): + document = complete_document() + document["Benchmarks"].pop() + self.write_json(document) + + self.assertEqual(out.main([self.input, "--results", self.results]), 1) + self.assertFalse(os.path.exists(self.results)) + + def test_rejects_malformed_json(self): + with open(self.input, "w", encoding="utf-8") as handle: + handle.write("not json") + + self.assertEqual(out.main([self.input, "--results", self.results]), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/rust/benches/bench.rs b/rust/benches/bench.rs index 7d58782..b5e0f47 100644 --- a/rust/benches/bench.rs +++ b/rust/benches/bench.rs @@ -1,391 +1,29 @@ -//! Benchmark suite comparing SQLite vs Doublets performance +//! SQLite versus Doublets benchmark suite. //! -//! This benchmark measures basic CRUD operations with links on both storage systems. +//! Preparation is performed by `benchmarks::measure` outside the measured +//! region. Every operation is run against all six storage variants. -use criterion::{criterion_group, criterion_main, Criterion, Throughput}; -use sqlite_vs_doublets::{ - benched::{DoubletsSplitVolatileBenched, DoubletsUnitedVolatileBenched, SqliteMemoryBenched}, - Benched, Links, BACKGROUND_LINK_COUNT, BENCHMARK_LINK_COUNT, -}; -use std::time::Duration; +#![feature(allocator_api)] -/// Run create operations benchmark -fn bench_create(c: &mut Criterion) { - let mut group = c.benchmark_group("create"); - let link_count = *BENCHMARK_LINK_COUNT; - group.throughput(Throughput::Elements(link_count as u64)); +mod benchmarks; - group.bench_function("SQLite_Memory", |b| { - let mut benched = SqliteMemoryBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - // Create background links - for _ in 0..*BACKGROUND_LINK_COUNT { - fork.create_point(); - } - // Benchmark operation - for _ in 0..link_count { - fork.create_point(); - } - }); - }); - - group.bench_function("Doublets_United_Volatile", |b| { - let mut benched = DoubletsUnitedVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - for _ in 0..*BACKGROUND_LINK_COUNT { - fork.create_point(); - } - for _ in 0..link_count { - fork.create_point(); - } - }); - }); - - group.bench_function("Doublets_Split_Volatile", |b| { - let mut benched = DoubletsSplitVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - for _ in 0..*BACKGROUND_LINK_COUNT { - fork.create_point(); - } - for _ in 0..link_count { - fork.create_point(); - } - }); - }); - - group.finish(); -} - -/// Run delete operations benchmark -fn bench_delete(c: &mut Criterion) { - let mut group = c.benchmark_group("delete"); - let link_count = *BENCHMARK_LINK_COUNT; - group.throughput(Throughput::Elements(link_count as u64)); - - group.bench_function("SQLite_Memory", |b| { - let mut benched = SqliteMemoryBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - // Create links to delete - let mut ids = Vec::with_capacity(link_count); - for _ in 0..link_count { - ids.push(fork.create_point()); - } - // Benchmark delete - for id in ids { - fork.delete(id); - } - }); - }); - - group.bench_function("Doublets_United_Volatile", |b| { - let mut benched = DoubletsUnitedVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - let mut ids = Vec::with_capacity(link_count); - for _ in 0..link_count { - ids.push(fork.create_point()); - } - for id in ids { - fork.delete(id); - } - }); - }); - - group.bench_function("Doublets_Split_Volatile", |b| { - let mut benched = DoubletsSplitVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - let mut ids = Vec::with_capacity(link_count); - for _ in 0..link_count { - ids.push(fork.create_point()); - } - for id in ids { - fork.delete(id); - } - }); - }); - - group.finish(); -} - -/// Run update operations benchmark -fn bench_update(c: &mut Criterion) { - let mut group = c.benchmark_group("update"); - let link_count = *BENCHMARK_LINK_COUNT; - group.throughput(Throughput::Elements(link_count as u64)); - - group.bench_function("SQLite_Memory", |b| { - let mut benched = SqliteMemoryBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - // Create links to update - let mut ids = Vec::with_capacity(link_count); - for _ in 0..link_count { - ids.push(fork.create_point()); - } - // Benchmark update - for (i, id) in ids.iter().enumerate() { - fork.update(*id, (i + 1) as u64, (i + 2) as u64); - } - }); - }); - - group.bench_function("Doublets_United_Volatile", |b| { - let mut benched = DoubletsUnitedVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - let mut ids = Vec::with_capacity(link_count); - for _ in 0..link_count { - ids.push(fork.create_point()); - } - for (i, id) in ids.iter().enumerate() { - fork.update(*id, (i + 1) as u64, (i + 2) as u64); - } - }); - }); - - group.bench_function("Doublets_Split_Volatile", |b| { - let mut benched = DoubletsSplitVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - let mut ids = Vec::with_capacity(link_count); - for _ in 0..link_count { - ids.push(fork.create_point()); - } - for (i, id) in ids.iter().enumerate() { - fork.update(*id, (i + 1) as u64, (i + 2) as u64); - } - }); - }); - - group.finish(); -} - -/// Run query all benchmark -fn bench_query_all(c: &mut Criterion) { - let mut group = c.benchmark_group("query_all"); - let link_count = *BENCHMARK_LINK_COUNT; - group.throughput(Throughput::Elements(link_count as u64)); - - group.bench_function("SQLite_Memory", |b| { - let mut benched = SqliteMemoryBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - // Create links to query - for _ in 0..link_count { - fork.create_point(); - } - // Benchmark query - let _ = fork.query_all(); - }); - }); - - group.bench_function("Doublets_United_Volatile", |b| { - let mut benched = DoubletsUnitedVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - for _ in 0..link_count { - fork.create_point(); - } - let _ = fork.query_all(); - }); - }); - - group.bench_function("Doublets_Split_Volatile", |b| { - let mut benched = DoubletsSplitVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - for _ in 0..link_count { - fork.create_point(); - } - let _ = fork.query_all(); - }); - }); - - group.finish(); -} - -/// Run query by ID benchmark -fn bench_query_by_id(c: &mut Criterion) { - let mut group = c.benchmark_group("query_by_id"); - let link_count = *BENCHMARK_LINK_COUNT; - group.throughput(Throughput::Elements(link_count as u64)); - - group.bench_function("SQLite_Memory", |b| { - let mut benched = SqliteMemoryBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - // Create links - let mut ids = Vec::with_capacity(link_count); - for _ in 0..link_count { - ids.push(fork.create_point()); - } - // Benchmark query - for id in &ids { - let _ = fork.query_by_id(*id); - } - }); - }); - - group.bench_function("Doublets_United_Volatile", |b| { - let mut benched = DoubletsUnitedVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - let mut ids = Vec::with_capacity(link_count); - for _ in 0..link_count { - ids.push(fork.create_point()); - } - for id in &ids { - let _ = fork.query_by_id(*id); - } - }); - }); - - group.bench_function("Doublets_Split_Volatile", |b| { - let mut benched = DoubletsSplitVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - let mut ids = Vec::with_capacity(link_count); - for _ in 0..link_count { - ids.push(fork.create_point()); - } - for id in &ids { - let _ = fork.query_by_id(*id); - } - }); - }); - - group.finish(); -} - -/// Run query by source benchmark -fn bench_query_by_source(c: &mut Criterion) { - let mut group = c.benchmark_group("query_by_source"); - let link_count = *BENCHMARK_LINK_COUNT; - group.throughput(Throughput::Elements(link_count as u64)); - - group.bench_function("SQLite_Memory", |b| { - let mut benched = SqliteMemoryBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - // Create links with various sources - for i in 0..link_count { - let id = fork.create_point(); - fork.update(id, (i % 10 + 1) as u64, id); - } - // Benchmark query - for i in 1..=10 { - let _ = fork.query_by_source(i as u64); - } - }); - }); - - group.bench_function("Doublets_United_Volatile", |b| { - let mut benched = DoubletsUnitedVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - for i in 0..link_count { - let id = fork.create_point(); - fork.update(id, (i % 10 + 1) as u64, id); - } - for i in 1..=10 { - let _ = fork.query_by_source(i as u64); - } - }); - }); - - group.bench_function("Doublets_Split_Volatile", |b| { - let mut benched = DoubletsSplitVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - for i in 0..link_count { - let id = fork.create_point(); - fork.update(id, (i % 10 + 1) as u64, id); - } - for i in 1..=10 { - let _ = fork.query_by_source(i as u64); - } - }); - }); - - group.finish(); -} - -/// Run query by target benchmark -fn bench_query_by_target(c: &mut Criterion) { - let mut group = c.benchmark_group("query_by_target"); - let link_count = *BENCHMARK_LINK_COUNT; - group.throughput(Throughput::Elements(link_count as u64)); - - group.bench_function("SQLite_Memory", |b| { - let mut benched = SqliteMemoryBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - // Create links with various targets - for i in 0..link_count { - let id = fork.create_point(); - fork.update(id, id, (i % 10 + 1) as u64); - } - // Benchmark query - for i in 1..=10 { - let _ = fork.query_by_target(i as u64); - } - }); - }); - - group.bench_function("Doublets_United_Volatile", |b| { - let mut benched = DoubletsUnitedVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - for i in 0..link_count { - let id = fork.create_point(); - fork.update(id, id, (i % 10 + 1) as u64); - } - for i in 1..=10 { - let _ = fork.query_by_target(i as u64); - } - }); - }); - - group.bench_function("Doublets_Split_Volatile", |b| { - let mut benched = DoubletsSplitVolatileBenched::setup(()); - b.iter(|| { - let mut fork = benched.fork(); - for i in 0..link_count { - let id = fork.create_point(); - fork.update(id, id, (i % 10 + 1) as u64); - } - for i in 1..=10 { - let _ = fork.query_by_target(i as u64); - } - }); - }); - - group.finish(); -} - -/// Configure criterion for faster CI runs -fn configure_criterion() -> Criterion { - Criterion::default() - .sample_size(10) - .measurement_time(Duration::from_secs(1)) - .warm_up_time(Duration::from_millis(500)) -} +use criterion::{criterion_group, criterion_main}; criterion_group! { name = benches; - config = configure_criterion(); - targets = bench_create, - bench_delete, - bench_update, - bench_query_all, - bench_query_by_id, - bench_query_by_source, - bench_query_by_target + config = benchmarks::configure_criterion(); + targets = + benchmarks::links::create, + benchmarks::links::update, + benchmarks::links::delete, + benchmarks::each::all, + benchmarks::each::identity, + benchmarks::each::concrete, + benchmarks::each::outgoing, + benchmarks::each::incoming, + benchmarks::objects::create, + benchmarks::objects::read, + benchmarks::objects::delete, } criterion_main!(benches); diff --git a/rust/benches/benchmarks/each.rs b/rust/benches/benchmarks/each.rs new file mode 100644 index 0000000..626ac20 --- /dev/null +++ b/rust/benches/benchmarks/each.rs @@ -0,0 +1,51 @@ +//! Link enumeration and restriction benchmarks. + +use criterion::Criterion; +use sqlite_vs_doublets::{ + fill_background, fill_benchmarked, Links, BACKGROUND_LINK_COUNT, BENCHMARK_LINK_COUNT, +}; + +macro_rules! query_benchmark { + ($function:ident, $group:literal, $query:expr) => { + pub fn $function(criterion: &mut Criterion) { + benchmark_operation!( + criterion, + $group, + *BENCHMARK_LINK_COUNT, + |storage| { + let background = fill_background(storage, *BACKGROUND_LINK_COUNT); + fill_benchmarked(storage, &background, *BENCHMARK_LINK_COUNT) + }, + $query, + ); + } + }; +} + +query_benchmark!(all, "query_all", |storage, _created| { + criterion::black_box(storage.query_all()); +}); + +query_benchmark!(identity, "query_by_id", |storage, created| { + for link in created { + criterion::black_box(storage.query_by_id(link.id)); + } +}); + +query_benchmark!(concrete, "query_by_source_target", |storage, created| { + for link in created { + criterion::black_box(storage.query_by_source_target(link.source, link.target)); + } +}); + +query_benchmark!(outgoing, "query_by_source", |storage, created| { + for link in created { + criterion::black_box(storage.query_by_source(link.source)); + } +}); + +query_benchmark!(incoming, "query_by_target", |storage, created| { + for link in created { + criterion::black_box(storage.query_by_target(link.target)); + } +}); diff --git a/rust/benches/benchmarks/links.rs b/rust/benches/benchmarks/links.rs new file mode 100644 index 0000000..6fe6815 --- /dev/null +++ b/rust/benches/benchmarks/links.rs @@ -0,0 +1,57 @@ +//! Create, update and delete benchmarks. + +use criterion::Criterion; +use sqlite_vs_doublets::{ + fill_background, fill_benchmarked, Links, BACKGROUND_LINK_COUNT, BENCHMARK_LINK_COUNT, +}; + +pub fn create(criterion: &mut Criterion) { + benchmark_operation!( + criterion, + "create", + *BENCHMARK_LINK_COUNT, + |storage| fill_background(storage, *BACKGROUND_LINK_COUNT), + |storage, background| { + let created = fill_benchmarked(storage, &background, *BENCHMARK_LINK_COUNT); + criterion::black_box(created); + }, + ); +} + +pub fn update(criterion: &mut Criterion) { + benchmark_operation!( + criterion, + "update", + *BENCHMARK_LINK_COUNT, + |storage| { + let background = fill_background(storage, *BACKGROUND_LINK_COUNT); + let created = fill_benchmarked(storage, &background, *BENCHMARK_LINK_COUNT); + let update_sources = fill_background(storage, *BENCHMARK_LINK_COUNT); + (background, created, update_sources) + }, + |storage, (background, created, update_sources)| { + for (index, link) in created.into_iter().enumerate() { + // A dedicated source makes the new doublet unique. Updating to + // an existing pair is not a valid Doublets operation. + storage.update(link.id, update_sources[index], background[0]); + } + }, + ); +} + +pub fn delete(criterion: &mut Criterion) { + benchmark_operation!( + criterion, + "delete", + *BENCHMARK_LINK_COUNT, + |storage| { + let background = fill_background(storage, *BACKGROUND_LINK_COUNT); + fill_benchmarked(storage, &background, *BENCHMARK_LINK_COUNT) + }, + |storage, created| { + for link in created { + storage.delete(link.id); + } + }, + ); +} diff --git a/rust/benches/benchmarks/mod.rs b/rust/benches/benchmarks/mod.rs new file mode 100644 index 0000000..e884829 --- /dev/null +++ b/rust/benches/benchmarks/mod.rs @@ -0,0 +1,157 @@ +//! Shared infrastructure of the benchmark suite. +//! +//! Every operation is measured on the same six subjects, so the modules below +//! only describe *what* is prepared and *what* is measured, and the +//! [`benchmark_operation`] macro takes care of running it everywhere: +//! +//! | Subject | Database | Memory | +//! |-------------------------------|----------|---------------| +//! | `SQLite_Memory` | SQLite | volatile | +//! | `SQLite_File` | SQLite | non-volatile | +//! | `Doublets_United_Volatile` | Doublets | volatile | +//! | `Doublets_United_NonVolatile` | Doublets | non-volatile | +//! | `Doublets_Split_Volatile` | Doublets | volatile | +//! | `Doublets_Split_NonVolatile` | Doublets | non-volatile | +//! +//! The data of an iteration is prepared *outside* of the measured region with +//! [`Criterion::iter_custom`](criterion::Bencher::iter_custom), so that the +//! reported time is the time of the operation itself and not the time of +//! filling the storage. + +use criterion::{Bencher, Criterion}; +use sqlite_vs_doublets::{Benched, Links, Objects}; +use std::{ + env, + ops::DerefMut, + time::{Duration, Instant}, +}; + +/// Prepares the storage of every iteration and measures `operation` on it. +/// +/// `prepare` runs on an empty (forked) storage and its result is handed to +/// `operation`, which is the only part of the iteration that is timed. Dropping +/// the fork empties the storage again for the next iteration. +pub fn measure(bencher: &mut Bencher<'_>, mut prepare: P, mut operation: O) +where + B: Benched + DerefMut, + B::Target: Links + Objects + Sized, + P: FnMut(&mut B::Target) -> S, + O: FnMut(&mut B::Target, S), +{ + let mut benched = B::setup(()); + bencher.iter_custom(|iterations| { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + let mut fork = benched.fork(); + let storage = &mut **fork; + let prepared = prepare(storage); + + let start = Instant::now(); + operation(storage, prepared); + elapsed += start.elapsed(); + } + elapsed + }); +} + +/// Measures one operation on one subject. +macro_rules! benchmark_subject { + ($group:expr, $count:expr, $name:literal, $subject:ty, $prepare:expr, $operation:expr) => { + $group.bench_with_input( + criterion::BenchmarkId::new($name, $count), + &$count, + |bencher, _| { + crate::benchmarks::measure::<$subject, _, _, _>(bencher, $prepare, $operation) + }, + ); + }; +} + +/// Measures one operation on every subject of the comparison. +/// +/// `$prepare` and `$operation` are repeated for each subject, so that both +/// closures are inferred against the storage type of that subject. +macro_rules! benchmark_operation { + ($criterion:expr, $name:literal, $count:expr, $prepare:expr, $operation:expr $(,)?) => {{ + let count = $count; + let mut group = $criterion.benchmark_group($name); + group.throughput(criterion::Throughput::Elements(count as u64)); + benchmark_subject!( + group, + count, + "SQLite_Memory", + sqlite_vs_doublets::benched::SqliteMemoryBenched, + $prepare, + $operation + ); + benchmark_subject!( + group, + count, + "SQLite_File", + sqlite_vs_doublets::benched::SqliteFileBenched, + $prepare, + $operation + ); + benchmark_subject!( + group, + count, + "Doublets_United_Volatile", + sqlite_vs_doublets::benched::DoubletsUnitedVolatileBenched, + $prepare, + $operation + ); + benchmark_subject!( + group, + count, + "Doublets_United_NonVolatile", + sqlite_vs_doublets::benched::DoubletsUnitedNonVolatileBenched, + $prepare, + $operation + ); + benchmark_subject!( + group, + count, + "Doublets_Split_Volatile", + sqlite_vs_doublets::benched::DoubletsSplitVolatileBenched, + $prepare, + $operation + ); + benchmark_subject!( + group, + count, + "Doublets_Split_NonVolatile", + sqlite_vs_doublets::benched::DoubletsSplitNonVolatileBenched, + $prepare, + $operation + ); + group.finish(); + }}; +} + +pub mod each; +pub mod links; +pub mod objects; + +fn env_value(name: &str, default: T) -> T { + env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +/// Criterion configuration of the suite. +/// +/// The defaults keep a pull request run short; the full run of the `main` +/// branch raises them through the environment. +pub fn configure_criterion() -> Criterion { + Criterion::default() + .sample_size(env_value("BENCHMARK_SAMPLE_SIZE", 10usize)) + .measurement_time(Duration::from_secs(env_value( + "BENCHMARK_MEASUREMENT_SECONDS", + 1u64, + ))) + .warm_up_time(Duration::from_millis(env_value( + "BENCHMARK_WARM_UP_MILLISECONDS", + 500u64, + ))) +} diff --git a/rust/benches/benchmarks/objects.rs b/rust/benches/benchmarks/objects.rs new file mode 100644 index 0000000..c3b29a8 --- /dev/null +++ b/rust/benches/benchmarks/objects.rs @@ -0,0 +1,44 @@ +//! Object-like (blog post) benchmarks. + +use criterion::Criterion; +use sqlite_vs_doublets::{generate_posts, Objects, BENCHMARK_OBJECT_COUNT}; + +pub fn create(criterion: &mut Criterion) { + benchmark_operation!( + criterion, + "objects_create", + *BENCHMARK_OBJECT_COUNT, + |_storage| generate_posts(*BENCHMARK_OBJECT_COUNT), + |storage, posts| { + criterion::black_box(storage.create_posts(&posts)); + }, + ); +} + +pub fn read(criterion: &mut Criterion) { + benchmark_operation!( + criterion, + "objects_read", + *BENCHMARK_OBJECT_COUNT, + |storage| { + let posts = generate_posts(*BENCHMARK_OBJECT_COUNT); + storage.create_posts(&posts); + }, + |storage, _prepared| { + criterion::black_box(storage.read_posts()); + }, + ); +} + +pub fn delete(criterion: &mut Criterion) { + benchmark_operation!( + criterion, + "objects_delete", + *BENCHMARK_OBJECT_COUNT, + |storage| { + let posts = generate_posts(*BENCHMARK_OBJECT_COUNT); + storage.create_posts(&posts); + }, + |storage, _prepared| storage.delete_posts(), + ); +} diff --git a/rust/examples/inspect_storage.rs b/rust/examples/inspect_storage.rs new file mode 100644 index 0000000..0e87403 --- /dev/null +++ b/rust/examples/inspect_storage.rs @@ -0,0 +1,76 @@ +//! Prints what every benchmarked storage contains after a benchmark iteration. +//! +//! Useful to check by hand that all six subjects of the benchmarks behave the +//! same way — the assumption the whole comparison is based on. +//! +//! ```text +//! cargo run --release --example inspect_storage +//! ``` + +#![feature(allocator_api)] + +use sqlite_vs_doublets::benched::{ + DoubletsSplitNonVolatileBenched, DoubletsSplitVolatileBenched, + DoubletsUnitedNonVolatileBenched, DoubletsUnitedVolatileBenched, SqliteFileBenched, + SqliteMemoryBenched, +}; +use sqlite_vs_doublets::{ + fill_background, fill_benchmarked, generate_posts, Benched, Links, Objects, +}; +use std::ops::DerefMut; + +const BACKGROUND: usize = 100; +const BENCHMARKED: usize = 100; +const POSTS: usize = 10; + +fn inspect(name: &str, subject: &mut B) +where + B: Benched + DerefMut, + B::Target: Links + Objects + Sized, +{ + let mut fork = subject.fork(); + let storage = &mut **fork; + + let background = fill_background(storage, BACKGROUND); + let created = fill_benchmarked(storage, &background, BENCHMARKED); + let sample = created[created.len() / 2]; + let posts = generate_posts(POSTS); + storage.create_posts(&posts); + + println!("{name}:"); + println!(" links {}", storage.count()); + println!( + " by identity {:?}", + storage.query_by_id(sample.id) + ); + let outgoing = storage.query_by_source(sample.source).len(); + println!(" outgoing of the sample {outgoing}"); + let incoming = storage.query_by_target(sample.target).len(); + println!(" incoming of the sample {incoming}"); + println!(" blog posts {}", storage.count_posts()); + println!( + " first blog post {:?}", + storage.read_posts().first() + ); +} + +fn main() { + inspect("SQLite_Memory", &mut SqliteMemoryBenched::setup(())); + inspect("SQLite_File", &mut SqliteFileBenched::setup(())); + inspect( + "Doublets_United_Volatile", + &mut DoubletsUnitedVolatileBenched::setup(()), + ); + inspect( + "Doublets_United_NonVolatile", + &mut DoubletsUnitedNonVolatileBenched::setup(()), + ); + inspect( + "Doublets_Split_Volatile", + &mut DoubletsSplitVolatileBenched::setup(()), + ); + inspect( + "Doublets_Split_NonVolatile", + &mut DoubletsSplitNonVolatileBenched::setup(()), + ); +} diff --git a/rust/out.py b/rust/out.py new file mode 100644 index 0000000..440e743 --- /dev/null +++ b/rust/out.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Parse Criterion bencher output and publish the Rust benchmark report.""" + +import argparse +import os +import re +import sys + +REPOSITORY_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPOSITORY_ROOT, "scripts")) + +import benchmark_report as report # noqa: E402 + + +# Criterion writes the beginning and end of a bencher record separately. Error +# text can therefore occur between ``test ...`` and ``bench: ...``. Stop at the +# next test record so an aborted benchmark cannot borrow its successor's value. +BENCHER_PATTERN = re.compile( + r"test\s+(\w+)/(\w+)/(\d+)\s+\.\.\.\s*" + r"(?:(?!\btest\s)[\s\S])*?" + r"bench:\s+([\d,]+)\s+ns/iter" +) + + +def parse_text(content): + """Parse bencher text into ``{operation: {variant: ns_per_iteration}}``.""" + results = report.empty_results() + for match in BENCHER_PATTERN.finditer(content): + operation, variant, _size, nanoseconds = match.groups() + if operation in results: + results[operation][variant] = int(nanoseconds.replace(",", "")) + return results + + +def parse_results(path): + if not os.path.exists(path): + return report.empty_results() + with open(path, "r", encoding="utf-8") as handle: + return parse_text(handle.read()) + + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", nargs="?", default="out.txt") + parser.add_argument("--results", default="results.md") + parser.add_argument("--readme", action="append", default=[]) + parser.add_argument("--docs-dir") + parser.add_argument("--output-dir", default="") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv if argv is not None else sys.argv[1:]) + results = parse_results(args.input) + if not report.has_any_results(results): + print(f"No benchmark data found in {args.input}") + print(report.report_input_excerpt(args.input)) + return 1 + + missing = report.missing_measurements(results) + if missing: + print("Benchmark output is incomplete; missing:") + print("\n".join(missing)) + return 1 + + provenance = report.build_provenance(language="Rust") + section = report.render_results_section(results, provenance) + section += ( + "\n\n![Rust benchmark comparison](docs/benchmarks/bench_rust.png)" + "\n\n![Rust benchmark comparison, logarithmic scale]" + "(docs/benchmarks/bench_rust_log_scale.png)" + ) + with open(args.results, "w", encoding="utf-8") as handle: + handle.write(section + "\n") + print(report.format_results_table(results)) + print(f"Generated {args.results}") + + charts = report.generate_charts( + results, + "bench_rust", + "Benchmark Comparison: SQLite vs Doublets (Rust)", + args.output_dir, + ) + report.copy_charts(charts, args.docs_dir) + + for readme in args.readme: + changed = report.update_markers( + readme, + section, + report.RUST_START_MARKER, + report.RUST_END_MARKER, + ) + print(f"{'Updated' if changed else 'No changes needed in'} {readme}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/rust/src/benched/doublets_benched.rs b/rust/src/benched/doublets_benched.rs index a1613f2..8777b4e 100644 --- a/rust/src/benched/doublets_benched.rs +++ b/rust/src/benched/doublets_benched.rs @@ -1,10 +1,12 @@ //! Benched implementations for Doublets -use crate::{Benched, Fork, Links}; use crate::doublets_impl::{ - create_split_volatile, create_united_volatile, DoubletsLinks, DoubletsSplitVolatile, - DoubletsUnitedVolatile, + create_split_non_volatile, create_split_volatile, create_united_non_volatile, + create_united_volatile, DoubletsLinks, DoubletsSplitNonVolatile, DoubletsSplitVolatile, + DoubletsUnitedNonVolatile, DoubletsUnitedVolatile, }; +use crate::{temp_path, Benched, Fork, Links}; +use std::path::PathBuf; /// Benched implementation for Doublets United (unit) store with volatile storage pub struct DoubletsUnitedVolatileBenched { @@ -79,3 +81,110 @@ impl std::ops::DerefMut for DoubletsSplitVolatileBenched { &mut self.links } } + +/// Benched implementation for Doublets United (unit) store with non-volatile storage +/// +/// The links are stored in a memory-mapped file inside the temporary directory +/// of the machine, which is removed when the benchmark subject is dropped. +pub struct DoubletsUnitedNonVolatileBenched { + links: DoubletsLinks, + path: PathBuf, +} + +impl Benched for DoubletsUnitedNonVolatileBenched { + type Builder = (); + + fn setup(_builder: Self::Builder) -> Self { + let path = temp_path("united.links"); + // Start from an empty store even if a previous run left a file. + let _ = std::fs::remove_file(&path); + Self { + links: create_united_non_volatile(&path), + path, + } + } + + fn fork(&mut self) -> Fork { + Fork::new(self) + } + + unsafe fn unfork(&mut self) { + self.links.delete_all(); + } +} + +impl std::ops::Deref for DoubletsUnitedNonVolatileBenched { + type Target = DoubletsLinks; + + fn deref(&self) -> &Self::Target { + &self.links + } +} + +impl std::ops::DerefMut for DoubletsUnitedNonVolatileBenched { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.links + } +} + +impl Drop for DoubletsUnitedNonVolatileBenched { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} + +/// Benched implementation for Doublets Split store with non-volatile storage +/// +/// The data and the index are stored in two memory-mapped files inside the +/// temporary directory of the machine, both removed when the benchmark subject +/// is dropped. +pub struct DoubletsSplitNonVolatileBenched { + links: DoubletsLinks, + data_path: PathBuf, + index_path: PathBuf, +} + +impl Benched for DoubletsSplitNonVolatileBenched { + type Builder = (); + + fn setup(_builder: Self::Builder) -> Self { + let data_path = temp_path("split.data.links"); + let index_path = temp_path("split.index.links"); + let _ = std::fs::remove_file(&data_path); + let _ = std::fs::remove_file(&index_path); + Self { + links: create_split_non_volatile(&data_path, &index_path), + data_path, + index_path, + } + } + + fn fork(&mut self) -> Fork { + Fork::new(self) + } + + unsafe fn unfork(&mut self) { + self.links.delete_all(); + } +} + +impl std::ops::Deref for DoubletsSplitNonVolatileBenched { + type Target = DoubletsLinks; + + fn deref(&self) -> &Self::Target { + &self.links + } +} + +impl std::ops::DerefMut for DoubletsSplitNonVolatileBenched { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.links + } +} + +impl Drop for DoubletsSplitNonVolatileBenched { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.data_path); + let _ = std::fs::remove_file(&self.index_path); + } +} diff --git a/rust/src/benched/sqlite_benched.rs b/rust/src/benched/sqlite_benched.rs index bc9caa6..6e1b8c1 100644 --- a/rust/src/benched/sqlite_benched.rs +++ b/rust/src/benched/sqlite_benched.rs @@ -1,7 +1,8 @@ //! Benched implementation for SQLite -use crate::{Benched, Fork}; use crate::sqlite_impl::SqliteLinks; +use crate::{temp_path, Benched, Fork}; +use std::path::PathBuf; /// Benched implementation for SQLite with in-memory database pub struct SqliteMemoryBenched { @@ -39,3 +40,54 @@ impl std::ops::DerefMut for SqliteMemoryBenched { &mut self.links } } + +/// Benched implementation for SQLite with a file based database +/// +/// The file lives in the temporary directory of the machine and is removed when +/// the benchmark subject is dropped. +pub struct SqliteFileBenched { + links: SqliteLinks, + path: PathBuf, +} + +impl Benched for SqliteFileBenched { + type Builder = (); + + fn setup(_builder: Self::Builder) -> Self { + let path = temp_path("sqlite.db"); + // Start from an empty database even if a previous run left a file. + let _ = std::fs::remove_file(&path); + Self { + links: SqliteLinks::new_file(&path), + path, + } + } + + fn fork(&mut self) -> Fork { + Fork::new(self) + } + + unsafe fn unfork(&mut self) { + self.links.reset(); + } +} + +impl std::ops::Deref for SqliteFileBenched { + type Target = SqliteLinks; + + fn deref(&self) -> &Self::Target { + &self.links + } +} + +impl std::ops::DerefMut for SqliteFileBenched { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.links + } +} + +impl Drop for SqliteFileBenched { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} diff --git a/rust/src/doublets_impl.rs b/rust/src/doublets_impl.rs index 9cb2760..458f0e8 100644 --- a/rust/src/doublets_impl.rs +++ b/rust/src/doublets_impl.rs @@ -1,13 +1,13 @@ //! Doublets storage implementation for links -use crate::{Link, Links}; +use crate::{BlogPost, Link, Links, Objects}; use doublets::{ - mem::Alloc, + mem::{Alloc, FileMapped}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, Doublets, DoubletsExt, }; -use std::alloc::Global; +use std::{alloc::Global, collections::HashMap, fs::OpenOptions, path::Path}; /// Type alias for Doublets united (unit) store with volatile (in-memory) storage. /// Each link is stored as a contiguous unit containing (id, source, target). @@ -18,14 +18,73 @@ pub type DoubletsUnitedVolatile = unit::Store, G pub type DoubletsSplitVolatile = split::Store, Global>, Alloc, Global>>; +/// Type alias for Doublets united (unit) store with non-volatile (file-mapped) storage. +/// Same layout as [`DoubletsUnitedVolatile`], but the links live in a memory-mapped file. +pub type DoubletsUnitedNonVolatile = unit::Store>>; + +/// Type alias for Doublets split store with non-volatile (file-mapped) storage. +/// Same layout as [`DoubletsSplitVolatile`], but data and index live in two memory-mapped files. +pub type DoubletsSplitNonVolatile = + split::Store>, FileMapped>>; + +/// Opens (creating it when missing) a file and maps it as Doublets memory. +pub fn map_file>(path: P) -> FileMapped { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(path) + .expect("Failed to open the links file"); + FileMapped::new(file).expect("Failed to map the links file") +} + +/// Markers of the object like structures stored as links. +/// +/// The markers mirror the ones of the C# `DoubletsDbContext`: a marker for the +/// blog post itself and one marker per property. +#[derive(Debug, Clone, Copy)] +struct Markers { + blog_post: usize, + title: usize, + content: usize, + publication_date_time: usize, + empty_string: usize, +} + +/// State needed to store object like structures as links. +/// +/// Doublets has no notion of a string, so every string is stored as a sequence +/// of links: every distinct character becomes a point link (a "symbol"), and a +/// string becomes a left fold of `(sequence, symbol)` links. Identical strings +/// (and identical prefixes) are therefore stored exactly once, which is how the +/// C# benchmark stores unicode sequences as well. +#[derive(Default)] +struct ObjectsState { + markers: Option, + symbols: HashMap, + characters: HashMap, +} + +impl ObjectsState { + fn clear(&mut self) { + self.markers = None; + self.symbols.clear(); + self.characters.clear(); + } +} + /// Wrapper to adapt doublets::Doublets to our Links trait pub struct DoubletsLinks { store: S, + objects: ObjectsState, } impl DoubletsLinks { pub fn new(store: S) -> Self { - Self { store } + Self { + store, + objects: ObjectsState::default(), + } } pub fn into_inner(self) -> S { @@ -35,8 +94,13 @@ impl DoubletsLinks { impl + DoubletsExt> Links for DoubletsLinks { fn create(&mut self, source: u64, target: u64) -> u64 { + // `create_by` passes its argument as a *restriction* of the query, and + // the memory stores ignore it — it always creates an empty link. The + // source and the target are only assigned by the update that follows, + // which is exactly what `create_link` does (the equivalent of + // `CreateAndUpdate` of the C# `Platform.Data.Doublets`). self.store - .create_by([source as usize, target as usize]) + .create_link(source as usize, target as usize) .expect("Failed to create link") as u64 } @@ -57,6 +121,8 @@ impl + DoubletsExt> Links for DoubletsLinks { } fn delete_all(&mut self) { + // The markers and the symbols are links too, they are gone as well. + self.objects.clear(); let any = self.store.constants().any; let ids: Vec = self .store @@ -77,9 +143,9 @@ impl + DoubletsExt> Links for DoubletsLinks { } fn query_by_id(&self, id: u64) -> Option { - self.store.get_link(id as usize).map(|link| { - Link::new(link.index as u64, link.source as u64, link.target as u64) - }) + self.store + .get_link(id as usize) + .map(|link| Link::new(link.index as u64, link.source as u64, link.target as u64)) } fn query_by_source(&self, source: u64) -> Vec { @@ -111,6 +177,208 @@ impl + DoubletsExt> Links for DoubletsLinks { } } +impl + DoubletsExt> DoubletsLinks { + /// Returns the markers of the object like structures, creating them on first use. + fn markers(&mut self) -> Markers { + if let Some(markers) = self.objects.markers { + return markers; + } + let markers = Markers { + blog_post: self.point(), + title: self.point(), + content: self.point(), + publication_date_time: self.point(), + empty_string: self.point(), + }; + self.objects.markers = Some(markers); + markers + } + + fn point(&mut self) -> usize { + self.store + .create_point() + .expect("Failed to create a marker") + } + + /// Returns the point link representing `character`, creating it on first use. + fn symbol(&mut self, character: char) -> usize { + if let Some(&symbol) = self.objects.symbols.get(&character) { + return symbol; + } + let symbol = self.point(); + self.objects.symbols.insert(character, symbol); + self.objects.characters.insert(symbol, character); + symbol + } + + /// Stores `text` as a sequence of links and returns the link of the sequence. + fn create_sequence(&mut self, text: &str) -> usize { + let mut sequence: Option = None; + for character in text.chars() { + let symbol = self.symbol(character); + sequence = Some(match sequence { + None => symbol, + Some(previous) => self + .store + .get_or_create(previous, symbol) + .expect("Failed to create a sequence link"), + }); + } + sequence.unwrap_or_else(|| self.markers().empty_string) + } + + /// Restores the string stored as the sequence of links `sequence`. + fn read_sequence(&self, sequence: usize) -> String { + if Some(sequence) == self.objects.markers.map(|markers| markers.empty_string) { + return String::new(); + } + let mut characters = Vec::new(); + let mut current = sequence; + loop { + if let Some(&character) = self.objects.characters.get(¤t) { + characters.push(character); + break; + } + match self.store.get_link(current) { + Some(link) => { + match self.objects.characters.get(&link.target) { + Some(&character) => characters.push(character), + // Not a sequence of this store, stop instead of looping. + None => break, + } + current = link.source; + } + None => break, + } + } + characters.reverse(); + characters.into_iter().collect() + } + + /// Sets the value of a property of an object, as `PropertiesOperator` does in C#. + fn set_property(&mut self, object: usize, marker: usize, value: usize) { + let property = self + .store + .get_or_create(object, marker) + .expect("Failed to create a property link"); + self.store + .get_or_create(property, value) + .expect("Failed to create a property value link"); + } + + /// Returns the value of a property of an object. + fn property(&self, object: usize, marker: usize) -> Option { + let any = self.store.constants().any; + let property = self.store.search(object, marker)?; + self.store + .find([any, property, any]) + .map(|link| link.target) + } +} + +impl + DoubletsExt> Objects for DoubletsLinks { + fn create_posts(&mut self, posts: &[BlogPost]) -> Vec { + let markers = self.markers(); + let mut ids = Vec::with_capacity(posts.len()); + for post in posts { + let title = self.create_sequence(&post.title); + let content = self.create_sequence(&post.content); + let date = self.create_sequence(&post.publication_date_time.to_string()); + + // The blog post link is `(blog post marker, itself)`, exactly like + // `CreateAndUpdate(_blogPostMarker, Constants.Itself)` in C#. + let object = self.store.create().expect("Failed to create a blog post"); + self.store + .update(object, markers.blog_post, object) + .expect("Failed to mark a blog post"); + + self.set_property(object, markers.title, title); + self.set_property(object, markers.content, content); + self.set_property(object, markers.publication_date_time, date); + ids.push(object as u64); + } + ids + } + + fn read_posts(&self) -> Vec { + let markers = match self.objects.markers { + Some(markers) => markers, + None => return Vec::new(), + }; + self.objects_of(markers) + .into_iter() + .map(|object| { + let title = self + .property(object, markers.title) + .map(|value| self.read_sequence(value)) + .unwrap_or_default(); + let content = self + .property(object, markers.content) + .map(|value| self.read_sequence(value)) + .unwrap_or_default(); + let date = self + .property(object, markers.publication_date_time) + .map(|value| self.read_sequence(value)) + .unwrap_or_default(); + BlogPost { + id: object as u64, + title, + content, + publication_date_time: date.parse().unwrap_or_default(), + } + }) + .collect() + } + + fn delete_posts(&mut self) { + let markers = match self.objects.markers { + Some(markers) => markers, + None => return, + }; + let any = self.store.constants().any; + for object in self.objects_of(markers) { + let properties: Vec = self + .store + .each_iter([any, object, any]) + .map(|link| link.index) + .filter(|&index| index != object) + .collect(); + for property in properties { + let values: Vec = self + .store + .each_iter([any, property, any]) + .map(|link| link.index) + .filter(|&index| index != property) + .collect(); + for value in values { + let _ = self.store.delete(value); + } + let _ = self.store.delete(property); + } + let _ = self.store.delete(object); + } + } + + fn count_posts(&self) -> usize { + match self.objects.markers { + Some(markers) => self.objects_of(markers).len(), + None => 0, + } + } +} + +impl + DoubletsExt> DoubletsLinks { + /// Links of all stored blog posts: `(blog post marker, itself)` links. + fn objects_of(&self, markers: Markers) -> Vec { + let any = self.store.constants().any; + self.store + .each_iter([any, markers.blog_post, any]) + .map(|link| link.index) + .filter(|&index| index != markers.blog_post) + .collect() + } +} + /// Create a new in-memory doublets united store pub fn create_united_volatile() -> DoubletsLinks { let mem = Alloc::new(Global); @@ -127,6 +395,27 @@ pub fn create_split_volatile() -> DoubletsLinks { DoubletsLinks::new(store) } +/// Create a new file-mapped doublets united store +pub fn create_united_non_volatile>( + path: P, +) -> DoubletsLinks { + let mem = map_file(path); + let store = DoubletsUnitedNonVolatile::new(mem).expect("Failed to create doublets store"); + DoubletsLinks::new(store) +} + +/// Create a new file-mapped doublets split store +pub fn create_split_non_volatile, Q: AsRef>( + data_path: P, + index_path: Q, +) -> DoubletsLinks { + let data_mem = map_file(data_path); + let index_mem = map_file(index_path); + let store = DoubletsSplitNonVolatile::new(data_mem, index_mem) + .expect("Failed to create doublets store"); + DoubletsLinks::new(store) +} + #[cfg(test)] mod tests { use super::*; @@ -153,6 +442,23 @@ mod tests { assert_eq!(link.target, id); } + /// Regression test: `create` used to call `create_by([source, target])`, + /// which the memory stores read as a restriction and ignore, so every + /// benchmarked link was created empty instead of connecting two links. + #[test] + fn test_create_assigns_source_and_target() { + let mut db = create_united_volatile(); + let first = db.create_point(); + let second = db.create_point(); + let id = db.create(first, second); + + let link = db.query_by_id(id).unwrap(); + assert_eq!(link.source, first); + assert_eq!(link.target, second); + assert_eq!(db.query_by_source(first).len(), 2); + assert_eq!(db.query_by_target(second).len(), 2); + } + #[test] fn test_update() { let mut db = create_united_volatile(); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 04f91f4..8db569f 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -9,14 +9,21 @@ pub mod benched; pub mod doublets_impl; pub mod exclusive; pub mod fork; +pub mod objects; pub mod sqlite_impl; pub use benched::Benched; pub use exclusive::Exclusive; pub use fork::Fork; +pub use objects::{generate_posts, BlogPost, Objects, BENCHMARK_OBJECT_COUNT}; use once_cell::sync::Lazy; -use std::env; +use std::{ + env, + path::PathBuf, + process, + sync::atomic::{AtomicUsize, Ordering}, +}; /// Number of links to use for benchmarking pub static BENCHMARK_LINK_COUNT: Lazy = Lazy::new(|| { @@ -34,6 +41,22 @@ pub static BACKGROUND_LINK_COUNT: Lazy = Lazy::new(|| { .unwrap_or(3000) }); +/// Returns a unique path inside the temporary directory of the machine. +/// +/// The non-volatile (file backed) benchmark subjects store their data there, so +/// that parallel runs of the benchmarks never share a file, and so that nothing +/// is left in the working directory. +pub fn temp_path(name: &str) -> PathBuf { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + let unique = COUNTER.fetch_add(1, Ordering::Relaxed); + env::temp_dir().join(format!( + "sqlite-vs-doublets-{}-{}-{}", + process::id(), + unique, + name + )) +} + /// A link structure representing a doublet (source -> target relationship) #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Link { @@ -88,6 +111,37 @@ pub trait Links { fn count(&self) -> usize; } +/// Creates `count` background point links and returns their identifiers. +/// +/// Background links make every benchmark run against a non-empty storage, so +/// that queries have to search through unrelated data, as they would in a real +/// application. +pub fn fill_background(links: &mut impl Links, count: usize) -> Vec { + (0..count).map(|_| links.create_point()).collect() +} + +/// Creates `count` links between the given background links. +/// +/// Link number `i` connects `background[i % len]` to a background link further +/// down the list, so that every created doublet is unique (Doublets stores +/// every doublet exactly once) while sources and targets are shared by several +/// links — which is what makes the outgoing and incoming queries meaningful. +pub fn fill_benchmarked(links: &mut impl Links, background: &[u64], count: usize) -> Vec { + assert!( + !background.is_empty(), + "background links are required to create benchmarked links" + ); + let len = background.len(); + (0..count) + .map(|index| { + let source = background[index % len]; + let target = background[(index % len + 1 + index / len) % len]; + let id = links.create(source, target); + Link::new(id, source, target) + }) + .collect() +} + /// Macro for running benchmarks with proper setup and teardown #[macro_export] macro_rules! bench { diff --git a/rust/src/objects.rs b/rust/src/objects.rs new file mode 100644 index 0000000..894d3c4 --- /dev/null +++ b/rust/src/objects.rs @@ -0,0 +1,164 @@ +//! Object-like structures (blog posts) shared by both storages. +//! +//! The C# part of this repository compares SQLite and Doublets on an object +//! like structure — a blog post with a title, a content and a publication date. +//! This module provides the same model for the Rust benchmarks, so that both +//! languages measure the same three operations: +//! +//! | Operation | Meaning | +//! |-----------------------|-----------------------------------------------| +//! | `Objects Create List` | save a list of blog posts into empty storage | +//! | `Objects Read List` | read every stored blog post back | +//! | `Objects Delete List` | delete every stored blog post | +//! +//! The generated data matches `csharp/Model/BlogPosts.cs`: the title is +//! `Blog post {n}`, the content is one of five lorem ipsum paragraphs and the +//! publication date is within the last 30 days. Generation is deterministic +//! (a small xorshift generator seeded by [`OBJECT_SEED`]) so that every run and +//! every storage is benchmarked on exactly the same data. + +use once_cell::sync::Lazy; +use std::env; + +/// Number of blog posts used by the object benchmarks. +/// +/// Can be overridden with the `BENCHMARK_OBJECT_COUNT` environment variable, +/// which the CI workflow lowers for pull request runs. +pub static BENCHMARK_OBJECT_COUNT: Lazy = Lazy::new(|| { + env::var("BENCHMARK_OBJECT_COUNT") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(1000) +}); + +/// Seed of the deterministic generator of blog posts. +pub const OBJECT_SEED: u64 = 0x5148_5157_2D31_3235; + +/// Lorem ipsum paragraphs used as blog post contents. +/// +/// The same five paragraphs are used by `csharp/Model/BlogPosts.cs`. +pub const CONTENTS: [&str; 5] = [ + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis malesuada blandit mauris nec bibendum.", + "Curabitur tincidunt nibh sit amet finibus dictum. Suspendisse aliquet arcu non rutrum ultrices.", + "Donec vitae felis lectus. Aenean velit sapien, porttitor ut feugiat a, consectetur et risus.", + "Aliquam sed egestas felis. Maecenas sollicitudin nisl in sapien posuere vulputate.", + "Ut a eleifend augue, eget posuere augue. Proin purus neque, pretium condimentum ipsum ut.", +]; + +/// An object like structure: a blog post. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlogPost { + /// Storage assigned identifier, `0` before the post is saved. + pub id: u64, + /// Unique title of the post. + pub title: String, + /// Body of the post. + pub content: String, + /// Publication date as a Unix timestamp in seconds. + pub publication_date_time: i64, +} + +impl BlogPost { + /// Creates a not yet stored blog post. + pub fn new( + title: impl Into, + content: impl Into, + publication_date_time: i64, + ) -> Self { + Self { + id: 0, + title: title.into(), + content: content.into(), + publication_date_time, + } + } +} + +/// Deterministic xorshift64* generator, so every benchmark sees the same data. +struct Generator(u64); + +impl Generator { + fn next(&mut self) -> u64 { + let mut state = self.0; + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + self.0 = state; + state + } +} + +/// Generates `count` blog posts, mirroring `BlogPosts.GenerateData` of the C# benchmark. +pub fn generate_posts(count: usize) -> Vec { + // A fixed "now" keeps the generated data stable between runs. + const NOW: i64 = 1_700_000_000; + const SECONDS_IN_30_DAYS: i64 = 30 * 24 * 60 * 60; + + let mut generator = Generator(OBJECT_SEED); + (0..count) + .map(|index| { + let content = CONTENTS[(generator.next() % CONTENTS.len() as u64) as usize]; + let age = (generator.next() % SECONDS_IN_30_DAYS as u64) as i64; + BlogPost::new(format!("Blog post {}", index + 1), content, NOW - age) + }) + .collect() +} + +/// Storage of object like structures. +/// +/// Implemented natively by every benchmarked backend: SQLite stores blog posts +/// as rows of a table, Doublets stores them as links (with strings represented +/// as sequences of links). +pub trait Objects { + /// Saves every post of the list, returns the assigned identifiers. + fn create_posts(&mut self, posts: &[BlogPost]) -> Vec; + + /// Reads every stored post back. + fn read_posts(&self) -> Vec; + + /// Deletes every stored post. + fn delete_posts(&mut self); + + /// Number of stored posts. + fn count_posts(&self) -> usize; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_data_is_deterministic() { + assert_eq!(generate_posts(16), generate_posts(16)); + } + + #[test] + fn generated_titles_match_the_csharp_benchmark() { + let posts = generate_posts(3); + assert_eq!(posts[0].title, "Blog post 1"); + assert_eq!(posts[2].title, "Blog post 3"); + } + + #[test] + fn generated_contents_are_taken_from_the_lorem_ipsum_paragraphs() { + for post in generate_posts(64) { + assert!(CONTENTS.contains(&post.content.as_str())); + } + } + + #[test] + fn generated_dates_are_within_the_last_30_days() { + let posts = generate_posts(64); + let newest = posts + .iter() + .map(|post| post.publication_date_time) + .max() + .unwrap(); + let oldest = posts + .iter() + .map(|post| post.publication_date_time) + .min() + .unwrap(); + assert!(newest - oldest <= 30 * 24 * 60 * 60); + } +} diff --git a/rust/src/sqlite_impl.rs b/rust/src/sqlite_impl.rs index 1f0d92b..b4022a0 100644 --- a/rust/src/sqlite_impl.rs +++ b/rust/src/sqlite_impl.rs @@ -1,6 +1,6 @@ //! SQLite implementation for links storage -use crate::{Link, Links}; +use crate::{BlogPost, Link, Links, Objects}; use rusqlite::{params, Connection}; use std::path::Path; @@ -24,34 +24,7 @@ impl SqliteLinks { } fn init(conn: Connection) -> Self { - conn.execute( - "CREATE TABLE IF NOT EXISTS links ( - id INTEGER PRIMARY KEY, - source INTEGER NOT NULL, - target INTEGER NOT NULL - )", - [], - ) - .expect("Failed to create links table"); - - // Create indexes for efficient queries - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_source ON links(source)", - [], - ) - .expect("Failed to create source index"); - - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_target ON links(target)", - [], - ) - .expect("Failed to create target index"); - - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_source_target ON links(source, target)", - [], - ) - .expect("Failed to create source_target index"); + Self::create_schema(&conn); // Get the max ID to continue from let next_id: u64 = conn @@ -63,37 +36,42 @@ impl SqliteLinks { Self { conn, next_id } } - /// Drop all tables and recreate them - pub fn reset(&mut self) { - self.conn - .execute("DROP TABLE IF EXISTS links", []) - .expect("Failed to drop links table"); - self.conn - .execute( - "CREATE TABLE links ( + /// Creates the tables and the indexes used by the benchmarks. + /// + /// `links` stores the link related benchmark data, `blog_posts` stores the + /// object like structures — the same shape the C# benchmark stores through + /// Entity Framework Core (a primary key, a unique title, a content and a + /// publication date). + fn create_schema(conn: &Connection) { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS links ( id INTEGER PRIMARY KEY, source INTEGER NOT NULL, target INTEGER NOT NULL - )", - [], - ) - .expect("Failed to create links table"); - - self.conn - .execute("CREATE INDEX idx_source ON links(source)", []) - .expect("Failed to create source index"); - - self.conn - .execute("CREATE INDEX idx_target ON links(target)", []) - .expect("Failed to create target index"); + ); + CREATE INDEX IF NOT EXISTS idx_source ON links(source); + CREATE INDEX IF NOT EXISTS idx_target ON links(target); + CREATE INDEX IF NOT EXISTS idx_source_target ON links(source, target); + CREATE TABLE IF NOT EXISTS blog_posts ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + content TEXT NOT NULL, + publication_date_time INTEGER NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_blog_posts_title ON blog_posts(title);", + ) + .expect("Failed to create the benchmark schema"); + } + /// Drop all tables and recreate them + pub fn reset(&mut self) { self.conn - .execute( - "CREATE INDEX idx_source_target ON links(source, target)", - [], + .execute_batch( + "DROP TABLE IF EXISTS links; + DROP TABLE IF EXISTS blog_posts;", ) - .expect("Failed to create source_target index"); - + .expect("Failed to drop the benchmark tables"); + Self::create_schema(&self.conn); self.next_id = 1; } } @@ -228,6 +206,70 @@ impl Links for SqliteLinks { } } +impl Objects for SqliteLinks { + fn create_posts(&mut self, posts: &[BlogPost]) -> Vec { + let transaction = self + .conn + .unchecked_transaction() + .expect("Failed to start a transaction"); + let mut ids = Vec::with_capacity(posts.len()); + { + let mut statement = transaction + .prepare( + "INSERT INTO blog_posts (title, content, publication_date_time) + VALUES (?1, ?2, ?3)", + ) + .expect("Failed to prepare the blog post insert"); + for post in posts { + statement + .execute(params![ + post.title, + post.content, + post.publication_date_time + ]) + .expect("Failed to insert a blog post"); + ids.push(transaction.last_insert_rowid() as u64); + } + } + transaction.commit().expect("Failed to commit blog posts"); + ids + } + + fn read_posts(&self) -> Vec { + let mut statement = self + .conn + .prepare("SELECT id, title, content, publication_date_time FROM blog_posts") + .expect("Failed to prepare the blog post query"); + + statement + .query_map([], |row| { + Ok(BlogPost { + id: row.get::<_, i64>(0)? as u64, + title: row.get(1)?, + content: row.get(2)?, + publication_date_time: row.get(3)?, + }) + }) + .expect("Failed to query blog posts") + .filter_map(|post| post.ok()) + .collect() + } + + fn delete_posts(&mut self) { + self.conn + .execute("DELETE FROM blog_posts", []) + .expect("Failed to delete blog posts"); + } + + fn count_posts(&self) -> usize { + self.conn + .query_row("SELECT COUNT(*) FROM blog_posts", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap_or(0) as usize + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/test_out.py b/rust/test_out.py new file mode 100644 index 0000000..0b27574 --- /dev/null +++ b/rust/test_out.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Regression tests for the Criterion result parser and Rust report CLI.""" + +import os +import tempfile +import unittest + +import out + + +def complete_output(): + lines = ["Gnuplot not found, using plotters backend"] + value = 1_000 + for operation, _label in out.report.OPERATIONS: + for variant, _variant_label, _color in out.report.VARIANTS: + lines.append( + f"test {operation}/{variant}/10 ... bench: {value:,} ns/iter (+/- 10)" + ) + value += 100 + return "\n".join(lines) + "\n" + + +class ParserTests(unittest.TestCase): + def test_parses_every_operation_and_variant(self): + results = out.parse_text(complete_output()) + + self.assertFalse(out.report.missing_measurements(results)) + self.assertEqual(results["create"]["Doublets_United_Volatile"], 1_000) + + def test_accepts_error_text_inside_a_record(self): + results = out.parse_text( + "test create/SQLite_Memory/10 ... Criterion.rs ERROR: stale cache\n" + "bench: 12,345 ns/iter (+/- 10)\n" + ) + + self.assertEqual(results["create"]["SQLite_Memory"], 12_345) + + def test_does_not_borrow_the_next_record_measurement(self): + results = out.parse_text( + "test create/SQLite_Memory/10 ...\n" + "test create/SQLite_File/10 ... bench: 42 ns/iter (+/- 1)\n" + ) + + self.assertNotIn("SQLite_Memory", results["create"]) + self.assertEqual(results["create"]["SQLite_File"], 42) + + +class MainTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.input = os.path.join(self.temporary.name, "out.txt") + self.results = os.path.join(self.temporary.name, "results.md") + self.readme = os.path.join(self.temporary.name, "README.md") + + def write(self, path, content): + with open(path, "w", encoding="utf-8") as handle: + handle.write(content) + + def test_generates_results_and_updates_each_readme(self): + self.write(self.input, complete_output()) + self.write( + self.readme, + "# Results\n\n" + f"{out.report.RUST_START_MARKER}\nold\n" + f"{out.report.RUST_END_MARKER}\n", + ) + + result = out.main( + [ + self.input, + "--results", + self.results, + "--readme", + self.readme, + "--output-dir", + self.temporary.name, + ] + ) + + self.assertEqual(result, 0) + with open(self.results, "r", encoding="utf-8") as handle: + self.assertIn("Objects Delete List", handle.read()) + with open(self.readme, "r", encoding="utf-8") as handle: + self.assertIn("Doublets Split NonVolatile", handle.read()) + + def test_rejects_partial_output(self): + self.write( + self.input, + "test create/SQLite_Memory/10 ... bench: 42 ns/iter (+/- 1)\n", + ) + + self.assertEqual(out.main([self.input, "--results", self.results]), 1) + self.assertFalse(os.path.exists(self.results)) + + def test_empty_input_reports_failure(self): + self.write(self.input, "") + + self.assertEqual(out.main([self.input, "--results", self.results]), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/rust/tests/storages.rs b/rust/tests/storages.rs new file mode 100644 index 0000000..9ec3854 --- /dev/null +++ b/rust/tests/storages.rs @@ -0,0 +1,172 @@ +//! Behaviour tests shared by every benchmarked storage. +//! +//! The benchmarks only make sense when every variant implements exactly the +//! same semantics, so the same checks are executed against all six subjects: +//! SQLite (in-memory and file based) and Doublets (united and split stores, +//! each of them volatile and non-volatile). + +#![feature(allocator_api)] + +use sqlite_vs_doublets::benched::{ + DoubletsSplitNonVolatileBenched, DoubletsSplitVolatileBenched, + DoubletsUnitedNonVolatileBenched, DoubletsUnitedVolatileBenched, SqliteFileBenched, + SqliteMemoryBenched, +}; +use sqlite_vs_doublets::{ + fill_background, fill_benchmarked, generate_posts, Benched, Links, Objects, +}; +use std::ops::DerefMut; + +const BACKGROUND: usize = 16; +const BENCHMARKED: usize = 32; +const POSTS: usize = 8; + +/// Checks the link related operations benchmarked by `benches/benchmarks`. +fn check_links(subject: &mut B) +where + B: Benched + DerefMut, + B::Target: Links + Sized, +{ + { + let mut fork = subject.fork(); + let links = &mut **fork; + + let background = fill_background(links, BACKGROUND); + assert_eq!(background.len(), BACKGROUND); + let created = fill_benchmarked(links, &background, BENCHMARKED); + assert_eq!(created.len(), BENCHMARKED); + assert_eq!(links.count(), BACKGROUND + BENCHMARKED); + assert_eq!(links.query_all().len(), BACKGROUND + BENCHMARKED); + + // Every created doublet has to be unique, otherwise Doublets would + // return an already existing link instead of creating a new one. + let mut doublets: Vec<(u64, u64)> = created.iter().map(|l| (l.source, l.target)).collect(); + doublets.sort_unstable(); + doublets.dedup(); + assert_eq!(doublets.len(), BENCHMARKED); + + let sample = created[BENCHMARKED / 2]; + assert_eq!(links.query_by_id(sample.id), Some(sample)); + + let all = links.query_all(); + let outgoing = all.iter().filter(|l| l.source == sample.source).count(); + assert_eq!(links.query_by_source(sample.source).len(), outgoing); + let incoming = all.iter().filter(|l| l.target == sample.target).count(); + assert_eq!(links.query_by_target(sample.target).len(), incoming); + let concrete = all + .iter() + .filter(|l| l.source == sample.source && l.target == sample.target) + .count(); + let queried = links.query_by_source_target(sample.source, sample.target); + assert_eq!(queried.len(), concrete); + + // Updating to a doublet that is not stored yet, as the update benchmark does. + let unused = links.create_point(); + links.update(sample.id, unused, background[0]); + let updated = links + .query_by_id(sample.id) + .expect("updated link is missing"); + assert_eq!((updated.source, updated.target), (unused, background[0])); + + let before = links.count(); + links.delete(sample.id); + assert_eq!(links.count(), before - 1); + assert!(links.query_by_id(sample.id).is_none()); + + links.delete_all(); + assert_eq!(links.count(), 0); + assert!(links.query_all().is_empty()); + } + + // Dropping the fork has to restore the empty state for the next iteration. + assert_eq!(subject.count(), 0); + assert!(subject.query_all().is_empty()); +} + +/// Checks the object like (blog post) operations benchmarked by `benches/benchmarks/objects`. +fn check_objects(subject: &mut B) +where + B: Benched + DerefMut, + B::Target: Objects + Links + Sized, +{ + let expected = generate_posts(POSTS); + { + let mut fork = subject.fork(); + let storage = &mut **fork; + + let ids = storage.create_posts(&expected); + assert_eq!(ids.len(), POSTS); + assert_eq!(storage.count_posts(), POSTS); + + let mut read = storage.read_posts(); + read.sort_by(|left, right| left.title.cmp(&right.title)); + let mut expected = expected.clone(); + expected.sort_by(|left, right| left.title.cmp(&right.title)); + assert_eq!(read.len(), expected.len()); + for (read, expected) in read.iter().zip(expected.iter()) { + assert_eq!(read.title, expected.title); + assert_eq!(read.content, expected.content); + assert_eq!(read.publication_date_time, expected.publication_date_time); + } + + storage.delete_posts(); + assert_eq!(storage.count_posts(), 0); + assert!(storage.read_posts().is_empty()); + } + + assert_eq!(subject.count_posts(), 0); +} + +/// Objects and links live in the same storage, they must not disturb each other. +fn check_objects_and_links(subject: &mut B) +where + B: Benched + DerefMut, + B::Target: Objects + Links + Sized, +{ + let mut fork = subject.fork(); + let storage = &mut **fork; + + let background = fill_background(storage, BACKGROUND); + fill_benchmarked(storage, &background, BENCHMARKED); + let posts = generate_posts(POSTS); + storage.create_posts(&posts); + + assert_eq!(storage.count_posts(), POSTS); + assert_eq!(storage.read_posts().len(), POSTS); + + storage.delete_posts(); + assert_eq!(storage.count_posts(), 0); +} + +macro_rules! storage_tests { + ($module:ident, $benched:ty) => { + mod $module { + use super::*; + + #[test] + fn links() { + check_links(&mut <$benched>::setup(())); + } + + #[test] + fn objects() { + check_objects(&mut <$benched>::setup(())); + } + + #[test] + fn objects_and_links() { + check_objects_and_links(&mut <$benched>::setup(())); + } + } + }; +} + +storage_tests!(sqlite_memory, SqliteMemoryBenched); +storage_tests!(sqlite_file, SqliteFileBenched); +storage_tests!(doublets_united_volatile, DoubletsUnitedVolatileBenched); +storage_tests!( + doublets_united_non_volatile, + DoubletsUnitedNonVolatileBenched +); +storage_tests!(doublets_split_volatile, DoubletsSplitVolatileBenched); +storage_tests!(doublets_split_non_volatile, DoubletsSplitNonVolatileBenched); diff --git a/scripts/benchmark_report.py b/scripts/benchmark_report.py new file mode 100644 index 0000000..a37e98b --- /dev/null +++ b/scripts/benchmark_report.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Shared benchmark reporting helpers for the SQLite vs Doublets comparison.""" + +# The language-specific pipelines parse their own benchmark formats, then use +# this module for consistent Markdown tables, speedup annotations, linear and +# logarithmic charts, and in-place result-section updates. The report format +# follows the sibling LinksPlatform database comparisons. + +import os +import re +import shutil +from datetime import datetime, timezone + +try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + import numpy as np + + HAS_MATPLOTLIB = True +except ImportError: # pragma: no cover - exercised only without matplotlib + print("Warning: matplotlib/numpy not installed, skipping chart generation") + HAS_MATPLOTLIB = False + +RUST_START_MARKER = "" +RUST_END_MARKER = "" +CSHARP_START_MARKER = "" +CSHARP_END_MARKER = "" + +# Operations shared by every language of this comparison. The first element of +# each pair is the identifier used by the benchmark runner, the second one is +# the label used in reports. +LINK_OPERATIONS = ( + ("create", "Create"), + ("update", "Update"), + ("delete", "Delete"), + ("query_all", "Each All"), + ("query_by_id", "Each Identity"), + ("query_by_source_target", "Each Concrete"), + ("query_by_source", "Each Outgoing"), + ("query_by_target", "Each Incoming"), +) + +# Object-like structures (blog posts), the operations the C# comparison has +# been built around from the beginning. +OBJECT_OPERATIONS = ( + ("objects_create", "Objects Create List"), + ("objects_read", "Objects Read List"), + ("objects_delete", "Objects Delete List"), +) + +OPERATIONS = LINK_OPERATIONS + OBJECT_OPERATIONS + +# Benchmarked backends: identifier, label and chart color. +DOUBLETS_VARIANTS = ( + ("Doublets_United_Volatile", "Doublets United Volatile", "salmon"), + ("Doublets_United_NonVolatile", "Doublets United NonVolatile", "red"), + ("Doublets_Split_Volatile", "Doublets Split Volatile", "lightgreen"), + ("Doublets_Split_NonVolatile", "Doublets Split NonVolatile", "green"), +) + +SQLITE_VARIANTS = ( + ("SQLite_Memory", "SQLite Memory", "lightblue"), + ("SQLite_File", "SQLite File", "royalblue"), +) + +VARIANTS = DOUBLETS_VARIANTS + SQLITE_VARIANTS + +# Doublets cells are annotated relative to the fastest SQLite measurement of +# the same operation, the same way Comparisons.Neo4jVSDoublets annotates +# against the fastest of the two Neo4j modes. +BASELINES = tuple(key for key, _label, _color in SQLITE_VARIANTS) + + +def empty_results(operations=OPERATIONS): + """Build an empty ``{operation: {variant: nanoseconds}}`` mapping.""" + return {op: {} for op, _label in operations} + + +def has_any_results(results): + """Return ``True`` when at least one measurement was parsed.""" + return any(measurements for measurements in results.values()) + + +def missing_measurements(results, operations=OPERATIONS, variants=VARIANTS): + """List expected ``operation/variant`` pairs absent from parsed output.""" + missing = [] + for operation, _operation_label in operations: + measured = results.get(operation, {}) + for variant, _variant_label, _color in variants: + if not measured.get(variant): + missing.append(f"{operation}/{variant}") + return missing + + +def baseline_of(measurements, baselines=BASELINES): + """Fastest baseline (SQLite) measurement of a single operation, 0 if none.""" + values = [measurements.get(key, 0) for key in baselines] + values = [value for value in values if value] + return min(values) if values else 0 + + +def format_speedup(value, baseline): + """Annotate ``value`` with how it compares to the ``baseline`` measurement.""" + if not value: + return "N/A" + if not baseline: + return f"{value}" + if value <= baseline: + return f"{value} ({baseline / value:.1f}x faster)" + return f"{value} ({value / baseline:.1f}x slower)" + + +def format_results_table(results, operations=OPERATIONS, variants=VARIANTS, baselines=BASELINES): + """Render the Markdown results table (all numbers in nanoseconds).""" + labels = [label for _key, label, _color in variants] + cells_by_variant = [] + for key, _label, _color in variants: + column = [] + for op, _op_label in operations: + measurements = results.get(op, {}) + value = measurements.get(key, 0) + if key in baselines: + column.append(str(value) if value else "N/A") + else: + column.append(format_speedup(value, baseline_of(measurements, baselines))) + cells_by_variant.append(column) + + widths = [ + max(len(label), *(len(cell) for cell in column)) + for label, column in zip(labels, cells_by_variant) + ] + operation_width = max(len("Operation"), *(len(label) for _key, label in operations)) + + header = "| " + "Operation".ljust(operation_width) + " | " + header += " | ".join(label.ljust(width) for label, width in zip(labels, widths)) + header += " |" + separator = "|" + "-" * (operation_width + 2) + separator += "".join("|" + "-" * (width + 2) for width in widths) + "|" + + lines = [header, separator] + for index, (_op, op_label) in enumerate(operations): + row = "| " + op_label.ljust(operation_width) + " | " + row += " | ".join( + column[index].ljust(width) for column, width in zip(cells_by_variant, widths) + ) + row += " |" + lines.append(row) + + return "\n".join(lines) + + +def build_provenance( + benchmark_links=None, + background_links=None, + object_count=None, + generated_at=None, + language=None, +): + """Describe how and when the committed results were produced.""" + benchmark_links = benchmark_links or os.environ.get("BENCHMARK_LINK_COUNT", "1000") + background_links = background_links or os.environ.get("BACKGROUND_LINK_COUNT", "3000") + if object_count is not False: + object_count = object_count or os.environ.get("BENCHMARK_OBJECT_COUNT", "1000") + generated_at = generated_at or datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + source = "a local benchmark run" + repository = os.environ.get("GITHUB_REPOSITORY") + run_id = os.environ.get("GITHUB_RUN_ID") + if repository and run_id: + source = ( + f"[GitHub Actions run {run_id}]" + f"(https://github.com/{repository}/actions/runs/{run_id})" + ) + + prefix = f"_Generated {generated_at}" + if language: + prefix += f" for {language}" + quantities = ( + f"{benchmark_links} benchmarked links, " + f"{background_links} background links" + ) + if object_count is not False: + quantities += f", {object_count} objects" + return f"{prefix} by {source} — {quantities}._" + + +def render_results_section(results, provenance=None, **table_options): + """Render a results section: provenance line plus the results table.""" + provenance = provenance if provenance is not None else build_provenance() + return f"{provenance}\n\n{format_results_table(results, **table_options)}" + + +def update_markers(path, section, start_marker, end_marker): + """Replace a marked Markdown section and report whether it changed.""" + with open(path, "r", encoding="utf-8") as handle: + document = handle.read() + + if start_marker not in document or end_marker not in document: + raise ValueError(f"{path} does not contain the {start_marker} / {end_marker} markers") + + pattern = re.compile( + re.escape(start_marker) + r".*?" + re.escape(end_marker), + re.DOTALL, + ) + replacement = f"{start_marker}\n{section}\n{end_marker}" + updated = pattern.sub(lambda _match: replacement, document, count=1) + + if updated == document: + return False + + with open(path, "w", encoding="utf-8") as handle: + handle.write(updated) + return True + + +def _series(results, variant, operations): + """Measurements of one variant across all operations, 0 when missing.""" + return [results.get(op, {}).get(variant, 0) for op, _label in operations] + + +def _ensure_min_visible(values, minimum): + """Keep non-zero bars at least ``minimum`` wide so they stay visible.""" + return [max(value, minimum) if value > 0 else 0 for value in values] + + +def _plot(results, path, title, log_scale, operations, variants): + positions = np.arange(len(operations)) + width = 0.8 / len(variants) + figure, axes = plt.subplots(figsize=(12, 8)) + + series = {key: _series(results, key, operations) for key, _label, _color in variants} + + if log_scale: + plotted = series + else: + # On a linear scale Doublets bars are invisible next to SQLite, so give + # every non-zero measurement a minimum visible width (~0.5% of the + # maximum), matching the sibling benchmark charts. + all_values = [value for values in series.values() for value in values] + max_value = max(all_values) if all_values else 1 + minimum = max_value * 0.005 + plotted = {key: _ensure_min_visible(values, minimum) for key, values in series.items()} + + offset_base = (len(variants) - 1) / 2 + for index, (key, label, color) in enumerate(variants): + offset = (index - offset_base) * width + axes.barh(positions + offset, plotted[key], width, label=label, color=color) + + axes.set_xlabel("Time (ns) – log scale" if log_scale else "Time (ns)") + axes.set_title(title) + axes.set_yticks(positions) + axes.set_yticklabels([label for _op, label in operations]) + if log_scale: + axes.set_xscale("log") + axes.legend() + figure.tight_layout() + + directory = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + figure.savefig(path) + plt.close(figure) + print(f"Generated {path}") + + +def generate_charts( + results, + prefix, + title, + output_dir="", + operations=OPERATIONS, + variants=VARIANTS, +): + """Generate linear/log charts and return their paths when available.""" + if not HAS_MATPLOTLIB: + return [] + + linear = os.path.join(output_dir, f"{prefix}.png") if output_dir else f"{prefix}.png" + logarithmic = ( + os.path.join(output_dir, f"{prefix}_log_scale.png") + if output_dir + else f"{prefix}_log_scale.png" + ) + _plot(results, linear, title, False, operations, variants) + _plot(results, logarithmic, title, True, operations, variants) + return [linear, logarithmic] + + +def copy_charts(charts, docs_dir): + """Copy generated charts into the documentation directory.""" + if not docs_dir: + return [] + os.makedirs(docs_dir, exist_ok=True) + copied = [] + for chart in charts: + if os.path.exists(chart): + destination = os.path.join(docs_dir, os.path.basename(chart)) + shutil.copyfile(chart, destination) + copied.append(destination) + print(f"Copied {chart} -> {destination}") + return copied + + +def report_input_excerpt(path, lines=20): + """Describe the tail of ``path`` so an unparsable run can be diagnosed.""" + if not os.path.exists(path): + return f"{path} does not exist" + + with open(path, "r", encoding="utf-8") as handle: + content = handle.read() + + if not content.strip(): + return f"{path} is empty" + + tail = content.splitlines()[-lines:] + return "\n".join([f"Last {len(tail)} line(s) of {path}:", *tail]) diff --git a/scripts/test_benchmark_report.py b/scripts/test_benchmark_report.py new file mode 100644 index 0000000..11c170e --- /dev/null +++ b/scripts/test_benchmark_report.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Tests for shared result formatting and Markdown publication helpers.""" + +import os +import tempfile +import unittest + +import benchmark_report as report + + +class ResultTests(unittest.TestCase): + def test_fastest_sqlite_variant_is_the_baseline(self): + self.assertEqual( + report.baseline_of({"SQLite_Memory": 200, "SQLite_File": 100}), 100 + ) + + def test_formats_faster_and_slower_measurements(self): + self.assertEqual(report.format_speedup(100, 1_000), "100 (10.0x faster)") + self.assertEqual(report.format_speedup(1_000, 100), "1000 (10.0x slower)") + + def test_reports_every_missing_measurement(self): + results = report.empty_results([("create", "Create")]) + + missing = report.missing_measurements( + results, + [("create", "Create")], + [("SQLite_Memory", "SQLite Memory", "blue")], + ) + + self.assertEqual(missing, ["create/SQLite_Memory"]) + + +class MarkerTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.path = os.path.join(self.temporary.name, "README.md") + + def write(self, content): + with open(self.path, "w", encoding="utf-8") as handle: + handle.write(content) + + def read(self): + with open(self.path, "r", encoding="utf-8") as handle: + return handle.read() + + def test_updates_only_the_selected_language(self): + self.write( + f"{report.RUST_START_MARKER}\nold rust\n{report.RUST_END_MARKER}\n" + f"{report.CSHARP_START_MARKER}\nold csharp\n{report.CSHARP_END_MARKER}\n" + ) + + report.update_markers( + self.path, + r"new rust \g<0> $1", + report.RUST_START_MARKER, + report.RUST_END_MARKER, + ) + + self.assertIn(r"new rust \g<0> $1", self.read()) + self.assertIn("old csharp", self.read()) + + def test_update_is_idempotent(self): + self.write( + f"{report.RUST_START_MARKER}\nold\n{report.RUST_END_MARKER}\n" + ) + report.update_markers( + self.path, + "new", + report.RUST_START_MARKER, + report.RUST_END_MARKER, + ) + + self.assertFalse( + report.update_markers( + self.path, + "new", + report.RUST_START_MARKER, + report.RUST_END_MARKER, + ) + ) + + def test_missing_markers_raise(self): + self.write("# No results\n") + + with self.assertRaises(ValueError): + report.update_markers( + self.path, + "new", + report.RUST_START_MARKER, + report.RUST_END_MARKER, + ) + + +if __name__ == "__main__": + unittest.main()