From 1defaa6353a59ae983157821ee4a0686f526477c Mon Sep 17 00:00:00 2001 From: Ryan <16667079+mccaffers@users.noreply.github.com> Date: Tue, 4 Aug 2026 09:37:08 +0100 Subject: [PATCH] Refactoring to support live trading and research --- .gitignore | 5 +- .gitmodules | 9 +- ARCHITECTURE.md | 346 +++++ CMakeLists.txt | 15 + CONTRIBUTING.md | 2 + CONVENTIONS.md | 23 +- ENVIRONMENT.md | 67 + QUICKSTART.md | 169 +++ README.md | 148 +-- REQUIREMENTS.md | 76 ++ documents/questdb.md | 34 +- external/aws | 1 + scripts/arguments/build.sh | 4 +- scripts/build.sh | 21 +- scripts/build_dep.sh | 69 + scripts/live.sh | 52 + scripts/load.sh | 34 +- scripts/run.sh | 2 +- source/analysis/analysisCommand.cppm | 32 + source/analysis/queue/analysisBridge.cpp | 39 + source/analysis/queue/analysisBridge.hpp | 50 + source/analysis/queue/analysisRunner.cpp | 70 + source/analysis/queue/analysisRunner.hpp | 22 + source/analysis/queue/drainExperiments.cpp | 266 ++++ source/analysis/queue/drainExperiments.hpp | 37 + .../analysis/reporting/experimentElastic.cppm | 41 + .../analysis/reporting/experimentResults.hpp | 133 ++ .../config/dipRecovery/dipRecoverySweep.cppm | 59 + .../config/dipRecovery/makeDipRecovery.cppm | 53 + source/experiments/experimentsCommand.cppm | 244 ++++ source/ingest/ingestCommand.cppm | 42 +- source/ingest/udpPorts.hpp | 20 - source/live/broker/igMarkets.cppm | 286 +++++ source/live/broker/igRequests.cppm | 595 +++++++++ source/live/broker/marketDefinitions.cppm | 189 +++ source/live/config/liveSettings.cppm | 131 ++ source/live/execution/brokerOrderSink.cppm | 246 ++++ source/live/execution/liveStrategyRunner.cppm | 790 ++++++++++++ source/live/execution/orderChannel.cppm | 552 ++++++++ source/live/execution/orderRequest.cppm | 139 ++ .../live/execution/redisPositionCounter.cppm | 136 ++ source/live/execution/redisPositionFeed.cppm | 132 ++ source/live/execution/redisTradeGate.cppm | 60 + source/live/liveCommand.cppm | 231 ++++ source/live/monitoring/liveReporter.cppm | 169 +++ source/live/monitoring/liveTrace.cppm | 253 ++++ source/live/winners/liveStrategyCache.cppm | 124 ++ source/live/winners/liveWinners.cppm | 611 +++++++++ .../config/fvgStrategy/fvgStrategySweep.cppm | 78 ++ .../config/fvgStrategy/makeFvgStrategy.cppm | 70 + .../keltnerFadeStrategySweep.cppm | 59 + .../makeKeltnerFadeStrategy.cppm | 63 + .../liquiditySweepReversalStrategySweep.cppm | 81 ++ .../makeLiquiditySweepReversalStrategy.cppm | 74 ++ .../makeNyOpenRangeBreakoutStrategy.cppm | 74 ++ .../nyOpenRangeBreakoutStrategySweep.cppm | 70 + .../makeOhlcBreakoutStrategy.cppm | 12 +- .../ohlcBreakoutStrategySweep.cppm | 54 +- .../config/randomStrategy/makeStrategy.cppm | 4 +- .../randomStrategy/randomStrategySweep.cppm | 14 +- .../makeRangeVelocityStrategy.cppm | 79 ++ .../rangeVelocityStrategySweep.cppm | 81 ++ .../load/config/runConfigurationBuilder.cppm | 68 +- .../makeSessionRangeBreakoutStrategy.cppm | 70 + .../sessionRangeBreakoutStrategySweep.cppm | 64 + .../makeSqueezeBreakoutStrategy.cppm | 70 + .../squeezeBreakoutStrategySweep.cppm | 73 ++ source/load/loadCommand.cppm | 124 +- source/load/redisLoader.cpp | 143 ++- source/load/redisLoader.hpp | 44 +- source/main.cpp | 11 + source/positions/positionSync.cppm | 516 ++++++++ source/positions/positionsCommand.cppm | 125 ++ source/run/execution/backtestRunner.cppm | 76 +- source/run/execution/runnerBridge.cpp | 86 +- source/run/execution/runnerBridge.hpp | 48 +- source/run/execution/tickCache.cppm | 226 ++++ source/run/operations.cppm | 313 ++++- source/run/queue/drainRuns.cpp | 259 +++- source/run/queue/drainRuns.hpp | 10 +- source/run/queue/redisRunner.hpp | 11 +- source/run/queue/rollingWindow.cppm | 132 ++ source/run/queue/runQueue.cpp | 25 +- source/run/queue/runQueue.hpp | 31 +- source/run/reporting/elasticClient.cppm | 172 ++- source/run/reporting/elasticPublisher.cpp | 711 ++++++++++- source/run/reporting/elasticPublisher.hpp | 110 +- source/run/reporting/outcomeIndices.cpp | 42 + source/run/reporting/outcomeIndices.hpp | 77 ++ source/run/reporting/resultsSummary.cppm | 126 +- source/run/reporting/tradeDocument.cpp | 73 ++ source/run/reporting/tradeDocument.hpp | 65 + source/run/reporting/tradingResults.cpp | 24 +- source/run/reporting/tradingResults.hpp | 21 +- source/run/trading/reviewStopAndLimit.cppm | 8 +- source/run/trading/runLoop.cppm | 190 ++- source/run/trading/tradeManager.cppm | 59 +- source/shared/aws/dynamoAuth.cpp | 126 ++ source/shared/aws/dynamoAuth.hpp | 51 + source/shared/experiments/chainMatcher.cppm | 780 ++++++++++++ .../shared/experiments/experimentConfig.hpp | 203 +++ .../experimentRunConfiguration.hpp | 61 + source/shared/ig/igRestClient.cpp | 175 +++ source/shared/ig/igRestClient.hpp | 69 + source/shared/ipc/engineControl.hpp | 6 +- source/{ingest => shared/net}/tickPacket.cppm | 7 +- source/shared/net/udpPorts.hpp | 24 + source/{ingest => shared/net}/udpReceiver.cpp | 13 +- source/{ingest => shared/net}/udpReceiver.hpp | 14 +- source/shared/questdb/connectionFactory.cppm | 52 + source/shared/questdb/databaseConnection.cppm | 107 +- source/shared/questdb/sqlManager.cppm | 151 ++- source/shared/redis/apiRequestGate.cpp | 134 ++ source/shared/redis/apiRequestGate.hpp | 84 ++ .../shared/redis/client/syncRedisClient.cpp | 83 ++ .../shared/redis/client/syncRedisClient.hpp | 132 ++ .../redis/connection/redisConnection.cpp | 7 + source/shared/redis/positionClustering.cpp | 405 ++++++ source/shared/redis/positionClustering.hpp | 267 ++++ source/shared/redis/positionManager.cpp | 535 ++++++++ source/shared/redis/positionManager.hpp | 209 +++ source/shared/redis/tradeLocks.cpp | 144 +++ source/shared/redis/tradeLocks.hpp | 85 ++ source/shared/tradingDefinitions.hpp | 1 + .../config/configuration.hpp | 27 +- .../config/runConfiguration.hpp | 48 + .../tradingDefinitions/strategyConfig.hpp | 35 +- .../variables/fvgVariables.hpp | 43 + .../variables/keltnerFadeVariables.hpp | 34 + .../liquiditySweepReversalVariables.hpp | 46 + .../nyOpenRangeBreakoutVariables.hpp | 38 + .../variables/ohlcBreakoutVariables.hpp | 10 +- .../variables/ohlcVariables.hpp | 33 +- .../variables/rangeBarVariables.hpp | 50 + .../variables/rangeVelocityVariables.hpp | 41 + .../sessionRangeBreakoutVariables.hpp | 33 + .../variables/squeezeBreakoutVariables.hpp | 40 + .../variables/strategyVariables.cpp | 80 ++ .../variables/strategyVariables.hpp | 14 + .../variables/tradingVariables.hpp | 36 +- source/shared/utilities/atr.cppm | 71 ++ source/shared/utilities/backtestLog.cppm | 41 + source/shared/utilities/backtestLog.hpp | 74 +- source/shared/utilities/barStore.cppm | 221 ++++ source/shared/utilities/jsonParser.cpp | 8 + source/shared/utilities/jsonParser.hpp | 4 + source/shared/utilities/marketHours.cppm | 182 +++ source/shared/utilities/ohlcBuilder.cppm | 138 +- source/shared/utilities/queueKeys.hpp | 43 + source/shared/utilities/rangeBarBuilder.cppm | 295 +++++ source/shared/utilities/swingPivots.cppm | 74 ++ .../conditions/entryConditions.cppm | 160 +++ source/strategies/fvg/fvgStrategy.cppm | 295 +++++ .../keltnerFade/keltnerFadeStrategy.cppm | 216 ++++ .../liquiditySweepReversalStrategy.cppm | 398 ++++++ .../nyOpenRangeBreakoutStrategy.cppm | 267 ++++ .../ohlcBreakout/ohlcBreakoutStrategy.cppm | 190 +-- .../randomStrategy/randomStrategy.cppm | 15 +- .../rangeVelocity/rangeVelocityStrategy.cppm | 283 +++++ .../sessionRangeBreakoutStrategy.cppm | 249 ++++ .../squeezeBreakoutStrategy.cppm | 290 +++++ source/strategies/strategy.cppm | 17 +- source/strategies/strategyFactory.cppm | 77 ++ source/strategies/timeCapExit.cppm | 40 + source/tracking/dealPacket.cppm | 175 +++ source/tracking/trackingCommand.cppm | 215 ++++ source/tracking/trackingReport.cppm | 173 +++ tests/CMakeLists.txt | 36 + tests/atr.cpp | 98 ++ tests/barStore.cpp | 240 ++++ tests/chainMatcher.cpp | 569 +++++++++ tests/db.cpp | 53 + tests/dealPacket.cpp | 200 +++ tests/entryConditions.cpp | 156 +++ tests/experimentConfig.cpp | 262 ++++ tests/experimentSweep.cpp | 147 +++ tests/fvg.cpp | 582 +++++++++ tests/igRequests.cpp | 598 +++++++++ tests/jsonParser.cpp | 92 +- tests/keltnerFade.cpp | 350 +++++ tests/liquiditySweepReversal.cpp | 639 ++++++++++ tests/liveRunner.cpp | 847 ++++++++++++ tests/liveStrategyCache.cpp | 105 ++ tests/liveTrace.cpp | 272 ++++ tests/liveWinners.cpp | 950 ++++++++++++++ tests/marketHours.cpp | 212 +++ tests/nyOpenRangeBreakout.cpp | 415 ++++++ tests/ohlc.cpp | 93 +- tests/ohlcBreakout.cpp | 207 ++- tests/orderChannel.cpp | 625 +++++++++ tests/orderRequest.cpp | 129 ++ tests/outcomeIndices.cpp | 138 ++ tests/positionBook.cpp | 161 +++ tests/positionClustering.cpp | 218 ++++ tests/positionCounterCache.cpp | 76 ++ tests/positionManager.cpp | 156 +++ tests/positionSync.cpp | 303 +++++ tests/rangeBar.cpp | 326 +++++ tests/rangeVelocity.cpp | 555 ++++++++ tests/rollingWindow.cpp | 149 +++ tests/sessionRangeBreakout.cpp | 392 ++++++ tests/sqlManager.cpp | 74 ++ tests/squeezeBreakout.cpp | 442 +++++++ tests/sweep.cpp | 1132 ++++++++++++++++- tests/swingPivots.cpp | 94 ++ tests/tickCache.cpp | 268 ++++ tests/tickPacket.cpp | 38 +- tests/trackingReport.cpp | 287 +++++ tests/tradeDocument.cpp | 113 ++ tests/tradeLocks.cpp | 25 + tests/tradeManager.cpp | 515 +++++++- 211 files changed, 32970 insertions(+), 958 deletions(-) create mode 100644 ARCHITECTURE.md create mode 100644 ENVIRONMENT.md create mode 100644 QUICKSTART.md create mode 100644 REQUIREMENTS.md create mode 160000 external/aws mode change 100644 => 100755 scripts/build.sh mode change 100644 => 100755 scripts/build_dep.sh create mode 100755 scripts/live.sh create mode 100644 source/analysis/analysisCommand.cppm create mode 100644 source/analysis/queue/analysisBridge.cpp create mode 100644 source/analysis/queue/analysisBridge.hpp create mode 100644 source/analysis/queue/analysisRunner.cpp create mode 100644 source/analysis/queue/analysisRunner.hpp create mode 100644 source/analysis/queue/drainExperiments.cpp create mode 100644 source/analysis/queue/drainExperiments.hpp create mode 100644 source/analysis/reporting/experimentElastic.cppm create mode 100644 source/analysis/reporting/experimentResults.hpp create mode 100644 source/experiments/config/dipRecovery/dipRecoverySweep.cppm create mode 100644 source/experiments/config/dipRecovery/makeDipRecovery.cppm create mode 100644 source/experiments/experimentsCommand.cppm delete mode 100644 source/ingest/udpPorts.hpp create mode 100644 source/live/broker/igMarkets.cppm create mode 100644 source/live/broker/igRequests.cppm create mode 100644 source/live/broker/marketDefinitions.cppm create mode 100644 source/live/config/liveSettings.cppm create mode 100644 source/live/execution/brokerOrderSink.cppm create mode 100644 source/live/execution/liveStrategyRunner.cppm create mode 100644 source/live/execution/orderChannel.cppm create mode 100644 source/live/execution/orderRequest.cppm create mode 100644 source/live/execution/redisPositionCounter.cppm create mode 100644 source/live/execution/redisPositionFeed.cppm create mode 100644 source/live/execution/redisTradeGate.cppm create mode 100644 source/live/liveCommand.cppm create mode 100644 source/live/monitoring/liveReporter.cppm create mode 100644 source/live/monitoring/liveTrace.cppm create mode 100644 source/live/winners/liveStrategyCache.cppm create mode 100644 source/live/winners/liveWinners.cppm create mode 100644 source/load/config/fvgStrategy/fvgStrategySweep.cppm create mode 100644 source/load/config/fvgStrategy/makeFvgStrategy.cppm create mode 100644 source/load/config/keltnerFadeStrategy/keltnerFadeStrategySweep.cppm create mode 100644 source/load/config/keltnerFadeStrategy/makeKeltnerFadeStrategy.cppm create mode 100644 source/load/config/liquiditySweepReversalStrategy/liquiditySweepReversalStrategySweep.cppm create mode 100644 source/load/config/liquiditySweepReversalStrategy/makeLiquiditySweepReversalStrategy.cppm create mode 100644 source/load/config/nyOpenRangeBreakoutStrategy/makeNyOpenRangeBreakoutStrategy.cppm create mode 100644 source/load/config/nyOpenRangeBreakoutStrategy/nyOpenRangeBreakoutStrategySweep.cppm create mode 100644 source/load/config/rangeVelocityStrategy/makeRangeVelocityStrategy.cppm create mode 100644 source/load/config/rangeVelocityStrategy/rangeVelocityStrategySweep.cppm create mode 100644 source/load/config/sessionRangeBreakoutStrategy/makeSessionRangeBreakoutStrategy.cppm create mode 100644 source/load/config/sessionRangeBreakoutStrategy/sessionRangeBreakoutStrategySweep.cppm create mode 100644 source/load/config/squeezeBreakoutStrategy/makeSqueezeBreakoutStrategy.cppm create mode 100644 source/load/config/squeezeBreakoutStrategy/squeezeBreakoutStrategySweep.cppm create mode 100644 source/positions/positionSync.cppm create mode 100644 source/positions/positionsCommand.cppm create mode 100644 source/run/execution/tickCache.cppm create mode 100644 source/run/queue/rollingWindow.cppm create mode 100644 source/run/reporting/outcomeIndices.cpp create mode 100644 source/run/reporting/outcomeIndices.hpp create mode 100644 source/run/reporting/tradeDocument.cpp create mode 100644 source/run/reporting/tradeDocument.hpp create mode 100644 source/shared/aws/dynamoAuth.cpp create mode 100644 source/shared/aws/dynamoAuth.hpp create mode 100644 source/shared/experiments/chainMatcher.cppm create mode 100644 source/shared/experiments/experimentConfig.hpp create mode 100644 source/shared/experiments/experimentRunConfiguration.hpp create mode 100644 source/shared/ig/igRestClient.cpp create mode 100644 source/shared/ig/igRestClient.hpp rename source/{ingest => shared/net}/tickPacket.cppm (96%) create mode 100644 source/shared/net/udpPorts.hpp rename source/{ingest => shared/net}/udpReceiver.cpp (89%) rename source/{ingest => shared/net}/udpReceiver.hpp (75%) create mode 100644 source/shared/questdb/connectionFactory.cppm create mode 100644 source/shared/redis/apiRequestGate.cpp create mode 100644 source/shared/redis/apiRequestGate.hpp create mode 100644 source/shared/redis/client/syncRedisClient.cpp create mode 100644 source/shared/redis/client/syncRedisClient.hpp create mode 100644 source/shared/redis/positionClustering.cpp create mode 100644 source/shared/redis/positionClustering.hpp create mode 100644 source/shared/redis/positionManager.cpp create mode 100644 source/shared/redis/positionManager.hpp create mode 100644 source/shared/redis/tradeLocks.cpp create mode 100644 source/shared/redis/tradeLocks.hpp create mode 100644 source/shared/tradingDefinitions/variables/fvgVariables.hpp create mode 100644 source/shared/tradingDefinitions/variables/keltnerFadeVariables.hpp create mode 100644 source/shared/tradingDefinitions/variables/liquiditySweepReversalVariables.hpp create mode 100644 source/shared/tradingDefinitions/variables/nyOpenRangeBreakoutVariables.hpp create mode 100644 source/shared/tradingDefinitions/variables/rangeBarVariables.hpp create mode 100644 source/shared/tradingDefinitions/variables/rangeVelocityVariables.hpp create mode 100644 source/shared/tradingDefinitions/variables/sessionRangeBreakoutVariables.hpp create mode 100644 source/shared/tradingDefinitions/variables/squeezeBreakoutVariables.hpp create mode 100644 source/shared/utilities/atr.cppm create mode 100644 source/shared/utilities/backtestLog.cppm create mode 100644 source/shared/utilities/barStore.cppm create mode 100644 source/shared/utilities/marketHours.cppm create mode 100644 source/shared/utilities/rangeBarBuilder.cppm create mode 100644 source/shared/utilities/swingPivots.cppm create mode 100644 source/strategies/conditions/entryConditions.cppm create mode 100644 source/strategies/fvg/fvgStrategy.cppm create mode 100644 source/strategies/keltnerFade/keltnerFadeStrategy.cppm create mode 100644 source/strategies/liquiditySweepReversal/liquiditySweepReversalStrategy.cppm create mode 100644 source/strategies/nyOpenRangeBreakout/nyOpenRangeBreakoutStrategy.cppm create mode 100644 source/strategies/rangeVelocity/rangeVelocityStrategy.cppm create mode 100644 source/strategies/sessionRangeBreakout/sessionRangeBreakoutStrategy.cppm create mode 100644 source/strategies/squeezeBreakout/squeezeBreakoutStrategy.cppm create mode 100644 source/strategies/strategyFactory.cppm create mode 100644 source/strategies/timeCapExit.cppm create mode 100644 source/tracking/dealPacket.cppm create mode 100644 source/tracking/trackingCommand.cppm create mode 100644 source/tracking/trackingReport.cppm create mode 100644 tests/atr.cpp create mode 100644 tests/barStore.cpp create mode 100644 tests/chainMatcher.cpp create mode 100644 tests/dealPacket.cpp create mode 100644 tests/entryConditions.cpp create mode 100644 tests/experimentConfig.cpp create mode 100644 tests/experimentSweep.cpp create mode 100644 tests/fvg.cpp create mode 100644 tests/igRequests.cpp create mode 100644 tests/keltnerFade.cpp create mode 100644 tests/liquiditySweepReversal.cpp create mode 100644 tests/liveRunner.cpp create mode 100644 tests/liveStrategyCache.cpp create mode 100644 tests/liveTrace.cpp create mode 100644 tests/liveWinners.cpp create mode 100644 tests/marketHours.cpp create mode 100644 tests/nyOpenRangeBreakout.cpp create mode 100644 tests/orderChannel.cpp create mode 100644 tests/orderRequest.cpp create mode 100644 tests/outcomeIndices.cpp create mode 100644 tests/positionBook.cpp create mode 100644 tests/positionClustering.cpp create mode 100644 tests/positionCounterCache.cpp create mode 100644 tests/positionManager.cpp create mode 100644 tests/positionSync.cpp create mode 100644 tests/rangeBar.cpp create mode 100644 tests/rangeVelocity.cpp create mode 100644 tests/rollingWindow.cpp create mode 100644 tests/sessionRangeBreakout.cpp create mode 100644 tests/sqlManager.cpp create mode 100644 tests/squeezeBreakout.cpp create mode 100644 tests/swingPivots.cpp create mode 100644 tests/tickCache.cpp create mode 100644 tests/trackingReport.cpp create mode 100644 tests/tradeDocument.cpp create mode 100644 tests/tradeLocks.cpp diff --git a/.gitignore b/.gitignore index e71ada3..cf0ee80 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,7 @@ dump.rdb .infisical.json cmake-build-debug .idea/ -.clang-tidy \ No newline at end of file +.clang-tidy +# AWS SDK for C++ install prefix (built per-node by scripts/build_dep.sh +# from the external/aws submodule) +external/aws-install/ diff --git a/.gitmodules b/.gitmodules index 27faf9d..d677d5f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,12 @@ +[submodule "external/boost-decimal"] + path = external/boost-decimal + url = https://github.com/boostorg/decimal.git [submodule "external/libpqxx"] path = external/libpqxx url = https://github.com/jtv/libpqxx.git -[submodule "external/boost-decimal"] - path = external/boost-decimal - url = https://github.com/boostorg/decimal [submodule "external/Catch2"] path = external/Catch2 url = https://github.com/catchorg/Catch2.git +[submodule "external/aws"] + path = external/aws + url = https://github.com/aws/aws-sdk-cpp diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..8c50200 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,346 @@ +# Architecture + +How the engine is put together and how data flows through it. Commands and usage are in [QUICKSTART.md](QUICKSTART.md); configuration in [ENVIRONMENT.md](ENVIRONMENT.md). + +The binary (`source/main.cpp`) dispatches eight subcommands: `load`, `run`, `experiments`, `analysis`, `ingest`, `live`, `tracking`, and `positions`. + +## System context + +```mermaid +flowchart LR + subgraph external[External processes] + CS[IG streamer] + PY[Python monitor] + end + subgraph engine[BacktestingEngine subcommands] + IN[ingest] + LO[load] + RU[run] + EX[experiments] + AN[analysis] + LI[live] + TR[tracking] + PO[positions] + end + subgraph services[Services] + RD[(Redis)] + QD[(QuestDB)] + ES[(Elasticsearch)] + IG[IG REST API] + DY[(AWS DynamoDB)] + end + CS -- UDP 11111 ticks --> IN + IN -- ILP over HTTP 9000 --> QD + CS -- UDP 11110 ticks --> LI + CS -- UDP 11112 deals --> TR + TR -- PO# to PH# archive, book prune --> RD + TR -- live-trades audit docs --> ES + LO -- sweep payloads --> RD + RD -- work queue --> RU + QD -- ticks, pgwire 8812 --> RU + RU -- results --> ES + RU -- rolling window requeue --> RD + EX -- experiment payloads --> RD + RD -- experiment queue --> AN + QD -- ticks, pgwire 8812 --> AN + QD -- OHLC and range-bar warm-up at startup --> LI + AN -- aggregate docs --> ES + ES -- winning runs --> LI + LI -- locks, caps, position book --> RD + IG -- open positions --> PO + PO -- position book + cluster sets --> RD + PO -- cycle reports --> ES + LI -- orders --> IG + DY -- IG session tokens --> LI + DY -- IG session tokens --> PO + RU -- shared memory control --- PY +``` + +External to this repo: the IG streamer (an IG Lightstreamer client that publishes the tick feed on both tick ports and the account's deal updates on 11112), the IG login service (keeps session tokens fresh in DynamoDB), and the Python monitor/controller. The live position book and the `CG#` cluster membership sets are maintained in-engine by the `positions` subcommand ([Positions](#positions)) — the C++ replacement for the external C# position-producer cron. + +## Source layout + +| Directory | Contents | +| --- | --- | +| `source/main.cpp` | Subcommand router | +| `source/load/` | Sweep expansion and queueing: `loadCommand`, `runConfigurationBuilder`, one sweep-builder/factory pair per strategy under `config/` (`random`, `ohlcBreakout`, `fvg`, `keltnerFade`, `sessionRangeBreakout`, `squeezeBreakout`, `nyOpenRangeBreakout`, `liquiditySweepReversal`, `rangeVelocity`), `sweep/` (`parameterGenerator`, `parameterRange`, `sweepCombination`), `utility/symbolGroups` (symbol-group tokenising/validation behind the one-run-per-group contract), `redisLoader` | +| `source/run/` | Backtest worker: `runCommand`, `queue/` (`drainRuns` drain loop, `runQueue`/`redisRunner` queue primitives, `rollingWindow`), `backtestRunner`, `runnerBridge`/`tickCache` (cross-run tick superset cache), `operations`, `trading/` (`runLoop`, `tradeManager`, `reviewStopAndLimit`, `exitRules`), `reporting/` (`elasticPublisher`, `elasticClient` results-vs-winners routing, `outcomeIndices` weekly index/alias naming, `resultsSummary` performanceScore/calmarScore/maxDrawdownPercent, `tradingResults`, `tradeDocument`, `engineException`) | +| `source/experiments/` | Experiment producer: `experimentsCommand` (loadCommand's mirror for the experiment queue), one sweep-builder/factory pair per experiment under `config/` (`dipRecovery`) | +| `source/analysis/` | Experiment worker: `analysisCommand`, `analysisRunner`/`drainExperiments` (queue loop, drainRuns' mirror), `analysisBridge` (module-import seam), `experimentResults`/`experimentElastic` (aggregate-doc reporting) | +| `source/live/` | Live trading: `liveCommand`, `liveSettings`, `liveWinners`/`liveStrategyCache` (selection), `liveStrategyRunner` (workers), `brokerOrderSink`/`orderChannel`/`orderRequest` (order path), `redisTradeGate`/`redisPositionCounter`/`redisPositionFeed` (Redis seams), `broker/` (IG market calls), `monitoring/` (`liveReporter`, and `liveTrace` — structured live tracing to `live-traces`/`live-logs`, gated by `LIVE_TRACE_ENABLED`/`LIVE_LOG_SHIP_ENABLED`, imported by the order channel, sink, runner, `igRequests`, and `liveCommand`) | +| `source/ingest/` | UDP tick receiver → QuestDB ILP writer: `ingestCommand`, `questdbIngestClient` | +| `source/tracking/` | UDP deal receiver → per-deal enrichment: `trackingCommand`, `dealPacket` (256-byte deal wire format + decoder), `trackingReport` (the pure half: `PH#`/`PO#` lookup, close-pip calc, the `live-trades` document) | +| `source/positions/` | IG position producer: `positionsCommand` (minute loop, `--once` mode), `positionSync` (one sync cycle: `/positions` fetch + decode, `PO#`/`PL#` save/update, `CG#` cluster rebuild, `PL#` refresh, Elastic reports) | +| `source/strategies/` | `IStrategy` interface, `strategyFactory` (single factory + the `kActiveStrategies` live-eligibility list), the shared ATR entry gate (`conditions/entryConditions`), the shared time-cap exit (`timeCapExit`), `strategyErrors`, and the nine strategies: `randomStrategy`, `ohlcBreakout`, `fvg`, `keltnerFade`, `sessionRangeBreakout`, `squeezeBreakout`, `nyOpenRangeBreakout`, `liquiditySweepReversal`, `rangeVelocity` | +| `source/shared/` | Cross-cutting: `utilities/` (queue keys, base64, env, JSON, decimal JSON, OHLC builder + bar store, range-bar builder — the event-driven bar engine behind `rangeVelocity`, swing pivots — behind `liquiditySweepReversal`, EMA, ATR, symbol scale, market hours, logging, thread pool), `models/`, `net/` (UDP receiver, tick packet, `udpPorts` — the single source of truth for 11110/11111/11112), `questdb/`, `redis/` (client + trade locks, position manager, position clustering, API request gate), `ig/` (REST client), `aws/` (DynamoDB auth), `ipc/` (shared-memory control channel), `tradingDefinitions/` (config structs), `experiments/` (experiment model + the pure `chainMatcher`) | + +## Backtest pipeline + +### Queueing (`load`) + +`load` expands a parameter sweep (combinations × symbol groups) and pushes one *run* per symbol group. Everything on the wire is base64-encoded JSON. + +```mermaid +sequenceDiagram + participant L as LoadCommand + participant R as Redis + Note over L: confirm sweep size on stdin first + loop per symbol group (one RUN_ID each) + L->>R: SET one payload key per combination (7 day TTL) + L->>R: LPUSH payload key names onto the run's strategy list + L->>R: LPUSH run descriptor onto BACKTESTING_QUEUE_RUN + end +``` + +Order matters: payloads exist before their names are visible, and the strategy list is complete before the run is advertised — a worker that sees the run can immediately drain it. + +| Redis key | Type | Contents | +| --- | --- | --- | +| `BACKTESTING_QUEUE_RUN` | list | Run descriptors (symbols, date window, risk limits) — head of the run-queue ladder; fresh sweeps and hand-queued one-offs land here | +| `BACKTESTING_QUEUE_RUN_CHAIN:1..3` | list | Run descriptors re-queued by the rolling-window ladder, one queue per rung (`rolling::queueKeyFor`) | +| `BACKTESTING_QUEUE_STRATEGY:` | list | Payload key names for that run | +| `BACKTESTING_QUEUE_STRATEGY_PAYLOAD::` | string | One strategy config, 7-day safety-net TTL | + +### Draining (`run`) + +```mermaid +flowchart TD + P[Peek oldest run: LINDEX -1 down the queue ladder, first non-empty wins] --> V{descriptor parseable?} + V -- no --> PP[Retire poison pill: LREM by raw value] --> P + V -- yes --> C[Claim one strategy name: RPOP strategy list] + C -- nil --> RET[Retire drained run: LREM] --> P + C -- name --> T[Get tick slice: cached superset or QuestDB load] + T --> G[GETDEL payload, parse config, submit backtest to thread pool] + G --> C2[RPOP next name] + C2 -- name --> G + C2 -- nil --> RET +``` + +The queue semantics are deliberate: + +- **Runs live on a four-key strict-priority ladder.** `queue_keys::RUN_QUEUES` is `BACKTESTING_QUEUE_RUN` followed by `BACKTESTING_QUEUE_RUN_CHAIN:1..3` (one queue per rolling-window rung); the peek (`runQueue`'s `peekRunTail`) walks them in order and the first non-empty queue wins, so a fresh grid sweep always preempts the chained single-strategy backlog — and `LLEN` per key reads wave progress. +- **Run descriptors are shared adverts, not owned work items.** They are *peeked* (`LINDEX -1`), never popped, so any number of `run` processes can discover the same run and drain its strategy list in parallel. The strategy names are the exclusive work units — `RPOP` hands each to exactly one worker, and `GETDEL` consumes each payload exactly once. +- **Retirement is idempotent.** `LREM 0 ` removes the descriptor by value from the queue it was peeked from; when several workers converge on a drained run, the first `LREM` wins and the rest are no-ops. No distributed lock needed. +- **Crash safe.** A worker dying mid-run leaves the descriptor in place; the next peek re-discovers it. An unparseable descriptor is retired by its raw bytes without being parsed, so a poison pill can't wedge the queue. +- **Wasted work is bounded.** A worker claims its first strategy *before* any tick fetch, so losing the race on a nearly-drained run costs nothing. +- **Backtests pipeline across runs.** A run is retired as soon as its strategy list is drained — its backtests may still be executing while the worker moves on to the next run. The pool only quiesces immediately before a genuine QuestDB superset load (the cache's `beforeLoad` hook), not between runs. + +The worker sizes its thread pool at 80% of hardware threads and exposes an `active_jobs` gauge / `stop_signal` flag to the Python monitor over a memory-mapped file (`/tmp/EngineControlShm`, 12-byte magic + gauge + stop flag — `source/shared/ipc/engineControl`). + +### Tick superset cache + +The rolling-window ladder makes every surviving strategy its own single-strategy run, so a naive worker would pay a full QuestDB load per window. Instead, `runnerBridge` keeps a per-process `TickCache` (`source/run/execution/tickCache.cppm`): one months-deep **superset** of ticks per symbol set, with the month boundaries computed by QuestDB itself in the same statement (a single `now()`, so data and boundaries can't disagree). Each `(LAST_MONTHS, OFFSET_MONTHS)` window is then a contiguous slice of the superset, found by binary search on the boundaries. + +- Supersets are TTL'd and size-capped (`TICK_CACHE_TTL_MINUTES`, `TICK_CACHE_MAX_SUPERSETS` — see [ENVIRONMENT.md](ENVIRONMENT.md)); a window too deep for the resident superset falls back to an ad-hoc load without evicting it. +- Slices hold the superset alive via `shared_ptr`, so eviction can't pull the tick buffer out from under an in-flight backtest. +- The cache is single-threaded by design: only the drain loop touches it; pool threads only hold slice copies. Direct mode (`run `) bypasses it entirely. + +### Backtest execution + +Per strategy, `Operations::run` builds a `TradeManager` and the strategy instance, then `trading::runTicks` drives the tick loop: + +```mermaid +flowchart TD + T[Next tick] --> M[Mark open trades to market] + M --> S[Review stop and limit exits] + S --> L{Equity at or below loss floor?} + L -- yes --> X[Close all trades: LossLimitBreached] + L -- no --> G{Entry gates: market session, one per symbol, max open trades, trades per minute} + G -- blocked --> D2[Strategy during hook] --> T + G -- clear --> A{ATR entry conditions} + A -- rejected --> D2 + A -- clear --> DE{Strategy decide} + DE -- long or short --> O[Open trade] --> D2 + DE -- no signal --> D2 +``` + +With `PEAK_HOURS_ONLY` in the run config (set for newly queued sweeps), entries are only taken during the symbol's peak sessions (`marketHours`, all UTC: Asia 00:00–06:00, three hours from the London open, three hours from the New York open; weekends and unknown symbols blocked, DST handled). Only entries are gated — exits, mark-to-market, and the strategy `during` hook always run. + +The ATR entry gate (`source/strategies/conditions/entryConditions.cppm`, shared verbatim with `live`) sits after the caps and before `decide()`: the entry is skipped unless the ATR(10) gate series is warm and the current spread is at most 30% of the ATR; the strategy's ATR-multiple stop/limit distances are then converted to pips and clamped (stop 10–80, limit 3–300 pips). + +Fills model the spread honestly — a LONG opens at the ask and exits on the bid (vice versa for SHORT), the spread is booked as an equity dip at open, and gap-through stops fill at the actual tick price. An optional slippage stress (`ENTRY_SLIPPAGE_TENTH_PIPS`, frozen into the run config at `load` time — see [ENVIRONMENT.md](ENVIRONMENT.md)) additionally worsens every entry fill by a fixed fraction of a pip against the trade, leaving the stop/limit anchors on the raw tick. Commission and overnight funding are not modeled — see [documents/technicalDebtAudit.md](documents/technicalDebtAudit.md) for the full realism notes. + +### Rolling-window chaining + +A run that completes its window automatically re-queues the same strategy config, same UUID, under a fresh RUN_ID for the next window — walking the strategy forward through history: + +```mermaid +flowchart LR + W1[3 months, offset 0] --> W2[3 months, offset 3] --> W3[3 months, offset 6] --> W4[9 months, offset 0] +``` + +The 9-month rung is exported as `rolling::kFullHistory` — the terminal full-history run that `live` selects winners from; the shorter rungs are screening only. + +"Completes" embeds a performance gate (`operations.cppm`): a finished run only counts as `Completed` when its performance score clears 5 on more than 5 decisive trades. A run that exhausts its ticks but misses the gate finishes as `Underperformed` — it neither chains nor reaches the results/winners indices, landing only in the final-record index (one lucky winner on a quiet window is a sample, not a strategy); a liquidated run stops the same way. The re-queued advert does not go back onto `BACKTESTING_QUEUE_RUN`: it lands on the next rung's own chain queue (`rolling::queueKeyFor` → `BACKTESTING_QUEUE_RUN_CHAIN:`), so fresh sweeps and earlier rungs always outrank the chained backlog. + +### Results reporting + +Results go to Elasticsearch (`elasticPublisher`, retry with backoff, NDJSON dead-letter file on exhaustion). Reporting documents are queued in memory and delivered by a background flusher thread in periodic `_bulk` batches (`ELASTIC_FLUSH_SECONDS`, default 30s; a final flush runs at process exit) — publishing never blocks a worker thread, and fast sweeps produce a handful of bulk requests instead of a per-run PUT storm: + +The outcome indices are **weekly**: each `load` mints a batch label — `$BACKTEST_BATCH` or the current UTC ISO week, e.g. `2026-28` — stamps it into every run descriptor (`BATCH`/`EXECUTION_TS`, carried through Redis and every rolling-window rung), creates that week's indices, and atomically repoints the rolling `-current` aliases at them. One week's sweep therefore lands in its own index set while every earlier week stays untouched and searchable via the `backtesting-*` pattern, and outcome documents carry top-level `batch` + `executionTimestamp` fields. A config with an empty `BATCH` (pre-batch Redis payloads, hand-run configs) writes to the unsuffixed base names with no alias admin. The retired `trading_*` indices are a frozen archive — nothing writes to them anymore. + +| Index (weekly, + `-current` alias) | Written when | +| --- | --- | +| `backtesting-final-YYYY-WW` | Every run (id `RUN_ID:UUID`) | +| `backtesting-results-YYYY-WW` | Completed runs on the screening rungs — carries `results.performanceScore` | +| `backtesting-winners-YYYY-WW` | Completed runs over the terminal full-history window (`rolling::kFullHistory`) — the population `live` selects from | +| `backtesting-failures-YYYY-WW` | Loss-limit cutoffs (if the run opts in) | +| `backtesting-trades` (static) | Every closed trade (opt-in via `ELASTIC_TRADES_ENABLED=1`) | +| `backtesting-experiments-YYYY-WW` | One aggregate doc per experiment × symbol group (written by `analysis`, index/alias prepared by `experiments` — not by `load`) | +| `engine_exceptions` (static) | Failures anywhere in the pipeline | +| `live-trades` (static) | Live order audit trail (written by `live` — the order channel via `igRequests` — and by `tracking`, one document per deal update; not `run`) | +| `live-traces` (static) | Structured live trace events (written by `live` via `liveTrace`, gated by `LIVE_TRACE_ENABLED`, default on) | +| `live-logs` (static) | Every log line, shipped through the `backtest_log` sink (gated by `LIVE_LOG_SHIP_ENABLED`, default on) | +| `live-function-logs` (static) | The `positions` producer's cycle reports, carrying the full IG `/positions` snapshot | + +## Experiments + +`experiments` / `analysis` ask **occurrence-rate questions** of the tick history — "price drops 1% over 10 minutes, then recovers 0.5% in a further 10 minutes: how often, and how far does it run?" — without writing a strategy. The pair reuses the backtest queue architecture wholesale (same `RedisLoader`, same peek/RPOP/GETDEL/LREM semantics, same batch/weekly-index doctrine) on its own key family, so the two pipelines never contend: + +| Redis key | Type | Contents | +| --- | --- | --- | +| `BACKTESTING_QUEUE_EXPERIMENT_RUN` | list | Experiment run descriptors (symbols, tick window, batch) — a single queue: experiments never chain windows, so there is no `RUN_CHAIN` ladder here | +| `BACKTESTING_QUEUE_EXPERIMENT:` | list | Payload key names for that run | +| `BACKTESTING_QUEUE_EXPERIMENT_PAYLOAD::` | string | One experiment config, 7-day safety-net TTL | + +An **experiment** is a chain of activities counted non-overlapping against the tick stream. Four primitives exist (`source/shared/experiments/experimentConfig.hpp`): `DirectionalMove` (signed percent within a window), `StaysInBand` (trailing range within a width), `NewExtreme` (strict new high/low vs a lookback), and `RangeRelativeMove` (a move sized in multiples of the trailing high–low range). The matcher (`source/shared/experiments/chainMatcher.cppm`) is pure and I/O-free: leg 1 is *rolling* (trailing-window extremes via monotonic deques with timestamp expiry — the `rangeBarBuilder` pattern adapted to time), later legs anchor at the previous leg's completion tick, and all per-tick arithmetic is int64. Band/lookback legs require gap-free tick *coverage*, so a weekend gap can never satisfy "stays in band". The full semantics (touch-within-window, expiry, serial attempts, documented undercounts) are stated in the module comment and pinned by `tests/chainMatcher.cpp`. + +`analysis ` is `run`'s mirror for this queue: peek → claim-before-tick-load → one QuestDB load per run shared across the pool → one **aggregate document** per experiment × symbol group into the weekly `backtesting-experiments` index. Beyond raw occurrence counts, the doc carries what turning a pattern into a strategy needs: `attempts`/`failuresByLeg`/`completionRate` (P(chain | leg 1) — serial single-anchor, so conservative in clustered periods), month and hour-of-day occurrence buckets, per-symbol coverage, MFE/MAE excursion quantiles for completed and failed attempts separately, completion-time quantiles, and the mean spread at trigger — MFE p50 → limit distance, failed-attempt MAE p90 → stop distance, completion-time p90 → time cap. + +Adding an experiment sweep: a factory/sweep pair under `source/experiments/config/`, one branch in `experimentsCommand.cppm`, tests in `tests/experimentSweep.cpp`. No worker or queue changes — the worker evaluates whatever chain arrives. + +## Live trading + +`live` turns the best backtest results into live IG positions. The broker owns positions and exits: the engine mirrors the position book *from* Redis (maintained by the `positions` subcommand — see [Positions](#positions)) and emits opens and closes — there is no local mark-to-market or stop/limit review. + +```mermaid +flowchart TD + ES[(Elasticsearch backtesting-winners-current alias)] -- winners above LIVE_MIN_SCORE and LIVE_MIN_CALMAR_SCORE, within LIVE_MAX_DRAWDOWN_PERCENT --> SC[StrategyCache: one WorkerSpec per winner] + UDP[UDP tick stream 11110] --> RX[UdpReceiver + tick decode] + RX --> RT[StrategyRunner routes by symbol to every worker on it] + RT --> W[Worker thread per strategy instance] + RD[(Redis position book)] -- throttled sync ~15s --> W + W --> DEC{Strategy decide} + DEC --> C1{Trade rate cap} + C1 --> C2{Open trade cap via Redis} + C2 --> C3{Redis trade lock SET NX} + C3 --> OS[BrokerOrderSink] + W -- closed-trade diff from strategy during --> CL[Close intent] + CL --> OS + OS --> OC[OrderChannel] + OC --> IG[IG REST API] +``` + +Winner selection requires the run to have covered the ladder's terminal full-history window (`rolling::kFullHistory`) above `LIVE_MIN_SCORE`, with `results.maxDrawdownPercent` at or under `LIVE_MAX_DRAWDOWN_PERCENT` and `results.calmarScore` at or above `LIVE_MIN_CALMAR_SCORE` — the shorter rungs never qualify, and a spiky run cannot buy its way past the drawdown gate on expectancy. Winners are fetched per (active strategy, symbol) pair, and every winner becomes its own dedicated worker thread with its own tick queue — the receiver thread fans each tick out to every worker on its symbol, so N winning strategies on one symbol mean N workers each seeing the full stream. Workers whose winning config carries `PEAK_HOURS_ONLY` skip `decide()` outside the symbol's peak market sessions (`marketHours`); book sync and close handling never pause. + +Every ENTRY gate fails closed: if Redis is unreachable the position count and lock checks block the entry rather than allowing it. The broker-request gate distinguishes the two order classes: **opens** fail closed on any Redis uncertainty and respect the shared 30/min soft budget, while **closes** are risk-reducing and fail *open* — Redis uncertainty and the soft budget never stop a close from reaching IG (an unsent close leaves live exposure, the one outcome worse than an extra request; only a definite duplicate marker paces it). The open-trade count is cached per worker for 15 seconds and invalidated the moment that worker's own open or close changes it. + +A portfolio-level clustering gate (`source/shared/redis/positionClustering`) sits in the order channel, mirroring the C# engine's `CheckForCluster` and sharing its Redis wire contract: every symbol maps to one or more risk clusters, and an entry is blocked when any of its clusters is at capacity, already holds the same (symbol, strategy), or — for strict clusters — any deal from the same strategy. The `CG#` membership sets are written only by the `positions` producer's minute sync; the gate reads them under a 10-second `CLUSTER_LOCK#` cooldown and, like the other gates, fails closed — an unmapped symbol or any Redis failure blocks the entry. The lock has a write side too: on the Accepted branch the order channel calls `PositionClustering::markOpened`, which unconditionally re-arms `CLUSTER_LOCK#` (plain `SET`, not `NX`) on every cluster of the symbol for 2 minutes — the new deal will not appear in `CG#` until the producer's next sync, and without the hold another strategy in the cluster could clear a capacity check against sets that predate it. + +### Order placement + +```mermaid +sequenceDiagram + participant W as Worker + participant OC as OrderChannel + participant R as Redis + participant DY as DynamoDB + participant IG as IG REST + participant ES as Elasticsearch + W->>OC: OrderIntent (level, stop, limit, dealReference) + OC->>OC: market allowlist lookup (unknown symbol dropped) + OC->>R: cluster exposure gate (CG# sets under CLUSTER_LOCK cooldown) + OC->>R: extend trade lock to in-flight TTL + OC->>R: dedup key + per-minute rate budget (30 requests/min, shared) + OC->>DY: pull IG session tokens (MarketDataLive table) + OC->>IG: POST /positions/otc + OC->>IG: GET /confirms/{dealReference} + alt accepted + OC->>R: save position payload, append to strategy book, save deal receipt + OC->>ES: audit document to live-trades + else rejected + OC->>R: release trade lock (early re-entry) + else failed or exception + OC->>R: extend trade lock 2 minutes (failure brake) + end +``` + +The open POST is never blind-retried (it is not idempotent — a lost ACK could double a position): it is sent exactly once, and a transport failure or gateway 5xx/408 is resolved by polling `GET /confirms/{dealReference}` under the reference the engine minted into the body. An open that cannot be confirmed maps to Failed and is not re-sent; the 2-minute failure TTL on the trade lock brakes re-entry while the `positions` sync reconciles anything that did land from the broker's own book. + +Closes follow the same channel with their own dedup key (`API#close#` — deliberately distinct from the open's key, so a fresh open's 30-second marker can never refuse the close that follows it): a blank-dealId guard and a 5-minute recent-close window suppress duplicates, then `POST /positions/otc` with `_method: DELETE`. A close the gate *refused* (never sent) maps to Failed and keeps the book entry, so the sync-driven retry loop re-fires it; a genuine transport failure after send maps to Gone and prunes the entry (the `positions` sync restores it from the broker book if it does still exist); on success the position payload is archived to history and pruned from the strategy book. + +### Live Redis keys + +| Key | Purpose | +| --- | --- | +| `LOCK##` | Per-(strategy, direction) trade lock (`SET NX PX`, TTL `LIVE_TRADE_LOCK_SECONDS`) | +| `PL#` | Strategy position book (list of open positions) | +| `PO#` | Position payload — booked under IG's echoed deal reference, not the deal id | +| `PH#` | Closed-position archive (60 days) | +| `DealId##` | Deal receipt (60 days) | +| `REQ#` | Per-minute IG API request budget (shared with the C# engine) | +| `API#` | Open-order dedup marker (30s) | +| `API#close#` | Close-order dedup marker (30s) — its own namespace so an open's marker never blocks its close | +| `CG#` | Cluster membership set (`symbol#strategy#dealReference`), rebuilt each minute by the `positions` producer (its only writer) — the clustering gate reads it | +| `CLUSTER_LOCK#` | 10-second cluster cooldown try-lock (`SET NX PX`), left to expire after each check; re-armed for 2 minutes (plain `SET`, `markOpened`) on every accepted open of a symbol in the cluster | + +## Ingest + +```mermaid +flowchart LR + UDP[UDP datagrams 11111] --> D[Decode and validate tick packet] + D --> Q[Bounded queue, drop oldest at 1M lines] + Q --> B[Background writer: batches of 1000 lines or 100ms] + B --> QD[(QuestDB ILP over HTTP 9000)] +``` + +Tick packets are a fixed 40-byte little-endian layout (`bid f64, ask f64, timestamp micros i64, symbol char[16]` — `source/shared/net/tickPacket.cppm`). Malformed, non-finite, out-of-range, or unknown-symbol ticks are dropped at decode. The written columns (`ask`, `bid`, `timestamp`, one table per symbol) are exactly what `run` reads back through `SqlManager`. + +## Tracking + +```mermaid +flowchart LR + UDP[UDP datagrams 11112] --> D[Decode 256-byte deal packet] + D --> LOG[One timestamped log line per deal] + D --> RD[(Redis: PH#/PO# lookup, archive + prune on DELETED)] + D --> ES[(Elasticsearch: live-trades document)] +``` + +When the IG Lightstreamer feed pushes a `TRADE:*` account update, the external streamer serialises it as one fixed 256-byte little-endian **deal packet** — four doubles (`level`, `size`, `stopLevel`, `limitLevel`, each NaN when absent) followed by NUL-padded ASCII fields (`dealReference`, `dealId`, `dealIdOrigin`, `epic`, `direction` `BUY|SELL`, `status` `OPEN|UPDATED|DELETED`, `dealStatus` `ACCEPTED|REJECTED`, `currency`, `channel`, `expiry`, IG's raw `timestamp` string, `guaranteedStop`) — and fires it at UDP 11112. The full byte map lives in `source/tracking/dealPacket.cppm`, the C++ end of the same hand-rolled wire contract style as the tick feed. + +`tracking` shares the ingest/live `UdpReceiver`, decodes each datagram, and logs the deal in full — deals are account-level events (a handful per day, not a tick stream). The only structural rule is the datagram length (anything not exactly 256 bytes is counted as dropped); field content passes through unjudged — unlike ticks there is no plausibility gate, because what a given status or absent level means is for the consumer to decide. The decode handler then enriches each deal (`trackingCommand`, with the pure half in `trackingReport`): it looks the deal up in Redis by its reference (`PH#` history first, `PO#` live book as fallback), computes close pips for a `DELETED` deal with a known position and a real close level, archives the book entry the moment the broker says `DELETED` (`PO#` → `PH#`, pruned from the `PL#` list — idempotent when the strategy-close path already moved it), and queues one audit document per deal to the `live-trades` Elasticsearch index. + +## Positions + +`positions` is the IG position **producer** — the in-process replacement for the external C# `igmarkets_positions` cron. Every minute (first cycle immediately; `--once` runs a single cycle and exits) it mirrors the broker's own `/positions` book into the Redis position store that `live` reads, keeping the broker the source of truth for what is actually open. + +```mermaid +flowchart TD + A[Pull IG session from DynamoDB] --> B[GET /positions from IG REST] + B --> C{Fetch and decode OK?} + C -- no --> F[FAILED report to live-function-logs, abandon cycle] + C -- yes --> M[Per position: match epic against the market allowlist] + M --> S[Strategy attribution via DealId# receipt, else Unknown] + S --> U{PO# record present and decodable?} + U -- yes --> UP[Refresh it: 10 min TTL] + U -- no --> SV[Build fresh record from broker fields: 30 min TTL] + UP --> PL[Ensure deal is in the PL# strategy book] + SV --> PL + PL --> CG[Rebuild CG# cluster sets: temp SADD, RENAME swap, 5 min TTL] + CG --> OK[Success report with the raw broker book] + OK --> RF[Refresh every PL# list: SCAN PL#*, prune expired deals] +``` + +The doctrine is TTL-driven: `PO#` payloads deliberately carry a short TTL and live only as long as the producer keeps re-stamping them, so a deal the broker no longer reports simply expires and falls out of its `PL#` book on the refresh pass — nothing has to observe an explicit close event. Updates get 10 minutes and fresh saves 30 (a brand-new deal's receipt and book entries may lag a cycle); during Friday 21:55–21:59 UTC the update TTL stretches to 2 days + 2 hours, so positions still open at the IG weekly close survive Redis until Sunday-night trading resumes. Any fetch-stage failure — no DynamoDB session, HTTP failure, undecodable body — files a `FAILED-*` report to `live-function-logs` and abandons the whole cycle: the refresh pass never prunes against a book that could not be read (an unreadable book is not an empty one). + +The `CG#` rebuild is the write-side counterpart of the clustering *gate* in the live order channel: the producer is the only writer (per non-empty cluster: SADD into a temp key, atomic `RENAME` over `CG#` so stale members vanish with the swap, then a 5-minute TTL so a cluster whose deals all closed simply expires) and the gate only reads. Strategy attribution comes from the `DealId##` receipts the order channel writes at open — a deal opened outside the engine attributes to `Unknown`. Each cycle files a Success report to `live-function-logs` carrying the raw `/positions` body — the ground-truth account snapshot — after the `CG#` rebuild and before the closing `PL#` refresh pass. + +Loop mechanics: no UDP receiver holds the main thread here, so the command loops directly — 1-second sleep ticks against a steady-clock deadline that is re-armed *after* each cycle completes (a slow IG exchange delays the next cycle rather than causing a catch-up burst), with SIGINT/SIGTERM honoured within about a second. + +## Strategies + +`IStrategy` (`source/strategies/strategy.cppm`) is two hooks: `decide(tick)` returns an optional entry direction, `during(tick, tradeManager)` runs every tick for bar-building and trade management. `strategies::makeStrategy(config)` is the single factory shared by backtest and live, so a `StrategyConfig` travels byte-identical (same UUID) from sweep → queue → backtest → rolling-window re-queue → Elasticsearch → live selection. + +Nine strategies exist: `randomStrategy` (random entries — a testing/baseline harness, not a candidate edge), `ohlcBreakout` (multi-bar range breakout, EMA trend filter), `fvg` (fair-value-gap retracement with a higher-timeframe SMA trend filter), `keltnerFade` (mean reversion — fades stretches beyond a volatility band), `sessionRangeBreakout` (London open-range breakout of the Asian-session range), `squeezeBreakout` (volatility contraction/expansion — breaks of a single compressed bar, EMA trend filter), `nyOpenRangeBreakout` (opening-range breakout anchored to the New York open), `liquiditySweepReversal` (sweep of a prior swing pivot followed by a displacement reversal), and `rangeVelocity` (range-bar run velocity — momentum measured in event-driven range bars rather than time bars). Every strategy except `random` carries a time-cap exit (`timeCapExit`) alongside SL/TP. `scripts/load.sh` queues only seven of the nine by default: `random` is excluded (baseline harness) and `keltnerFade` was retired in 2026-28 — its sweep module remains, so load it explicitly to re-test. + +Adding a strategy: one branch in `strategyFactory.cppm`, a sweep/factory pair under `source/load/config/`, a branch in the `load` command, and (for live eligibility) an entry in `kActiveStrategies` (also in `strategyFactory.cppm`). diff --git a/CMakeLists.txt b/CMakeLists.txt index 09093c6..f779a8a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -232,6 +232,21 @@ target_link_libraries(BacktestingEngineLib PUBLIC Boost::decimal) target_link_libraries(BacktestingEngineLib PUBLIC pqxx OpenMP::OpenMP_CXX) +# AWS SDK for C++, DynamoDB client only — IG session credentials live in the +# MarketDataLive table (shared/aws/dynamoAuth). Deliberately NOT an +# add_subdirectory: the SDK is enormous and must not inherit this project's +# global module/OpenMP flags, so scripts/build_dep.sh builds and installs it +# once (static libs) into external/aws-install and it is consumed here as a +# prebuilt package. AWSSDK_LINK_LIBRARIES resolves to the dynamodb target, +# which chains core + CRT dependencies through its exported config. +list(APPEND CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/external/aws-install") +# The static aws-cpp-sdk-core's exported link interface names ZLIB::ZLIB +# (and curl/OpenSSL targets, already found above) but its config does not +# find_dependency them — resolve ZLIB here first. +find_package(ZLIB REQUIRED) +find_package(AWSSDK REQUIRED CONFIG COMPONENTS dynamodb) +target_link_libraries(BacktestingEngineLib PUBLIC ${AWSSDK_LINK_LIBRARIES}) + # Main executable add_executable(BacktestingEngine source/main.cpp) target_link_libraries(BacktestingEngine BacktestingEngineLib) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e9dd25..cc4e130 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,8 @@ I am actively experimenting with different approaches and want to avoid merge co The project is [MIT-licensed](LICENSE.MD), so you're very welcome to fork it and take the code in your own direction. +To build and test: `scripts/build.sh`, then `ctest --test-dir build` (see the README and [QUICKSTART.md](QUICKSTART.md) for detail). Code style lives in [CONVENTIONS.md](CONVENTIONS.md). + ## Use GitHub Issues for bugs, questions, and ideas Bugs, questions, and ideas are all welcome on the [Issues tab](https://github.com/mccaffers/backtesting-engine-cpp/issues). diff --git a/CONVENTIONS.md b/CONVENTIONS.md index 0adc3ba..8d95bc6 100644 --- a/CONVENTIONS.md +++ b/CONVENTIONS.md @@ -4,13 +4,24 @@ Plain headers live next to the code they belong to under `source/` (e.g. separate `include/` folder. The build exposes a single source root (`source/`), so every project `#include` is **source-relative and path-qualified**, e.g. `#include "shared/utilities/env.hpp"`. This makes each include / module global -module fragment dependency self-describing. Both `source/*.cpp` and -`source/*.cppm` are globbed automatically (`CONFIGURE_DEPENDS`), so a new file is -picked up without editing `CMakeLists.txt`. +module fragment dependency self-describing. (The build also exposes a second +PUBLIC include root, `external/`, only for vendored third-party headers like +`nlohmann/json.hpp`.) Both `source/*.cpp` and `source/*.cppm` are globbed +automatically (`CONFIGURE_DEPENDS`), so a new file under `source/` is picked up +without editing `CMakeLists.txt` — `source/` only; tests are not globbed, see +below. ### Pragma once Headers should use `#pragma once` directive to guard to prevent multiple inclusions of the same header file. -### Lower Camel Case names -camelCase applies to file names, types, and namespaces. -application.cpp / class Application() / namespace tradingDefinitions +### Naming +File names are lowerCamelCase, types are PascalCase, namespaces are snake_case. +databaseConnection.cppm / class DatabaseConnection / namespace symbol_scale +(the one legacy camelCase namespace is tradingDefinitions) + +### New files should come with ctests +only exception would be if they are using libraries (eg. boost), we don't need to test libraries. +Unlike `source/`, test translation units are NOT globbed: `tests/CMakeLists.txt` +lists every file explicitly in `add_executable(unit_tests ...)`, so a new +`tests/foo.cpp` silently never builds or runs until added there. Tests register +with ctest via `catch_discover_tests` in the same file. \ No newline at end of file diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md new file mode 100644 index 0000000..b54bb2d --- /dev/null +++ b/ENVIRONMENT.md @@ -0,0 +1,67 @@ +# Environment Variables + +All engine configuration is read from the environment via `env::getOr(name, fallback)` (`source/shared/utilities/env.cpp`) — every variable is **optional with a default** unless noted. An unset *or empty* variable falls back to its default. + +The `run` and `analysis` subcommands print a startup diagnostic dump of the whole environment to stderr, masking any variable whose name contains `PASSWORD`, `PASSWD`, `SECRET`, `TOKEN`, `CREDENTIAL`, `KEY`, or `AUTH`. + +## Quick reference + +| Variable | Default | `load` | `experiments` | `run` | `analysis` | `ingest` | `live` | `tracking` | `positions` | Purpose | +| --- | --- | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | --- | +| `REDIS_HOST` | `127.0.0.1` | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | ✓ | Redis host (port is always `6379`) | +| `BACKTEST_BATCH` | *(current UTC ISO week)* | ✓ | ✓ | | | | | | | Weekly batch label (`YYYY-WW`, e.g. `2026-28`) naming the Elasticsearch outcome indices (`backtesting-results-2026-28`, ...; `experiments`: `backtesting-experiments-2026-28`). Frozen into every queued run descriptor and carried through all rolling-window rungs; `scripts/load.sh` exports it once so its per-strategy invocations share one label. Set explicitly to re-run into a past week's indices | +| `ENTRY_SLIPPAGE_TENTH_PIPS` | `0` | ✓ | | | | | | | | Slippage stress toggle: every backtest entry fills this many **tenths of a pip** against the trade (`3` = 0.3 pip; SL/TP anchors stay on the raw tick). Read once at `load` and frozen into every queued run descriptor, carried through all rolling-window rungs, and recorded in each run's results doc. **Validated** — a non-integer or negative value fails the load | +| `QUESTDB_HOST` | `127.0.0.1` | | | ✓ | | ✓ | ✓ | | | QuestDB host (`run`: the required `` argument covers tick reads only — the OHLC/range-bar warm-up still connects to `$QUESTDB_HOST`, so `run ` reads ticks from the argv host but seeds bar histories from here; `live`: OHLC pre-population reads) | +| `QUESTDB_PORT` | `8812` | | | ✓ | ✓ | | ✓ | | | QuestDB pgwire port for tick reads (`live`: OHLC pre-population reads). **Validated** — a non-numeric value is a hard failure | +| `QUESTDB_ILP_PORT` | `9000` | | | | | ✓ | | | | QuestDB ILP-over-HTTP port for tick writes | +| `TICK_CACHE_TTL_MINUTES` | `60` | | | ✓ | | | | | | Lifetime of a cached tick superset before it is reloaded from QuestDB (queue mode only — direct mode and `analysis` bypass the cache). **Validated** — a non-numeric or negative value throws on the cache's first use (once the first run is claimed), aborting the worker; `0` passes | +| `TICK_CACHE_MAX_SUPERSETS` | `1` | | | ✓ | | | | | | How many tick supersets (one per symbol set) a worker keeps resident before evicting the oldest. **Validated** — same rule as the TTL | +| `ELASTIC_ENABLED` | `1` | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | ✓ | Master toggle for Elasticsearch reporting; set `0` to disable (`load`/`experiments`: also skips the weekly index/alias admin; `live`: also silences the trade audit, the trace documents, and the `live-logs` log feed; `tracking`: also silences the per-deal `live-trades` documents; `positions`: also silences the per-cycle `live-function-logs` reports) | +| `ELASTIC_HOST` | `http://localhost:9200` | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | ✓ | Elasticsearch base URL (results reporting; weekly index/alias admin at `load`/`experiments`; `analysis` experiment documents; live winner source + trade audit; `tracking` deal documents; `positions` cycle reports) | +| `ELASTIC_USER` | *(empty)* | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | ✓ | Basic-auth username; empty means no auth header | +| `ELASTIC_USER_PASSWORD` | *(empty)* | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | ✓ | Basic-auth password | +| `ELASTIC_TRADES_ENABLED` | `0` | | | ✓ | | | | | | Set `1` to bulk-index every closed trade into `backtesting-trades` | +| `ELASTIC_DEADLETTER_PATH` | `elastic_deadletter.ndjson` | | | ✓ | ✓ | | ✓ | ✓ | ✓ | File that failed Elasticsearch documents are appended to after retries are exhausted | +| `ELASTIC_FLUSH_SECONDS` | `30` | | | ✓ | ✓ | | ✓ | ✓ | ✓ | Cadence of the background `_bulk` flusher that delivers queued reporting documents (run outcomes, experiment documents, engine exceptions, live trade audits, tracking deal documents); intervals with an empty buffer send nothing. A non-numeric or `< 1` value falls back to `30` with a logged warning | +| `OHLC_PREPOPULATE` | `1` | | | ✓ | | | ✓ | | | Seed OHLC **and range** bar histories from QuestDB at each symbol's first tick (`run`: replay start; `live`: launch — big-bar strategies trade immediately instead of warming up for days). One switch covers both bar types; set `0` to disable all warm-up queries | +| `INGEST_BIND_ADDR` | `127.0.0.1` | | | | | ✓ | | | | UDP bind address for the tick receiver | +| `INGEST_UDP_PORT` | `11111` | | | | | ✓ | | | | UDP bind port (overridable by the command's port argument) | +| `TRACKING_BIND_ADDR` | `127.0.0.1` | | | | | | | ✓ | | UDP bind address for the deal/trade-update receiver | +| `TRACKING_UDP_PORT` | `11112` | | | | | | | ✓ | | UDP bind port (overridable by the command's port argument) | +| `LIVE_BIND_ADDR` | `127.0.0.1` | | | | | | ✓ | | | UDP bind address for the live tick receiver | +| `LIVE_UDP_PORT` | `11110` | | | | | | ✓ | | | UDP bind port (overridable by the command's port argument) | +| `LIVE_MIN_SCORE` | `20` | | | | | | ✓ | | | Winner floor on `results.performanceScore` when selecting strategies from Elasticsearch (top 3 per (symbol, strategy) pair is fixed in code, not env-configurable) | +| `LIVE_MAX_DRAWDOWN_PERCENT` | `10` | | | | | | ✓ | | | Hard ceiling on `results.maxDrawdownPercent` (peak-to-trough giveback) when selecting winners — the Calmar half of `performanceScore` only blends drawdown in, so this gate is what actually excludes spiky runs | +| `LIVE_MIN_CALMAR_SCORE` | `30` | | | | | | ✓ | | | Floor on `results.calmarScore` when selecting winners (Calmar ratio ~2 on the score scale where ratio 3 = 50): growth must be ~2x the worst giveback. Complements the drawdown ceiling — the floor rejects smooth-but-stagnant runs, the ceiling rejects fast growers with deep absolute givebacks | +| `LIVE_TRADE_LOCK_SECONDS` | `30` | | | | | | ✓ | | | TTL of the per-(strategy, direction) Redis trade lock | +| `LIVE_TRACE_ENABLED` | `1` | | | | | | ✓ | | | Emit live-mode trace documents (order/close lifecycle, book sync, IG guard refusals, startup/shutdown, minutely stats) to the Elasticsearch index `live-traces` through the async batch publisher; set `0` to disable. Every document carries an `env` field (`TRADING_ENVIRONMENT`). Delivery still requires `ELASTIC_ENABLED=1` | +| `LIVE_LOG_SHIP_ENABLED` | `1` | | | | | | ✓ | | | Ship every engine log line (`logLine`/`error`) as a document to the Elasticsearch index `live-logs`; set `0` to disable. An independent kill switch from `LIVE_TRACE_ENABLED` — the narrative log and the structured trace events are separate feeds. Delivery still requires `ELASTIC_ENABLED=1` | +| `TRADING_ENVIRONMENT` | `demo` | | | | | | ✓ | ✓ | ✓ | IG environment; lowercased into the DynamoDB credentials key `Auth#` (`demo`/`live`); `tracking` stamps it into each `live-trades` document's `env` field | + +## AWS credentials (`live` and `positions`) + +The IG session credentials are pulled from the DynamoDB table `MarketDataLive` (`source/shared/aws/dynamoAuth`). The engine reads no AWS variables itself — the AWS SDK's default credential/region chain applies: + +- `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION` — or a configured profile / instance role. + +Effectively **required** for `live` order placement and the `positions` sync (without a session, `positions` logs a warning and every cycle skips safely); there is no in-code default. + +## Build and script variables + +| Variable | Default | Read by | Purpose | +| --- | --- | --- | --- | +| `CC` / `CXX` | `clang` / `clang++` | `scripts/build.sh` (non-Homebrew path) | Pin a specific compiler, e.g. `CC=clang-20 CXX=clang++-20` | +| `ENABLE_COVERAGE` | `OFF` | `scripts/build.sh` | Instrument with Clang source-based coverage (CI turns it on for the SonarCloud report) | +| `CLEAN` | `0` | `scripts/test.sh` | `CLEAN=1` forces a clean reconfigure before the test build | + +`scripts/run.sh` additionally **requires** `ELASTIC_HOST`, `ELASTIC_USER`, `ELASTIC_USER_PASSWORD`, and `REDIS_HOST` to be set and non-empty — it aborts up front if any are missing (the engine itself would fall back to the defaults above). It also pins `ELASTIC_TRADES_ENABLED=0` for the run it launches (per-trade indexing off). + +## Secrets management + +I manage secrets with [Infisical](https://infisical.com/), which injects them into the process environment at runtime: + +``` +infisical run -- bash ./scripts/run.sh +``` + +If you're not using Infisical, export the variables yourself (shell profile or a sourced `.env`) before invoking the scripts. diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 0000000..f2bfe03 --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,169 @@ +# Quickstart + +How to build the engine and use each subcommand. Install the toolchain and system libraries first — see [REQUIREMENTS.md](REQUIREMENTS.md). Runtime configuration is environment-driven — see [ENVIRONMENT.md](ENVIRONMENT.md). For how the pieces fit together, see [ARCHITECTURE.md](ARCHITECTURE.md). + +## Build + +```bash +git clone --recurse-submodules https://github.com/mccaffers/backtesting-engine-cpp +cd backtesting-engine-cpp + +# One-off: vendored dependency builds (libpqxx, AWS SDK) +bash ./scripts/build_dep.sh + +# Configure (CMake + Ninja + Clang/libc++) and compile +bash ./scripts/build.sh +``` + +`build.sh` runs `build_dep.sh` automatically on first use, picks the right toolchain per platform (Homebrew LLVM on macOS, `$CC`/`$CXX` or `clang` elsewhere), and produces `build/BacktestingEngine`. + +The binary dispatches on its first argument. The subcommands are: + +``` +BacktestingEngine [args...] +``` + +(There is no `--help` — a missing or unknown subcommand just prints an error and exits 1.) + +## `load` — queue a parameter sweep + +Expands a strategy parameter sweep into individual backtest payloads and pushes them onto the Redis work queue. Needs **Redis** running. + +``` +BacktestingEngine load +``` + +The sweep name is required (exact, case-sensitive): `random` (random entries, SL/TP grid — a baseline harness), `ohlcBreakout` (multi-bar range breakout, EMA trend filter), `fvg` (fair-value-gap retracement), `keltnerFade` (volatility-band mean-reversion fade), `sessionRangeBreakout` (London open-range breakout), `squeezeBreakout` (compressed-bar breakout), `nyOpenRangeBreakout` (NY open-range breakout), `liquiditySweepReversal` (swing-pivot sweep + displacement reversal), or `rangeVelocity` (range-bar run velocity breakout). An unknown or missing name prints the valid set and exits 1. Before touching Redis it prints the total sweep size (combinations × symbol groups) and waits for confirmation on stdin — closed stdin counts as a decline, so a non-interactive invocation can't accidentally queue an enormous grid. (`keltnerFade` was retired from the default batch in 2026-28 — its 7 winners all landed on the EURUSD control symbol — but the sweep module remains; load it explicitly to re-test.) + +To queue the sweep as a slippage stress run, export `ENTRY_SLIPPAGE_TENTH_PIPS` (e.g. `3` = every entry fills 0.3 pip against the trade) before `load` — the value is frozen into every queued run and carried through the rolling-window ladder (see [ENVIRONMENT.md](ENVIRONMENT.md)). + +```bash +# via the wrapper script (builds first, checks Redis reachability) +REDIS_HOST=127.0.0.1 bash scripts/load.sh random + +# or directly +./build/BacktestingEngine load random +``` + +Invoked with no argument, `scripts/load.sh` queues a seven-strategy batch — `ohlcBreakout fvg sessionRangeBreakout squeezeBreakout nyOpenRangeBreakout liquiditySweepReversal rangeVelocity` — deliberately excluding `random` and `keltnerFade`, and exports a single `BACKTEST_BATCH` label first so every strategy lands in the same weekly indices. + +Each symbol group becomes its own run: payload keys are written first, then the payload key-names, then the run descriptor is advertised on `BACKTESTING_QUEUE_RUN` — so a worker never sees a run whose strategies aren't fully present. (Key details in [ARCHITECTURE.md](ARCHITECTURE.md#backtest-pipeline).) + +## `run` — execute backtests + +Drains the Redis queue and runs backtests against QuestDB tick data, reporting results to Elasticsearch. Needs **Redis**, **QuestDB**, and (unless `ELASTIC_ENABLED=0`) **Elasticsearch**. + +``` +BacktestingEngine run # queue mode: drain the Redis queue +BacktestingEngine run # direct mode: run one decoded strategy, bypassing Redis +``` + +```bash +# via the wrapper script (validates env vars, builds, checks Redis, runs queue mode) +infisical run -- bash ./scripts/run.sh +# or without Infisical: +ELASTIC_HOST=http://localhost:9200 ELASTIC_USER=elastic ELASTIC_USER_PASSWORD=password \ + REDIS_HOST=localhost bash scripts/run.sh + +# or directly +./build/BacktestingEngine run localhost + +``` + +Wrapper caveats: `run.sh` sources `scripts/clean.sh` first (pass `--clean` to wipe `build/` for a from-scratch rebuild), hardcodes `run localhost` — a remote QuestDB host can't be passed through the wrapper — and if Redis is unreachable it prints "skipping" and exits 0. + +Queue mode runs as a daemon — it never exits on an empty queue, it logs `run queues empty, waiting for work...` and re-peeks on a 1s timer until work reappears (like `analysis`); multiple `run` processes (across machines) can share one Redis and drain the same run in parallel. At startup it also maps a shared-memory control channel (`ipc::EngineControlChannel`, a 12-byte block at `/tmp/EngineControlShm` — `source/shared/ipc/engineControl.hpp`) that broadcasts the in-flight backtest count and reads a stop flag a local monitor can set: flipping the stop signal is the only graceful way to stop a worker — it finishes in-flight backtests, claims nothing more, and parks drained-and-paused. Best-effort: if the segment can't map, the engine still runs, just unmonitored. Tick data is cached across runs — one QuestDB superset load per symbol set serves every rolling window as a slice, and backtests from consecutive runs pipeline on the thread pool (tunable via the `TICK_CACHE_*` variables in [ENVIRONMENT.md](ENVIRONMENT.md)). Completed runs automatically re-queue themselves on the next rolling window — see the ladder in [ARCHITECTURE.md](ARCHITECTURE.md#rolling-window-chaining). A helper for producing a base64 config for direct mode lives in `scripts/arguments/build.sh`; direct mode bypasses both Redis and the tick cache. + +## `experiments` — queue an occurrence-rate experiment sweep + +Expands a parameter sweep of chained-activity experiments ("price drops X% in W minutes, then recovers Y% in Z minutes — how often?") into payloads on the experiment Redis queue — `load`'s twin for questions that don't need a strategy. Needs **Redis** running (and Elasticsearch for the weekly index/alias admin unless `ELASTIC_ENABLED=0`). + +``` +BacktestingEngine experiments +``` + +The sweep name is required: `dipRecovery` (drop size × drop window × recovery size × recovery window — one experiment per combination, evaluated per symbol group). An unknown or missing name prints the valid set and exits 1. Like `load`, it prints the total sweep size and waits for confirmation on stdin before touching Redis, and each symbol group becomes its own run on `BACKTESTING_QUEUE_EXPERIMENT_RUN`. The tick-history window (LAST_MONTHS/OFFSET_MONTHS, default 9/0) is declared per sweep, next to its grid. + +```bash +./build/BacktestingEngine experiments dipRecovery +``` + +## `analysis` — evaluate queued experiments + +Drains the experiment queue and counts each experiment's occurrences against QuestDB tick data, reporting one aggregate document per experiment × symbol group to the weekly `backtesting-experiments` Elasticsearch index — `run`'s twin for the experiment queue. Needs **Redis**, **QuestDB**, and (unless `ELASTIC_ENABLED=0`) **Elasticsearch**. Runs as a daemon (Ctrl+C to stop — no shared-memory stop channel). + +``` +BacktestingEngine analysis +``` + +```bash +./build/BacktestingEngine analysis localhost +``` + +Each document echoes the experiment chain and carries occurrence counts plus the strategy-shaping stats: `completionRate` (attempts vs completions), per-leg failure attribution, month/hour occurrence buckets, per-symbol coverage, MFE/MAE excursion quantiles (completed and failed attempts separately), completion-time quantiles, and mean spread at trigger. Multiple `analysis` workers can share the queue exactly like `run` workers. See [ARCHITECTURE.md](ARCHITECTURE.md#experiments). + +## `ingest` — stream ticks into QuestDB + +Binds a UDP socket, decodes incoming tick packets, and batch-writes them to QuestDB via ILP-over-HTTP. Needs **QuestDB** (it still binds and buffers if QuestDB is down, shedding once the buffer fills). Blocks until SIGINT/SIGTERM. + +``` +BacktestingEngine ingest [udp-port] # port defaults to $INGEST_UDP_PORT, then 11111 +``` + +```bash +bash scripts/ingest.sh # bind 11111 +bash scripts/ingest.sh 22222 # bind 22222 +``` + +The script probes QuestDB reachability and warns (but continues) if it's down. + +## `live` — live trading + +Pulls the winning backtest runs from the `backtesting-winners-current` Elasticsearch alias (the newest weekly winners index, repointed by each `load`), instantiates their strategies (one worker thread per symbol), and routes a live UDP tick stream to them. Entries are gated by Redis trade locks, position caps, portfolio cluster-exposure limits, and (for winners flagged `PEAK_HOURS_ONLY`) peak market sessions; orders are placed with the **IG REST API**, with session credentials pulled from DynamoDB. The broker owns positions and exits — the engine mirrors the position book *from* Redis and only emits opens/closes. Blocks until SIGINT/SIGTERM. + +Needs **Elasticsearch** (the `backtesting-winners-current` alias must exist — run one `load` first, or hand-park the alias on the legacy `trading_results` index — with at least one full-history run above `LIVE_MIN_SCORE` and `LIVE_MIN_CALMAR_SCORE`, within `LIVE_MAX_DRAWDOWN_PERCENT`; the shorter rolling-window rungs don't qualify — no winners is the one hard startup failure, exit 1), **Redis**, **QuestDB** (with `OHLC_PREPOPULATE=1`, the default, OHLC and range-bar histories seed from it at each symbol's first tick; a failed seed logs and falls back to a cold start), and a live tick feed on UDP. **AWS credentials** (DynamoDB) are effectively required to trade but not to start: without an IG session the engine warns and continues — every order fails and extends its trade lock, so it runs as an effective dry run. + +``` +BacktestingEngine live [udp-port] # port defaults to $LIVE_UDP_PORT, then 11110 +``` + +```bash +bash scripts/live.sh # bind 11110 +bash scripts/live.sh 22222 # bind 22222 +``` + +Set `TRADING_ENVIRONMENT=demo|live` to select the IG environment (credentials key `Auth#` in DynamoDB). A `LiveReporter` prints per-minute stats and a final summary on shutdown. + +## `tracking` — follow the account's deal updates + +Binds a UDP socket for the IG streamer's deal feed (256-byte deal packets — position opens, updates, and deletions pushed by IG's Lightstreamer `TRADE:*` channel) and prints one timestamped log line per decoded deal. On a DELETED (closed) deal it also computes the close pips and archives the position's Redis book entry (`PO#` → `PH#`, pruning the `PL#` list), and every deal ships a `live-trades` document to Elasticsearch. Uses **Redis** and **Elasticsearch**, but degrades gracefully without either — an unreachable Redis means fallback documents (never a crash), and undeliverable documents land in the dead-letter file. Blocks until SIGINT/SIGTERM. + +``` +BacktestingEngine tracking [udp-port] # port defaults to $TRACKING_UDP_PORT, then 11112 +``` + +Datagrams that aren't exactly 256 bytes are counted as dropped (totals are printed at shutdown). The wire format and decoder live in `source/tracking/dealPacket.cppm` — see [ARCHITECTURE.md](ARCHITECTURE.md#tracking). + +## `positions` — mirror the IG position book into Redis + +The in-process replacement for the external C# `igmarkets_positions` cron: every minute it GETs the account's open positions from the **IG REST API** and mirrors them into **Redis** — refreshing the `PO#` payloads and `PL#` strategy books that `live` reads, rebuilding the `CG#` cluster-exposure sets, and pruning deals the broker no longer reports. The first sync fires immediately, then one per minute; each cycle is reported to Elasticsearch (`live-function-logs`) unless `ELASTIC_ENABLED=0`. Loops until SIGINT/SIGTERM. + +Needs **Redis**, **AWS credentials** (IG session tokens from DynamoDB), and the **IG REST API**. + +``` +BacktestingEngine positions # loop forever, one sync per minute +BacktestingEngine positions --once # single cycle, then exit (cron parity / smoke test) +``` + +Set `TRADING_ENVIRONMENT=demo|live` to select the IG environment (credentials key `Auth#` in DynamoDB). Without a session every cycle logs a warning, files a FAILED report, and touches nothing — safe to leave running until the login service refreshes the tokens. See [ARCHITECTURE.md](ARCHITECTURE.md#positions). + +## Tests + +Catch2 tests run through ctest: + +```bash +bash ./scripts/test.sh # build + ctest --output-on-failure +CLEAN=1 bash ./scripts/test.sh # force a clean reconfigure first +``` + +For a local coverage report there is `scripts/local_test_coverage.sh` (Clang source-based coverage, same instrumentation CI uses for SonarCloud). diff --git a/README.md b/README.md index e7b8da6..beac8a2 100644 --- a/README.md +++ b/README.md @@ -1,141 +1,59 @@ -## C++ Backtesting Engine +# C++ Backtesting Engine Active development! -Feel free to explore, but this code base is usuable at the moment. +Feel free to explore, but this code base is usable at the moment. -### About The Project +## About The Project -I'm developing a high-performance C++ backtesting engine designed to analyze financial data and evaluate multiple trading strategies at scale. +I'm developing a high-performance C++ backtesting engine designed to analyze financial data and evaluate multiple trading strategies at scale — and to take the winners live. [![Build](https://github.com/mccaffers/backtesting-engine-cpp/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/mccaffers/backtesting-engine-cpp/actions/workflows/build.yml) [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=mccaffers_backtesting-engine-cpp&metric=bugs)](https://sonarcloud.io/summary/new_code?id=mccaffers_backtesting-engine-cpp) [![Code Smells](https://sonarcloud.io/api/project_badges/measure?project=mccaffers_backtesting-engine-cpp&metric=code_smells)](https://sonarcloud.io/summary/new_code?id=mccaffers_backtesting-engine-cpp) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=mccaffers_backtesting-engine-cpp&metric=coverage)](https://sonarcloud.io/summary/new_code?id=mccaffers_backtesting-engine-cpp) +The engine is C++23 (modules, `import std;`) and one binary with eight subcommands: + +- **`ingest`** — receives a UDP tick stream and writes it to QuestDB +- **`load`** — expands a strategy parameter sweep and queues it in Redis +- **`run`** — drains the queue, backtests against QuestDB ticks, reports results to Elasticsearch +- **`experiments`** — expands a parameter sweep of occurrence-rate questions ("price drops 1% in 10m, then recovers 0.5% in 10m — how often?") and queues it in Redis, no strategy required +- **`analysis`** — drains the experiment queue, counts pattern occurrences against QuestDB ticks, reports aggregate stats (rates, conditional completion, excursion quantiles) to Elasticsearch +- **`live`** — takes the winning backtests from Elasticsearch and trades them live via the IG REST API +- **`tracking`** — receives the IG account's deal/position updates over UDP, logs each one, archives closed deals in the Redis position book (`PO#` → `PH#`, pruning the `PL#` list), and ships a `live-trades` document to Elasticsearch per deal +- **`positions`** — mirrors the IG account's open positions into Redis every minute (the position book, strategy lists, and cluster-exposure sets the live engine reads) + I'm extracting results and creating various graphs for trend analyses using SciPy for calculations and Plotly for visualization. ![alt text](documents/images/random-indices-sp500-variable.svg) *Read more results on https://mccaffers.com/quantitative_analysis/randomly_trading/* -## Setup - -This backtesting engine can pull tick data from local files or from a Postgres database (I'm using QuestDB). Strategy execution is dispatched via a Redis list called `strategy_queue`, with each entry a Base64-encoded JSON payload, the `load` subcommand enqueues strategies (LPUSH) and the `run` subcommand dequeues and executes them (RPOP). The default workflow expects a local `redis-server` listening on `127.0.0.1:6379`. - -### Clone with submodules - -The project depends on two vendored libraries (`libpqxx` and `boost-decimal`) tracked as git submodules under `external/`. If you didn't clone with `--recurse-submodules`, run: - -``` -git submodule update --init --recursive -``` - -`scripts/build_dep.sh` does this for you on first run. - -### Install libpq (required by libpqxx) - -``` -For Ubuntu/Debian systems: sudo apt-get install libpq-dev -On Red Hat Linux (RHEL) systems: yum install postgresql-devel -For Mac Homebrew: brew install postgresql -For OpenSuse: zypper in postgresql-devel -For ArchLinux: pacman -S postgresql-libs -``` - -### Install Boost, OpenSSL, and Redis - -Boost.Redis is header-only but its single translation unit (compiled via `` from `source/shared/redis/boostRedisImpl.cpp`) pulls in Boost.Asio's SSL layer, so OpenSSL is a transitive requirement. A local `redis-server` on `127.0.0.1:6379` is also needed for the default `load`/`run` workflow. - -``` -For Mac Homebrew: brew install boost openssl redis -For Ubuntu/Debian systems: sudo apt-get install libboost-all-dev libssl-dev redis-server -``` - -The canonical CI prerequisite list lives in `.github/workflows/scripts/brew.sh` (`postgresql`, `pkg-config`, `boost`). - -![alt text](documents/flow.png) - -### Build dependencies - -`libpqxx` is built once via CMake. `boost-decimal` is header-only and pulled in via `add_subdirectory` from the top-level `CMakeLists.txt`, nothing to build. The script below handles the libpqxx build: - -``` -bash ./scripts/build_dep.sh -``` - -Xcode - Link Binary with Libraries (Source & Test) - -``` -./build/external/libpqxx/src/libpqxx-7.10.a -``` - -Xcode - Headers Path (for libpqxx and nlohmann/json) - -``` -"$(SRCROOT)/external/libpqxx/include/pqxx/internal" -"$(SRCROOT)/external/libpqxx/include/" -"$(SRCROOT)/external/" -``` - -Xcode - Library Path - -``` -"$(SRCROOT)/external/libpqxx/src" -"$(SRCROOT)/build/external/libpqxx/src" -"/opt/homebrew/Cellar/postgresql@14/14.15/lib/postgresql@14" -``` - -### Build the project - -`bash ./scripts/build.sh` +## Documentation -### Environment variables - -The engine reads its connection configuration from the environment. The following variables are **required** — `scripts/run.sh` validates them up front and aborts if any are missing or empty: - -| Variable | Used for | +| Document | Contents | | --- | --- | -| `ELASTIC_HOST` | Elasticsearch base URL that trading results are PUT to (e.g. `https://elastic.example.com:9200`) | -| `ELASTIC_USER` | Elasticsearch HTTP basic-auth username | -| `ELASTIC_USER_PASSWORD` | Elasticsearch HTTP basic-auth password | -| `REDIS_HOST` | Redis host for the `strategy_queue` list | - -I manage these secrets with [Infisical](https://infisical.com/), which injects them into the process environment at runtime, so I run the engine with: - -``` -infisical run -- sh ./scripts/run.sh -``` +| [QUICKSTART.md](QUICKSTART.md) | Building the engine and using each subcommand | +| [ARCHITECTURE.md](ARCHITECTURE.md) | How it fits together — data flow, queue design, live order path (Mermaid diagrams) | +| [ENVIRONMENT.md](ENVIRONMENT.md) | Every environment variable, per command, with defaults | +| [REQUIREMENTS.md](REQUIREMENTS.md) | Toolchain, system libraries, vendored dependencies, runtime services | -If you're not using Infisical, export the variables yourself (e.g. via your shell profile or a sourced `.env`) before invoking the script. +## Quick start -### Run via terminal +```bash +git clone --recurse-submodules https://github.com/mccaffers/backtesting-engine-cpp +cd backtesting-engine-cpp -`bash ./scripts/run.sh` builds the project, then, if `redis-cli ping` reaches a local Redis, enqueues an inline JSON strategy via `load` and executes it via `run localhost`. If Redis is unreachable the script prints a message and exits cleanly (see `scripts/run.sh:22-25`), so first-time users without Redis still get a clear signal. The script requires the [environment variables](#environment-variables) listed above; with Infisical that becomes `infisical run -- sh ./scripts/run.sh`. +bash ./scripts/build.sh # CMake + Ninja + Clang/libc++ (see REQUIREMENTS.md for the toolchain) +bash ./scripts/test.sh # Catch2 tests via ctest -The `BacktestingEngine` binary exposes a subcommand CLI: - -``` -BacktestingEngine load - Base64-encode a built-in strategy JSON (defined in - source/load/loadCommand.cppm) and LPUSH it onto the Redis - `strategy_queue` list. - -BacktestingEngine run - RPOP one Base64-encoded strategy from `strategy_queue` and execute it - against the supplied QuestDB host. - -BacktestingEngine run - Decode the supplied Base64 strategy and execute it directly, bypassing - Redis. +# with Redis, QuestDB, and Elasticsearch running (see QUICKSTART.md): +./build/BacktestingEngine load random # queue a sweep +./build/BacktestingEngine run localhost # drain and backtest ``` -Defaults are `127.0.0.1:6379` for the Redis endpoint and `strategy_queue` for the list key (see `source/shared/redis/redisRunner.hpp` and `source/shared/redis/redisLoader.hpp`). - -### Run tests via terminal - -`bash ./scripts/test.sh` - -### Contributing +## Contributing This is an active solo experiment, so I'm not accepting pull requests right now, but please fork freely and use [GitHub Issues](https://github.com/mccaffers/backtesting-engine-cpp/issues) for bugs, questions, and ideas. See [CONTRIBUTING.md](CONTRIBUTING.md) for details. -### License +## License + [MIT](https://choosealicense.com/licenses/mit/) diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md new file mode 100644 index 0000000..e2304d4 --- /dev/null +++ b/REQUIREMENTS.md @@ -0,0 +1,76 @@ +# Requirements + +System and dependency requirements for building and running the engine. For build/run instructions see [QUICKSTART.md](QUICKSTART.md); for runtime configuration see [ENVIRONMENT.md](ENVIRONMENT.md). + +## Toolchain + +The project is C++23 with modules and `import std;`, which constrains the toolchain tightly: + +| Requirement | Why | +| --- | --- | +| CMake 3.30.x, 4.2.x, or 4.3.x (exactly — not "anything newer") | `import std` is gated behind `CMAKE_EXPERIMENTAL_CXX_IMPORT_STD`, whose activation UUID is pinned per CMake version in `CMakeLists.txt` (branches exist for 3.30, 4.2, and 4.3 — other versions fail configure with instructions for adding a branch) | +| Ninja | The only CMake generator that supports C++23 modules / `import std` | +| Clang + libc++ with a `std` module | Apple Clang does **not** ship one. macOS: Homebrew LLVM (`brew install llvm`). Linux: Clang from [apt.llvm.org](https://apt.llvm.org) (CI pins version 20) with `libc++-dev`/`libc++abi-dev` | +| OpenMP runtime | `libomp` (Homebrew) / `libomp--dev` (apt) — the engine library links OpenMP | + +`scripts/build.sh` selects the toolchain automatically: on Homebrew systems it picks Homebrew LLVM and exports `COMPILER_PATH` so Clang finds `libc++.modules.json`; elsewhere it honours `$CC`/`$CXX` (falling back to `clang`/`clang++`). + +## System libraries + +### macOS (Homebrew) + +The canonical CI list lives in `.github/workflows/scripts/brew.sh`: + +``` +brew install postgresql@18 pkg-config boost llvm ninja libomp +``` + +`postgresql@18` provides libpq (needed by libpqxx). OpenSSL, CURL, and ZLIB are also required by CMake (`find_package`) and typically already present; `brew install openssl curl` covers them if not. + +### Ubuntu / Debian + +The canonical CI setup lives in `.github/workflows/scripts/ubuntu_deps.sh`: + +``` +# Clang + libc++ from apt.llvm.org (LLVM_VERSION defaults to 20) +wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && sudo ./llvm.sh 20 + +sudo apt-get install libc++-20-dev libc++abi-20-dev libomp-20-dev \ + ninja-build libssl-dev libpq-dev libcurl4-openssl-dev +``` + +**Boost ≥ 1.84 is required** (Boost.Redis was added in 1.84; Ubuntu's apt Boost predates it). CI builds Boost 1.90.0 from source — only headers plus Boost.System are needed since Boost.Redis and Boost.Asio are header-only. On macOS, `brew install boost` is current enough. + +## Vendored dependencies (`external/`) + +| Dependency | How it's tracked | How it's built | +| --- | --- | --- | +| [libpqxx](https://github.com/jtv/libpqxx) | git submodule | `add_subdirectory` from the top-level CMakeLists (prebuilt once by `scripts/build_dep.sh`) | +| [boost-decimal](https://github.com/boostorg/decimal) | git submodule | header-only, `add_subdirectory` — nothing to compile | +| [Catch2](https://github.com/catchorg/Catch2) | git submodule | `add_subdirectory`, tests only | +| nlohmann/json | vendored headers | header-only include | +| [aws-sdk-cpp](https://github.com/aws/aws-sdk-cpp) | git submodule (has nested CRT submodules — fetch with `--recursive`) | built + installed once by `scripts/build_dep.sh` into `external/aws-install` (static libs, DynamoDB client only), consumed via `find_package(AWSSDK)` | + +Fetch the submodules (the build scripts also do this on first run): + +``` +git submodule update --init --recursive +``` + +The AWS SDK is deliberately **not** an `add_subdirectory` — the SDK is enormous and must not inherit the project's module/OpenMP flags. On Linux, `scripts/build_dep.sh` builds it with the same Clang/libc++ toolchain as the engine, and a stale libstdc++-built install is detected (by its `__cxx11` ABI markers) and rebuilt automatically. The DynamoDB client is used by the `live` and `positions` commands to pull IG session credentials (`source/shared/aws/dynamoAuth`). + +## Runtime services + +None are needed to *build*; which ones you need to *run* depends on the command (see [QUICKSTART.md](QUICKSTART.md)): + +| Service | Default endpoint | Needed by | +| --- | --- | --- | +| Redis | `127.0.0.1:6379` | `load`, `run`, `experiments`, `analysis` (work queues); `live` (trade locks, position book, API rate budget); `positions` (position book + cluster-set writes); `tracking` (position-book reads for deal enrichment) | +| QuestDB | pgwire `:8812` (reads), ILP-over-HTTP `:9000` (writes) | `run`, `analysis` (tick reads), `live` (OHLC/range-bar warm-up reads — on by default, `OHLC_PREPOPULATE=0` disables), `ingest` (tick writes) | +| Elasticsearch | `http://localhost:9200` | `run` (results reporting), `experiments`/`analysis` (experiment index admin + aggregate docs), `live` (winner selection + trade audit), `positions` (cycle reports), `tracking` (`live-trades` docs) | +| IG REST API | — | `live` (order placement), `positions` (open-position reads) | +| AWS DynamoDB | `MarketDataLive` table | `live`, `positions` (IG session credentials) | + +The `tracking` subcommand binds a UDP socket for the IG streamer's deal feed and logs to stdout; it also enriches each deal from the Redis position book and ships `live-trades` documents to Elasticsearch. Both degrade gracefully (fallback documents, never a crash) when unreachable. + +A note on starting QuestDB locally is in [documents/questdb.md](documents/questdb.md). diff --git a/documents/questdb.md b/documents/questdb.md index 015d25c..5c34216 100644 --- a/documents/questdb.md +++ b/documents/questdb.md @@ -1,5 +1,35 @@ -Start QuestDB (on macOS) +# QuestDB + +Tick storage for the engine: `ingest` writes, `run`/`analysis` read, `live` reads for OHLC/range-bar warm-up (see the runtime-services table in [REQUIREMENTS.md](../REQUIREMENTS.md)). + +## Install & launch + +The command below is the author's setup — a local install under `$HOME/dev/questdb` with Apple-Silicon Homebrew OpenJDK 17: ``` JAVA_HOME="/opt/homebrew/opt/openjdk@17" sh $HOME/dev/questdb/questdb.sh start -d $HOME/dev/questdb/data -``` \ No newline at end of file +``` + +On a fresh machine any stock QuestDB works: `brew install questdb`, the release tarball, or Docker (`docker run -p 9000:9000 -p 8812:8812 questdb/questdb`). + +## Ports & credentials + +- **Writes**: ILP-over-HTTP on `9000` (`$QUESTDB_ILP_PORT`) — the `ingest` command (`source/ingest/ingestCommand.cppm`), batched by `source/ingest/questdbIngestClient.hpp`. +- **Reads**: pgwire on `8812` (`$QUESTDB_PORT`) — `source/shared/questdb/connectionFactory.cppm` builds the connection from the environment with the stock QuestDB credentials: dbname `qdb`, user `admin`, password `quest`. + +Host comes from `$QUESTDB_HOST` (the `run` command can also take it as an argument) — see [ENVIRONMENT.md](../ENVIRONMENT.md) for all three variables. + +## Schema contract + +One table per symbol, table name == symbol (`EURUSD`, ...): + +- `ask`, `bid` — scaled fixed-point INT32 "points", **not** floats: the real price times the symbol's multiplier (FX majors ×100000, JPY pairs & metals ×1000, indices/commodities ×100 — `source/shared/models/priceData.cppm`, `source/shared/utilities/symbolScale.cppm`). e.g. EURUSD 1.10001 is stored as 110001. +- designated `timestamp` — written in nanoseconds, the QuestDB HTTP default precision. + +The write side emits ILP lines of the form `{symbol} ask={..}i,bid={..}i {tsNanos}` (`ingestCommand.cppm`). The read side names columns explicitly and parses positionally as (symbol, ask, bid, timestamp) (`source/shared/questdb/sqlManager.cppm`), so tables must serve exactly those columns. + +Symbols are whitelisted: `SqlManager` throws `std::invalid_argument` for any symbol not in the `symbolScale` table, so a table named outside the whitelist is unreadable by the engine. + +## Reachability + +`scripts/ingest.sh` probes `http://$QUESTDB_HOST:$QUESTDB_ILP_PORT/exec?query=SELECT%201` before starting — non-fatal, since the ingest writer buffers (and eventually sheds) while QuestDB is down. diff --git a/external/aws b/external/aws new file mode 160000 index 0000000..cfaa2bb --- /dev/null +++ b/external/aws @@ -0,0 +1 @@ +Subproject commit cfaa2bbf96a1ecf471b644b6b2f0bdf7c3b10dfe diff --git a/scripts/arguments/build.sh b/scripts/arguments/build.sh index 547e346..27238c5 100644 --- a/scripts/arguments/build.sh +++ b/scripts/arguments/build.sh @@ -6,8 +6,8 @@ json='{ "UUID": "", "TRADING_VARIABLES": { "STRATEGY": "RandomStrategy", - "STOP_DISTANCE_IN_PIPS": 1, - "LIMIT_DISTANCE_IN_PIPS": 1, + "STOP_DISTANCE_IN_ATR": 1, + "LIMIT_DISTANCE_IN_ATR": 3, "TRADING_SIZE": 1 }, "OHLC_VARIABLES": [ diff --git a/scripts/build.sh b/scripts/build.sh old mode 100644 new mode 100755 index 6c67e3a..e1ae778 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -2,8 +2,25 @@ git submodule update --init --recursive -if [ -z "$(ls -A ./external/boost-decimal 2>/dev/null)" ] || [ -z "$(ls -A ./external/libpqxx 2>/dev/null)" ]; then - ./scripts/build_dep.sh +# An AWS SDK install produced by the old default-toolchain build (GCC/ +# libstdc++ on Linux) satisfies the AWSSDKConfig.cmake gate below but fails +# the engine's libc++ link with undefined std::__cxx11 references. Delete it +# here so the gate re-runs build_dep.sh, which rebuilds with clang/libc++. +# (build_dep.sh has the same heal, but it only helps if it actually runs.) +if ! command -v brew &>/dev/null; then + AWS_CORE_LIB="./external/aws-install/lib/libaws-cpp-sdk-core.a" + [ -f "$AWS_CORE_LIB" ] || AWS_CORE_LIB="./external/aws-install/lib64/libaws-cpp-sdk-core.a" + if [ -f "$AWS_CORE_LIB" ] && nm "$AWS_CORE_LIB" 2>/dev/null | grep -q '__cxx11'; then + echo "build.sh: existing AWS SDK install was built against libstdc++ — rebuilding with libc++" >&2 + rm -rf ./external/aws-install ./external/aws/build + fi +fi + +if [ -z "$(ls -A ./external/boost-decimal 2>/dev/null)" ] || [ -z "$(ls -A ./external/libpqxx 2>/dev/null)" ] \ + || [ ! -f ./external/aws-install/lib/cmake/AWSSDK/AWSSDKConfig.cmake ]; then + # bash (not ./) so a lost exec bit can't break the build; abort on failure + # rather than cascading into a confusing find_package(AWSSDK) error later. + bash ./scripts/build_dep.sh || { echo "build.sh: dependency build failed" >&2; exit 1; } fi BUILD_DIR="build" diff --git a/scripts/build_dep.sh b/scripts/build_dep.sh old mode 100644 new mode 100755 index 6ddabe9..341935b --- a/scripts/build_dep.sh +++ b/scripts/build_dep.sh @@ -58,3 +58,72 @@ fi # which is all a header-only lib needs. We just need the submodule fetched # above; no build step. build_dep "$EXTERNAL_DIR/libpqxx" + +# AWS SDK for C++ — DynamoDB client only (IG session credentials, see +# shared/aws/dynamoAuth). Unlike libpqxx it is NOT add_subdirectory'd into +# the main build (it is enormous, and the main build's module/OpenMP flags +# must not leak into it): it is built and INSTALLED once here, then found by +# the main CMakeLists via find_package(AWSSDK) against external/aws-install. +# Static libs so the engine binary carries no SDK dylibs. The source is the +# external/aws submodule, populated (with its nested CRT submodules) by the +# `git submodule update --init --recursive` at the top of this script. +# +# Because these static libs are linked into the engine, they MUST be built +# against the same C++ standard library as the main build. On Linux the +# engine is Clang + libc++ (required for `import std`), but CMake's default +# compiler pick is GCC/libstdc++ — that combination links the SDK's +# libstdc++-ABI symbols (std::__cxx11::…) into the .a files and they never +# resolve at the engine's libc++ link. Mirror build.sh's toolchain: honour +# $CC/$CXX, fall back to clang, and force -stdlib=libc++ off macOS (macOS +# clang already defaults to libc++, and the Homebrew/Apple split there is +# ABI-compatible, so leave that path untouched). +AWS_CMAKE_ARGS=() +AWS_CXX_FLAGS="-w" +if ! command -v brew &>/dev/null; then + AWS_CMAKE_ARGS+=( + -DCMAKE_C_COMPILER="${CC:-clang}" + -DCMAKE_CXX_COMPILER="${CXX:-clang++}" + ) + AWS_CXX_FLAGS="-w -stdlib=libc++" +fi + +AWS_SRC="$EXTERNAL_DIR/aws" +AWS_INSTALL="$EXTERNAL_DIR/aws-install" + +# Auto-heal installs produced by the old default-toolchain build: on Linux +# those are GCC/libstdc++ artifacts, recognisable by the __cxx11 ABI marker +# in their symbols. Nuke and rebuild rather than skipping below. +if ! command -v brew &>/dev/null; then + # `|| true`: on a fresh node aws-install doesn't exist yet, and a failing + # `find` under `set -e -o pipefail` would silently kill the whole script. + AWS_CORE_LIB="$(find "$AWS_INSTALL" -name 'libaws-cpp-sdk-core.a' 2>/dev/null | head -n1 || true)" + if [ -n "$AWS_CORE_LIB" ] && nm "$AWS_CORE_LIB" 2>/dev/null | grep -q '__cxx11'; then + echo "build_dep: existing AWS SDK install was built against libstdc++ — rebuilding with libc++" >&2 + rm -rf "$AWS_INSTALL" "$AWS_SRC/build" + fi +fi + +if [ ! -f "$AWS_SRC/CMakeLists.txt" ]; then + echo "build_dep: external/aws submodule not populated — run 'git submodule update --init --recursive'" >&2 + exit 1 +elif [ -f "$AWS_INSTALL/lib/cmake/AWSSDK/AWSSDKConfig.cmake" ]; then + echo "build_dep: AWS SDK already installed at external/aws-install — skipping" +else + # ${arr[@]+...} guards the empty-array expansion, which errors under + # `set -u` on the bash 3.2 that macOS ships. + cmake -S "$AWS_SRC" -B "$AWS_SRC/build" -G Ninja \ + ${AWS_CMAKE_ARGS[@]+"${AWS_CMAKE_ARGS[@]}"} \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_ONLY="dynamodb" \ + -DBUILD_SHARED_LIBS=OFF \ + -DFORCE_SHARED_CRT=OFF \ + -DENABLE_TESTING=OFF \ + -DAUTORUN_UNIT_TESTS=OFF \ + -DCMAKE_INSTALL_PREFIX="$AWS_INSTALL" \ + -DCMAKE_CXX_FLAGS="$AWS_CXX_FLAGS" \ + -DCMAKE_C_FLAGS="-w" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + ${SDK_PATH:+-DCMAKE_OSX_SYSROOT="$SDK_PATH"} + cmake --build "$AWS_SRC/build" --parallel "$JOBS" + cmake --install "$AWS_SRC/build" +fi diff --git a/scripts/live.sh b/scripts/live.sh new file mode 100755 index 0000000..f5f4c85 --- /dev/null +++ b/scripts/live.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Builds the engine and runs the `live` subcommand: pulls the winning backtest +# runs from the backtesting-winners-current Elasticsearch alias (the newest +# weekly winners index, repointed by each load), instantiates their +# strategies, then binds a UDP socket on the live tick stream and routes each +# decoded tick (tickPacket) to the cached strategies (one worker thread each). +# Entries are gated by a Redis trade lock per (strategy UUID, direction) plus +# an open-trade cap; orders are placed for real via the IG REST API +# (brokerOrderSink -> orderChannel, session tokens from DynamoDB). The broker +# owns positions and exits — the engine mirrors the position book from Redis +# and only emits opens/closes. Nothing is persisted to QuestDB. Blocks until +# SIGINT/SIGTERM. +# +# Config is read from the environment by LiveCommand (defaults in parens): +# LIVE_BIND_ADDR UDP bind address (127.0.0.1) +# LIVE_UDP_PORT UDP bind port (11110, == UDPPorts.PortLive) +# LIVE_MIN_SCORE winner floor on results.performanceScore (20) +# LIVE_MAX_DRAWDOWN_PERCENT hard ceiling on results.maxDrawdownPercent (10) +# LIVE_MIN_CALMAR_SCORE floor on results.calmarScore (30, Calmar ratio ~2) +# LIVE_TRADE_LOCK_SECONDS Redis trade-lock TTL (30) +# REDIS_HOST Redis host for locks/positions (127.0.0.1:6379) +# TRADING_ENVIRONMENT IG environment, DynamoDB key Auth# (demo) +# ELASTIC_HOST winners source (http://localhost:9200); +# with ELASTIC_USER / ELASTIC_USER_PASSWORD if the +# cluster needs basic auth. Reachable ES with at +# least one qualifying run is REQUIRED — otherwise +# the process exits 1 at startup. +# +# AWS credentials (SDK default chain: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY +# / AWS_DEFAULT_REGION or a profile) are needed to pull the IG session from the +# MarketDataLive DynamoDB table. +# +# An optional first argument overrides the bind port (argv[2] to the engine): +# sh scripts/live.sh # bind 11110 (or $LIVE_UDP_PORT) +# sh scripts/live.sh 22222 # bind 22222 + +current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +if ! source "$current_dir/build.sh"; then + echo "Error: Build failed. Aborting." + exit 1 +fi + +if [ ! -f "$BUILD_DIR/$EXECUTABLE_NAME" ]; then + echo "Error: Executable $EXECUTABLE_NAME not found in $BUILD_DIR." + ls -la "$BUILD_DIR" + exit 1 +fi + +# exec so signals reach the engine directly for clean shutdown. "$@" passes an +# optional bind-port override straight through to the subcommand. +exec ./"$BUILD_DIR/$EXECUTABLE_NAME" live "$@" diff --git a/scripts/load.sh b/scripts/load.sh index c16348f..4a3e2a5 100755 --- a/scripts/load.sh +++ b/scripts/load.sh @@ -1,8 +1,24 @@ #!/bin/bash -# Builds the engine and pushes the built-in strategy JSON -# (source/commands/loadCommand.cpp) onto the Redis `strategy_queue`. +# Builds the engine and runs the `load` subcommand per strategy: expands each +# sweep's parameter grid into keyed Redis payloads plus a run descriptor on +# BACKTESTING_QUEUE_RUN (see shared/utilities/queueKeys.hpp), and prepares the +# batch's weekly Elasticsearch outcome indices + -current aliases. current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +load_args=("$@") + +# One batch label for every invocation below: the label names the weekly +# Elasticsearch indices (backtesting-results-2026-28, ...), and pinning it +# here keeps a load that straddles the Sunday-midnight-UTC ISO-week boundary +# from splitting one batch across two labels. Pre-set BACKTEST_BATCH wins — +# that is the re-run/test override. +export BACKTEST_BATCH="${BACKTEST_BATCH:-$(date -u +%G-%V)}" + +# Strategies queued by a no-arg run; `random` is excluded — load it explicitly. +# keltnerFade retired 2026-28: 7 winners, all on the EURUSD control symbol the +# thesis said should do worst, zero on the six ranging crosses it targeted. +# Its sweep module remains — load it explicitly to re-test. +all_strategies=(ohlcBreakout fvg sessionRangeBreakout squeezeBreakout nyOpenRangeBreakout liquiditySweepReversal rangeVelocity) if ! source "$current_dir/build.sh"; then echo "Error: Build failed. Aborting." @@ -20,4 +36,16 @@ if ! redis-cli -h "$REDIS_HOST" ping >/dev/null 2>&1; then exit 0 fi -exec ./"$BUILD_DIR/$EXECUTABLE_NAME" load +if [ ${#load_args[@]} -gt 0 ]; then + exec ./"$BUILD_DIR/$EXECUTABLE_NAME" load "${load_args[@]}" +fi + +i=0 +for strategy in "${all_strategies[@]}"; do + echo "" + echo "=== [$((++i))/${#all_strategies[@]}] Loading strategy: $strategy ===" + if ! ./"$BUILD_DIR/$EXECUTABLE_NAME" load "$strategy"; then + echo "Error: load failed for '$strategy'. Aborting remaining strategies." + exit 1 + fi +done diff --git a/scripts/run.sh b/scripts/run.sh index e1e8e86..3efc518 100644 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -40,7 +40,7 @@ start_time=$(date +%s%N) # Invoke the `run` subcommand: BacktestingEngine pops a Base64-encoded # strategy off the Redis `strategy_queue` and executes it against the # QuestDB host passed as the second argument (here, localhost). -./"$BUILD_DIR/$EXECUTABLE_NAME" run localhost +ELASTIC_TRADES_ENABLED=0 ./"$BUILD_DIR/$EXECUTABLE_NAME" run localhost end_time=$(date +%s%N) elapsed=$(( (end_time - start_time) / 1000000 )) echo "Execution time: ${elapsed}ms" diff --git a/source/analysis/analysisCommand.cppm b/source/analysis/analysisCommand.cppm new file mode 100644 index 0000000..a82dc4c --- /dev/null +++ b/source/analysis/analysisCommand.cppm @@ -0,0 +1,32 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include "shared/utilities/env.hpp" +#include "analysis/queue/analysisRunner.hpp" + +export module analysisCommand; + +import std; + +export class AnalysisCommand { +public: + static int run(int argc, const char* argv[]); +}; + +int AnalysisCommand::run(const int argc, const char* argv[]) { + + env::printDiagnostics(argc, argv); + + if (argc < 3) { + std::println(stderr, "Usage: BacktestingEngine analysis "); + return 1; + } + + // Redis host from the environment, mirroring `run` (runCommand.cppm). + return AnalysisRunner::run(argv[2], env::getOr("REDIS_HOST", "127.0.0.1")); +} diff --git a/source/analysis/queue/analysisBridge.cpp b/source/analysis/queue/analysisBridge.cpp new file mode 100644 index 0000000..dcb2b6f --- /dev/null +++ b/source/analysis/queue/analysisBridge.cpp @@ -0,0 +1,39 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "analysis/queue/analysisBridge.hpp" + +import std; +import priceData; // PriceData +import backtestRunner; // loadTicks +import chainMatcher; // chain_matcher::evaluateExperiment +import experimentElastic; // ExperimentElastic::putExperimentResults + +// The tick buffer the opaque handle wraps. Kept out of drainExperiments.cpp +// so that TU never has to name a module type — see analysisBridge.hpp. +struct ExperimentTicksImpl { + std::vector ticks; +}; + +ExperimentTicks bridgeLoadExperimentTicks(const std::string& questdbHost, + const std::string& symbolsCsv, + const int lastMonths, + const int offsetMonths) { + auto impl = std::make_shared(ExperimentTicksImpl{ + loadTicks(questdbHost, symbolsCsv, lastMonths, offsetMonths)}); + const std::size_t count = impl->ticks.size(); + return ExperimentTicks{std::move(impl), count}; +} + +experiments::ExperimentOutcome bridgeEvaluateExperiment( + const ExperimentTicks& ticks, + const experiments::ExperimentConfig& config) { + return chain_matcher::evaluateExperiment(ticks.impl->ticks, config); +} + +void bridgePutExperimentResults(const ExperimentResults& results) { + ExperimentElastic::putExperimentResults(results); +} diff --git a/source/analysis/queue/analysisBridge.hpp b/source/analysis/queue/analysisBridge.hpp new file mode 100644 index 0000000..7b90d51 --- /dev/null +++ b/source/analysis/queue/analysisBridge.hpp @@ -0,0 +1,50 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +#pragma once +#include +#include +#include + +#include "shared/experiments/experimentConfig.hpp" +#include "analysis/reporting/experimentResults.hpp" + +// The analysis worker's import boundary to the backtestRunner / chainMatcher / +// experimentElastic modules — runnerBridge.hpp's exact doctrine: +// drainExperiments.cpp must stay a *purely textual* TU because its ThreadPool +// instantiates std::condition_variable_any::wait(lock, stop_token, pred), +// which the toolchain miscompiles when the same TU also imports a module. So +// the drain loop reaches the modules only through these plain (global-module) +// functions, never naming PriceData or importing anything itself. + +// Opaque handle to one run's tick buffer, defined in analysisBridge.cpp. +struct ExperimentTicksImpl; + +// Value-copyable without naming any module type: pool tasks capture it BY +// VALUE, so the shared_ptr keeps the tick buffer alive for the lifetime of +// each evaluation even after the drain loop moves to the next run. No +// TickCache in v1 — each experiment run is its own one-window load, so a +// cache would be all misses; the shared_ptr alone gives buffer-outlives-pool +// safety. +struct ExperimentTicks { + std::shared_ptr impl; + std::size_t tickCount = 0; // logging only +}; + +// One QuestDB load for the run's symbols/window (backtestRunner's loadTicks). +ExperimentTicks bridgeLoadExperimentTicks(const std::string& questdbHost, + const std::string& symbolsCsv, + int lastMonths, + int offsetMonths); + +// Replays one experiment over the loaded ticks (chain_matcher's +// evaluateExperiment — throws std::invalid_argument on a malformed chain). +experiments::ExperimentOutcome bridgeEvaluateExperiment( + const ExperimentTicks& ticks, + const experiments::ExperimentConfig& config); + +// Queues the aggregate document for Elasticsearch +// (ExperimentElastic::putExperimentResults). +void bridgePutExperimentResults(const ExperimentResults& results); diff --git a/source/analysis/queue/analysisRunner.cpp b/source/analysis/queue/analysisRunner.cpp new file mode 100644 index 0000000..10659b1 --- /dev/null +++ b/source/analysis/queue/analysisRunner.cpp @@ -0,0 +1,70 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "analysis/queue/analysisRunner.hpp" + +#include +#include +#include +#include + +#include +#include +#include + +#include "shared/utilities/backtestLog.hpp" +#include "run/reporting/elasticPublisher.hpp" +#include "shared/redis/connection/redisConnection.hpp" +#include "analysis/queue/drainExperiments.hpp" + +namespace asio = boost::asio; + +int AnalysisRunner::run(const std::string& questdbHost, + const std::string& redisHost, + const int redisPort) { + + backtest_log::set_quiet(true); // Mute logs to prevent interleaved thread spam + asio::io_context ioc; // Set up the async event loop + + const auto conn = redis_util::makeRedisConnection(ioc, redisHost, redisPort); + + // State trackers for the coroutine's outcome + int result = 0; + std::exception_ptr error; + + // Launch the async task to process experiment runs + asio::co_spawn( + ioc, + analysis_runner::drainExperiments(conn, questdbHost), + [&result, &error](std::exception_ptr e, int r) { // Completion callback + if (e) { + error = e; // Save exception if it failed + return; + } + result = r; // Save exit code if it succeeded + }); + + // Block the current thread and execute the async loop until finished + ioc.run(); + + // Unwrap and log any exceptions caught during the async execution + if (error) { + try { + std::rethrow_exception(error); + } catch (const std::exception& ex) { + std::println(stderr, "AnalysisRunner failed: {}", ex.what()); + // Surface the top-level failure in Elasticsearch alongside run + // outcomes. No RUN_ID is in scope here — this is a whole-process + // failure, not a single run. + elastic::putEngineException( + {elastic::nowIsoUtc(), "AnalysisRunner", ex.what(), ""}); + } + return 3; // Exit with error status + } + + // Return the final execution status code + return result; +} diff --git a/source/analysis/queue/analysisRunner.hpp b/source/analysis/queue/analysisRunner.hpp new file mode 100644 index 0000000..f3dcab4 --- /dev/null +++ b/source/analysis/queue/analysisRunner.hpp @@ -0,0 +1,22 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +// Analysis worker entry point — RedisRunner's mirror for the experiment +// queue. Drains BACKTESTING_QUEUE_EXPERIMENT_RUN: for each run it loads the +// QuestDB tick data once, then drains that run's per-RUN_ID experiment list, +// evaluating every experiment against the shared ticks and reporting one +// aggregate document each to Elasticsearch. Safe to launch many workers +// concurrently — they peek the same run, load ticks once each, and compete +// on RPOP of the shared experiment list. +class AnalysisRunner { +public: + static int run(const std::string& questdbHost, + const std::string& redisHost = "127.0.0.1", + int redisPort = 6379); +}; diff --git a/source/analysis/queue/drainExperiments.cpp b/source/analysis/queue/drainExperiments.cpp new file mode 100644 index 0000000..eac9556 --- /dev/null +++ b/source/analysis/queue/drainExperiments.cpp @@ -0,0 +1,266 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "analysis/queue/drainExperiments.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "shared/utilities/jsonParser.hpp" +#include "shared/utilities/queueKeys.hpp" +#include "shared/utilities/threadPool.hpp" +#include "run/reporting/elasticPublisher.hpp" +#include "run/reporting/tradingResults.hpp" // TradeFinal::localHostname +#include "run/queue/runQueue.hpp" +#include "analysis/queue/analysisBridge.hpp" +#include "analysis/reporting/experimentResults.hpp" + +namespace asio = boost::asio; +namespace redis = boost::redis; + +namespace analysis_runner { + +asio::awaitable drainExperiments(std::shared_ptr conn, + std::string questdbHost) { + int exitCode = 0; + + try { + // Same pool sizing as drainRuns: 80% of the available CPU threads, + // at least one worker. Deliberately no shm control channel in v1 — + // Ctrl+C is the only stop. + const unsigned hw = std::thread::hardware_concurrency(); + const unsigned threads = hw != 0 ? std::max(1u, hw * 4 / 5) : 1u; + ThreadPool pool(threads); + + const std::string hostname = TradeFinal::localHostname(); + + // Loop forever, claiming one experiment run per iteration. An empty + // queue makes us wait and re-peek; only an exception blows the loop. + bool waitingLogged = false; + for (;;) { + // Surface the first infrastructure-shaped task failure from any + // still-pipelining evaluation (experiment-scoped errors are + // contained inside the tasks themselves). + if (std::exception_ptr err = pool.takeError()) { + std::rethrow_exception(err); + } + const std::optional peeked = + co_await run_queue::peekQueueTail(conn, + queue_keys::EXPERIMENT_RUN); + if (!peeked.has_value()) { + // Queue empty: stay alive and poll until work reappears — a + // co_awaited timer, so this suspends rather than busy-waits. + if (!waitingLogged) { + std::println( + "AnalysisRunner: experiment queue empty, waiting for work..."); + waitingLogged = true; + } + + asio::steady_timer timer(co_await asio::this_coro::executor); + timer.expires_after(std::chrono::seconds(1)); + co_await timer.async_wait(asio::use_awaitable); + continue; // re-peek; never exit just because the queue is empty + } + waitingLogged = false; + + // An unparseable run descriptor is a poison pill: retire it + // (removeRun LREMs by the raw base64 value, so no parse needed) + // instead of letting it kill this worker and the next. + experiments::ExperimentRunConfiguration runCfg; + bool descriptorOk = true; + try { + runCfg = JsonParser::parseExperimentRunFromBase64( + peeked->descriptorB64); + } catch (const std::exception& ex) { + // co_await is illegal inside a catch handler, so the removal + // happens just below, outside the try/catch. + std::println(stderr, + "AnalysisRunner: unparseable experiment run descriptor, retiring: {}", + ex.what()); + elastic::putEngineException( + {elastic::nowIsoUtc(), "drainExperiments", + std::string("unparseable experiment run descriptor retired: ") + + ex.what(), + ""}); + descriptorOk = false; + } + if (!descriptorOk) { + co_await run_queue::removeRun(conn, peeked->queueKey, + peeked->descriptorB64); + continue; + } + const std::string experimentKey = + queue_keys::experimentKey(runCfg.RUN_ID); + + std::println("drainExperiments, RUN_ID={} SYMBOLS={} LAST_MONTHS={} OFFSET_MONTHS={} EXPERIMENT_KEY={}", + runCfg.RUN_ID, runCfg.SYMBOLS, runCfg.LAST_MONTHS, + runCfg.OFFSET_MONTHS, experimentKey); + + // Claim the first experiment BEFORE the expensive tick load: when + // several workers converge on a nearly-drained run, the losers + // would otherwise each pay a full QuestDB fetch only to find the + // list already empty. + std::optional payloadKey = + co_await run_queue::popStrategyKey(conn, experimentKey); + if (!payloadKey.has_value()) { + co_await run_queue::removeRun(conn, peeked->queueKey, + peeked->descriptorB64); + std::println("AnalysisRunner: RUN_ID={} already drained, retiring", + runCfg.RUN_ID); + continue; + } + + // One QuestDB load for the whole run, shared BY VALUE with every + // pooled evaluation below — the shared handle keeps the buffer + // alive until the last task holding it finishes. + const ExperimentTicks ticks = bridgeLoadExperimentTicks( + questdbHost, runCfg.SYMBOLS, runCfg.LAST_MONTHS, + runCfg.OFFSET_MONTHS); + + // Drain the run's experiment list, competing with any other + // workers: RPOP a payload key name, GETDEL the payload, hand the + // CPU-bound evaluation to the pool. submit() applies + // backpressure, so we pop at the rate the workers absorb. + int experimentsRun = 0; + for (;;) { + const std::optional experimentB64 = + co_await run_queue::takeStrategyPayload(conn, *payloadKey); + if (!experimentB64.has_value()) { + // The safety-net TTL reaped a long-stale payload; skip + // it — the run still drains. + std::println(stderr, + "AnalysisRunner: payload {} missing (expired?), skipping", + *payloadKey); + } else { + // A payload that fails to parse is a poison pill: report + // it and move on — it is already consumed, so it cannot + // recur. + try { + experiments::ExperimentConfig experiment = + JsonParser::parseExperimentFromBase64(*experimentB64); + // Evaluation errors (e.g. a chain this binary's + // matcher rejects) are likewise contained per task. + pool.submit([ticks, runCfg, hostname, + experiment = std::move(experiment)]() { + try { + const auto started = + std::chrono::steady_clock::now(); + const experiments::ExperimentOutcome outcome = + bridgeEvaluateExperiment(ticks, experiment); + const double durationSeconds = + std::chrono::duration( + std::chrono::steady_clock::now() - started) + .count(); + bridgePutExperimentResults(ExperimentResults{ + .RUN_ID = runCfg.RUN_ID, + .timestamp = elastic::nowIsoUtc(), + .hostname = hostname, + .durationSeconds = durationSeconds, + .runConfig = runCfg, + .experiment = experiment, + .occurrences = outcome.occurrences, + .ticksScanned = outcome.ticksScanned, + .daysSpanned = outcome.daysSpanned, + .occurrencesPerDay = + outcome.daysSpanned > 0.0 + ? outcome.occurrences / + outcome.daysSpanned + : 0.0, + .attempts = outcome.attempts, + .failuresByLeg = outcome.failuresByLeg, + .occurrencesByMonth = + outcome.occurrencesByMonth, + .occurrencesByHourUtc = + outcome.occurrencesByHourUtc, + .perSymbol = outcome.perSymbol, + .occurrencesBySymbol = + outcome.occurrencesBySymbol, + .completedAttempts = + outcome.completedAttempts, + .failedAttempts = outcome.failedAttempts, + .completionSeconds = + outcome.completionSeconds, + .meanSpreadAtTriggerPoints = + outcome.meanSpreadAtTriggerPoints, + .excursionOrientation = + outcome.excursionOrientation, + .samplesTruncated = + outcome.samplesTruncated, + }); + } catch (const std::exception& ex) { + std::println(stderr, + "AnalysisRunner: experiment failed (RUN_ID={} experiment={}): {}", + runCfg.RUN_ID, experiment.UUID, + ex.what()); + elastic::putEngineException( + {elastic::nowIsoUtc(), "experiment", + std::string("experiment ") + experiment.UUID + + " failed: " + ex.what(), + runCfg.RUN_ID}); + } + }); + ++experimentsRun; + } catch (const std::exception& ex) { + std::println(stderr, + "AnalysisRunner: unparseable experiment payload {} (RUN_ID={}), skipping: {}", + *payloadKey, runCfg.RUN_ID, ex.what()); + elastic::putEngineException( + {elastic::nowIsoUtc(), "drainExperiments", + std::string("unparseable experiment payload skipped: ") + + ex.what(), + runCfg.RUN_ID}); + } + } + + payloadKey = co_await run_queue::popStrategyKey(conn, experimentKey); + if (!payloadKey.has_value()) { + break; // experiment list drained + } + } + + // Retire the run as soon as its list is drained — evaluations may + // still be pipelining on the pool, but every payload was already + // destructively consumed, so the advert signals nothing claimable + // either way. LREM is idempotent across competing workers. + co_await run_queue::removeRun(conn, peeked->queueKey, + peeked->descriptorB64); + + std::println("AnalysisRunner: drained RUN_ID={} ({} experiment{} queued, evaluations pipelined)", + runCfg.RUN_ID, experimentsRun, + experimentsRun == 1 ? "" : "s"); + } + } catch (const std::exception& ex) { + std::println(stderr, "AnalysisRunner aborted: {}", ex.what()); + elastic::putEngineException( + {elastic::nowIsoUtc(), "drainExperiments", ex.what(), ""}); + exitCode = 3; + } catch (...) { + std::println(stderr, "AnalysisRunner aborted: unknown error"); + elastic::putEngineException( + {elastic::nowIsoUtc(), "drainExperiments", "unknown error", ""}); + exitCode = 3; + } + + // Only reached when an exception aborted the drain — the loop waits on an + // empty queue rather than exiting. Tear the connection down so + // io_context::run() can return. + conn->cancel(); + co_return exitCode; +} + +} // namespace analysis_runner diff --git a/source/analysis/queue/drainExperiments.hpp b/source/analysis/queue/drainExperiments.hpp new file mode 100644 index 0000000..403a4e9 --- /dev/null +++ b/source/analysis/queue/drainExperiments.hpp @@ -0,0 +1,37 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include + +#include +#include + +// The analysis worker's per-run drain loop, split out of analysisRunner.cpp so +// that TU stays focused on orchestration — drainRuns' structure. Like +// drainRuns this is a purely textual unit: it uses ThreadPool, whose +// std::condition_variable_any::wait instantiation the toolchain miscompiles +// when the same TU also imports a module, so it reaches the tick loader and +// the chain matcher only through analysisBridge.hpp's global-module functions +// and imports nothing itself. +namespace analysis_runner { + +// Drains the experiment run queue (queue_keys::EXPERIMENT_RUN — one queue, no +// priority ladder) on a single long-lived connection. Each run's ticks are +// loaded from QuestDB once (no cache — every experiment run is its own +// one-window load) and shared by value with every pooled evaluation; the +// run's experiment list is drained onto the pool, one aggregate Elasticsearch +// document per experiment. When the queue is empty it waits and re-peeks +// rather than exiting, so the worker stays up as a daemon (Ctrl+C to stop — +// deliberately no shm stop-channel in v1); only a Redis/DB/decode error +// leaves the loop (return 3). +boost::asio::awaitable drainExperiments( + std::shared_ptr conn, + std::string questdbHost); + +} // namespace analysis_runner diff --git a/source/analysis/reporting/experimentElastic.cppm b/source/analysis/reporting/experimentElastic.cppm new file mode 100644 index 0000000..7772d4d --- /dev/null +++ b/source/analysis/reporting/experimentElastic.cppm @@ -0,0 +1,41 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include + +#include "run/reporting/elasticPublisher.hpp" +#include "run/reporting/outcomeIndices.hpp" +#include "analysis/reporting/experimentResults.hpp" + +export module experimentElastic; + +import std; + +// Typed front-end over the shared Elasticsearch publisher for experiment +// results — a small PARALLEL client, deliberately not ElasticClient, which is +// welded to tradingDefinitions::Configuration and the results/winners index +// routing. Serialises on the calling thread, hands the document to the +// publisher's background flusher (elastic::enqueueDocument) and returns +// immediately — a pool worker finishing an evaluation never blocks on +// Elastic. +export class ExperimentElastic { +public: + static void putExperimentResults(const ExperimentResults& results); +}; + +void ExperimentElastic::putExperimentResults(const ExperimentResults& r) { + // Deterministic _id: RUN_ID is already per symbol group and each + // experiment produces exactly one aggregate doc per group, so + // RUN_ID:experiment-UUID identifies it stably (no :symbol suffix) and + // the flusher's bulk retries become idempotent overwrites. + elastic::enqueueDocument( + outcome_index::weeklyIndex(outcome_index::kExperimentsBase, + r.runConfig.BATCH), + nlohmann::json(r).dump(), + r.RUN_ID + ":" + r.experiment.UUID); +} diff --git a/source/analysis/reporting/experimentResults.hpp b/source/analysis/reporting/experimentResults.hpp new file mode 100644 index 0000000..6c56735 --- /dev/null +++ b/source/analysis/reporting/experimentResults.hpp @@ -0,0 +1,133 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "shared/experiments/experimentConfig.hpp" +#include "shared/experiments/experimentRunConfiguration.hpp" + +// Wire shape pushed to Elasticsearch by the analysis worker: ONE aggregate +// document per experiment x symbol group — the run descriptor and the full +// experiment chain echoed for Kibana filtering, plus the occurrence counts. +// Serialises the timestamp as `@timestamp` for Kibana (TradingResults' +// convention). +struct ExperimentResults { + std::string RUN_ID; + std::string timestamp; // serialised as @timestamp + std::string hostname; // machine that ran the analysis + double durationSeconds = 0.0; // wall-clock seconds for this evaluation + experiments::ExperimentRunConfiguration runConfig; // descriptor echo + experiments::ExperimentConfig experiment; // full CHAIN echo + std::uint64_t occurrences = 0; + std::uint64_t ticksScanned = 0; + double daysSpanned = 0.0; + double occurrencesPerDay = 0.0; // 0 when the stream spanned no time + // Conditionality: leg-1 completions (the attempt denominator) and chain + // deaths attributed to the leg being sought — completionRate is DERIVED + // in to_json (null, never a fake 0, when attempts == 0). + std::uint64_t attempts = 0; + std::vector failuresByLeg; + // Stability: completions bucketed by the completing tick's UTC calendar + // slot (absent months are absent keys) and hour-of-day. + std::map occurrencesByMonth; + std::array occurrencesByHourUtc{}; + // Per-symbol breakdown; the flat v1 map stays for compatibility. + std::map perSymbol; + std::map occurrencesBySymbol; + // Magnitude/timing/cost (phase 2) — see ExperimentOutcome for the + // semantics; nullopt populations serialise as JSON null. + std::optional completedAttempts; + std::optional failedAttempts; + std::optional completionSeconds; + std::optional meanSpreadAtTriggerPoints; + std::string excursionOrientation; // "up" | "down" + bool samplesTruncated = false; +}; + +inline void to_json(nlohmann::json& j, const ExperimentResults& r) { + j = nlohmann::json{ + {"RUN_ID", r.RUN_ID}, + {"@timestamp", r.timestamp}, + {"hostname", r.hostname}, + {"durationSeconds", r.durationSeconds}, + {"runConfig", r.runConfig}, + {"experiment", r.experiment}, + {"occurrences", r.occurrences}, + {"ticksScanned", r.ticksScanned}, + {"daysSpanned", r.daysSpanned}, + {"occurrencesPerDay", r.occurrencesPerDay}, + {"attempts", r.attempts}, + {"failuresByLeg", r.failuresByLeg}, + {"occurrencesByMonth", r.occurrencesByMonth}, + {"occurrencesByHourUtc", r.occurrencesByHourUtc}, + {"occurrencesBySymbol", r.occurrencesBySymbol}, + }; + // Null, never a fake 0.0: a chain whose leg 1 never fired has NO + // completion rate — 0 would read as "always fails". + j["completionRate"] = + r.attempts > 0 + ? nlohmann::json(static_cast(r.occurrences) / r.attempts) + : nlohmann::json(nullptr); + // Built by hand so the model header (experimentConfig.hpp) stays + // JSON-free — the doc shape belongs here. + nlohmann::json perSymbol = nlohmann::json::object(); + for (const auto& [symbol, s] : r.perSymbol) { + perSymbol[symbol] = nlohmann::json{ + {"occurrences", s.occurrences}, + {"ticksScanned", s.ticksScanned}, + {"daysSpanned", s.daysSpanned}, + }; + } + j["perSymbol"] = std::move(perSymbol); + + // Phase-2 magnitude/timing/cost: empty populations are JSON null, never + // fake zeros (the completionRate doctrine). + const auto quantileJson = [](const experiments::QuantilePair& q) { + return nlohmann::json{{"p50", q.p50}, {"p90", q.p90}}; + }; + const auto excursionJson = + [&quantileJson](const std::optional& s) { + if (!s) { + return nlohmann::json(nullptr); + } + return nlohmann::json{ + {"samples", s->samples}, + {"mfePoints", quantileJson(s->mfePoints)}, + {"maePoints", quantileJson(s->maePoints)}, + {"mfePercent", quantileJson(s->mfePercent)}, + {"maePercent", quantileJson(s->maePercent)}, + }; + }; + j["completedAttempts"] = excursionJson(r.completedAttempts); + j["failedAttempts"] = excursionJson(r.failedAttempts); + j["completionSeconds"] = r.completionSeconds + ? quantileJson(*r.completionSeconds) + : nlohmann::json(nullptr); + j["meanSpreadAtTriggerPoints"] = + r.meanSpreadAtTriggerPoints ? nlohmann::json(*r.meanSpreadAtTriggerPoints) + : nlohmann::json(nullptr); + j["excursionOrientation"] = r.excursionOrientation; + j["samplesTruncated"] = r.samplesTruncated; + // Top-level copies of the batch identity (also present under runConfig.*) + // so Kibana filters don't reach into the object — appendBatchMetadata's + // convention: absent keys, not empty strings, for pre-batch payloads. + if (!r.runConfig.EXECUTION_TS.empty()) { + j["executionTimestamp"] = r.runConfig.EXECUTION_TS; + } + if (!r.runConfig.BATCH.empty()) { + j["batch"] = r.runConfig.BATCH; + } +} diff --git a/source/experiments/config/dipRecovery/dipRecoverySweep.cppm b/source/experiments/config/dipRecovery/dipRecoverySweep.cppm new file mode 100644 index 0000000..e34889a --- /dev/null +++ b/source/experiments/config/dipRecovery/dipRecoverySweep.cppm @@ -0,0 +1,59 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include "shared/experiments/experimentConfig.hpp" + +export module dipRecoverySweep; + +export import parameterGenerator; // sweep::ParameterGenerator, Combination + +import std; +import makeDipRecovery; // sweep::makeDipRecoveryExperiment + +export namespace sweep { + +// Maps a swept combination onto a concrete experiment's config (minting a +// fresh UUID) — the experiment analogue of StrategyFactory. +using ExperimentFactory = + experiments::ExperimentConfig (*)(const Combination&); + +// One experiment sweep, fully specified: the parameter grid, the factory +// that maps each combination onto an ExperimentConfig, and the tick-history +// window the sweep runs over. The window is PER-SWEEP by design — each sweep +// builder declares its own LAST_MONTHS/OFFSET_MONTHS (default 9/0) colocated +// with the grid, rather than a global constant. +struct ExperimentSweepSpec { + ParameterGenerator generator; + ExperimentFactory factory = nullptr; + int lastMonths = 9; + int offsetMonths = 0; +}; + +// The first experiment sweep: how often does a sharp dip recover? The grid +// crosses drop size x drop window x recovery size x recovery window. +ExperimentSweepSpec buildDipRecoverySweep() { + ParameterGenerator generator; + // How deep the dip is, in percent of the pre-dip price (mapped to a + // negative DirectionalMove by makeDipRecoveryExperiment). + generator.addList("LEG1_DROP_PERCENT", {0.25, 0.5, 1.0, 2.0}); + // How fast the dip must land: the trailing window the drop is measured + // over (rolling extremes, not an anchored start). + generator.addList("LEG1_WINDOW_MINUTES", {5, 10, 30, 60}); + // How much of the dip must come back... + generator.addList("LEG2_RISE_PERCENT", {0.1, 0.25, 0.5, 1.0}); + // ...and how quickly, anchored at the dip's completion tick. + generator.addList("LEG2_WINDOW_MINUTES", {5, 10, 30, 60}); + return ExperimentSweepSpec{ + .generator = std::move(generator), + .factory = makeDipRecoveryExperiment, + .lastMonths = 9, + .offsetMonths = 0, + }; +} + +} // namespace sweep diff --git a/source/experiments/config/dipRecovery/makeDipRecovery.cppm b/source/experiments/config/dipRecovery/makeDipRecovery.cppm new file mode 100644 index 0000000..3532171 --- /dev/null +++ b/source/experiments/config/dipRecovery/makeDipRecovery.cppm @@ -0,0 +1,53 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/experiments/experimentConfig.hpp" + +export module makeDipRecovery; + +import std; +import sweepCombination; // sweep::Combination + +export namespace sweep { + +// Map one swept parameter combination onto a dipRecovery experiment: "price +// drops LEG1_DROP_PERCENT within LEG1_WINDOW_MINUTES, then rises +// LEG2_RISE_PERCENT within a further LEG2_WINDOW_MINUTES". Every parameter is +// read with get/getInt and no has() fallback: the sweep registers every name +// read here (buildDipRecoverySweep), so a missing one is a bug that should +// throw at queue time — the makeFvgStrategy doctrine. +experiments::ExperimentConfig makeDipRecoveryExperiment( + const sweep::Combination& combo) { + using experiments::Activity; + using experiments::ActivityType; + return experiments::ExperimentConfig{ + .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), + .NAME = "dipRecovery", + .CHAIN = { + // Leg 1: the dip — a NEGATIVE directional move (the grid sweeps + // the drop as a positive magnitude; the sign lives here). + Activity{ + .TYPE = ActivityType::DirectionalMove, + .MOVE_PERCENT = -combo.get("LEG1_DROP_PERCENT"), + .WINDOW_SECONDS = combo.getInt("LEG1_WINDOW_MINUTES") * 60, + }, + // Leg 2: the recovery — a positive move from the dip's + // completion tick (anchored by the matcher). + Activity{ + .TYPE = ActivityType::DirectionalMove, + .MOVE_PERCENT = combo.get("LEG2_RISE_PERCENT"), + .WINDOW_SECONDS = combo.getInt("LEG2_WINDOW_MINUTES") * 60, + }, + }, + }; +} + +} // namespace sweep diff --git a/source/experiments/experimentsCommand.cppm b/source/experiments/experimentsCommand.cppm new file mode 100644 index 0000000..a1f2135 --- /dev/null +++ b/source/experiments/experimentsCommand.cppm @@ -0,0 +1,244 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include + +#include +#include + +#include "run/reporting/elasticPublisher.hpp" // ensureIndexExists, repointAlias +#include "run/reporting/outcomeIndices.hpp" // kExperimentsBase + weekly names +#include "shared/utilities/env.hpp" +#include "shared/utilities/queueKeys.hpp" +#include "load/redisLoader.hpp" +#include "shared/experiments/experimentConfig.hpp" +#include "shared/experiments/experimentRunConfiguration.hpp" + +export module experimentsCommand; + +import std; +import backtestLog; // backtest_log::logLine +export import dipRecoverySweep; // sweep::ExperimentSweepSpec/ExperimentFactory, + // buildDipRecoverySweep (re-exported so tests + // reach the whole seam via one import) +import runConfigurationBuilder; // sweep::resolveSymbolGroups, currentBatchStamp +import symbolGroups; // sweep::cleanSymbols + +export namespace sweep { + +// Builds the keyed payloads for grid indices [begin, end): each combination +// is decoded lazily (combinationAt), mapped through the factory, and keyed +// under queue_keys::experimentPayloadKey(runId, ) — the +// exact parallel of buildStrategyChunk. Exported so tests can pin the +// chunk/key/payload contract without Redis. +std::vector buildExperimentChunk( + const ParameterGenerator& generator, + ExperimentFactory experimentFactory, + const std::string& runId, + std::size_t begin, + std::size_t end); + +} // namespace sweep + +export class ExperimentsCommand { +public: + static int run(int argc, const char* argv[]); +}; + +namespace { + +// Same digit grouping as LoadCommand (see loadCommand.cppm for why this is +// hand-rolled rather than {:L}). +std::string withThousands(const std::size_t n) { + std::string s = std::to_string(n); + for (std::size_t pos = s.size(); pos > 3;) { + pos -= 3; + s.insert(pos, ","); + } + return s; +} + +// Payloads per pipelined Redis request / progress heartbeat — LoadCommand's +// values, for the same bounded-memory reasons. +constexpr std::size_t kChunkSize = 1000; +constexpr std::size_t kProgressEvery = 100'000; + +// Feeds RedisLoader::loadKeyedPayloadStream one lazily-built chunk at a time. +// A copy of SweepChunkSource with the factory type swapped — deliberately not +// templated/shared: two small concrete classes read better than one generic +// seam, and the pair can drift independently. +class ExperimentChunkSource final : public RedisLoader::ChunkSource { +public: + ExperimentChunkSource(const sweep::ParameterGenerator& generator, + const sweep::ExperimentFactory experimentFactory, + std::string runId, + const std::size_t total) + : generator_(generator), + experimentFactory_(experimentFactory), + runId_(std::move(runId)), + total_(total) {} + + std::vector next() override { + const std::size_t begin = next_; + const std::size_t end = std::min(begin + kChunkSize, total_); + next_ = end; + if (begin != 0 && begin % kProgressEvery == 0) { + backtest_log::logLine("ExperimentsCommand: queued {}/{} experiments...", + withThousands(begin), withThousands(total_)); + } + return sweep::buildExperimentChunk(generator_, experimentFactory_, + runId_, begin, end); + } + +private: + const sweep::ParameterGenerator& generator_; + sweep::ExperimentFactory experimentFactory_; + std::string runId_; + std::size_t total_; + std::size_t next_ = 0; +}; + +// Create this batch's weekly experiments index and atomically repoint its +// -current alias — kExperimentsBase ONLY (deliberately not in kWeeklyBases: +// `load` must not create empty experiment indices, and `experiments` has no +// business touching the outcome bases). Best-effort, same doctrine as +// LoadCommand::prepareWeeklyOutcomeIndices; with $ELASTIC_ENABLED=0 every +// call is a successful no-op, keeping a Redis-only local run Elastic-free. +void prepareWeeklyExperimentsIndex(const std::string& batchLabel) { + const std::string index = + outcome_index::weeklyIndex(outcome_index::kExperimentsBase, batchLabel); + if (elastic::ensureIndexExists(index) != 0 || + elastic::repointAlias( + outcome_index::currentAlias(outcome_index::kExperimentsBase), + index) != 0) { + backtest_log::logLine( + "ExperimentsCommand: weekly index/alias admin failed at {} — the " + "next successful experiments load repoints the alias", + index); + } +} + +} // namespace + +std::vector sweep::buildExperimentChunk( + const ParameterGenerator& generator, + const ExperimentFactory experimentFactory, + const std::string& runId, + const std::size_t begin, + const std::size_t end) { + std::vector chunk; + chunk.reserve(end - begin); + for (std::size_t i = begin; i < end; ++i) { + const experiments::ExperimentConfig config = + experimentFactory(generator.combinationAt(i)); + // Pin to JSON explicitly to trigger the implicit conversion for + // .dump() — buildStrategyChunk's pattern. + const nlohmann::json j = config; + chunk.push_back({queue_keys::experimentPayloadKey(runId, config.UUID), + j.dump()}); + } + return chunk; +} + +int ExperimentsCommand::run(const int argc, const char* argv[]) { + + // Select the sweep from the command line, e.g. `experiments dipRecovery`. + // The name is required — omitting it or passing an unknown name is a + // usage error, not a crash, so report the valid choices and bail. A new + // sweep is one extra branch here plus a mention in the error. + const std::string_view sweepName = argc > 2 ? argv[2] : ""; + sweep::ExperimentSweepSpec spec; + if (sweepName == "dipRecovery") { + spec = sweep::buildDipRecoverySweep(); + } else { + std::println(stderr, + "ExperimentsCommand: unknown experiment sweep '{}' " + "(valid: dipRecovery)", + sweepName); + return 1; + } + + // Surface the full sweep size and wait for confirmation BEFORE anything + // touches Redis — LoadCommand's gate, for the same reasons. EOF (closed + // stdin) counts as a decline so a non-interactive invocation can't sail + // past the prompt. + const auto symbolGroups = + sweep::resolveSymbolGroups(spec.generator.symbolGroups()); + const auto combinationCount = spec.generator.combinationCount(); + backtest_log::logLine( + "ExperimentsCommand: '{}' sweep: {} experiment(s) x {} symbol group(s) = {} evaluation(s)", + sweepName, withThousands(combinationCount), symbolGroups.size(), + withThousands(combinationCount * symbolGroups.size())); + + std::print("Press Enter to queue them (Ctrl+C to abort)... "); + std::fflush(stdout); + if (std::string ack; !std::getline(std::cin, ack)) { + std::println(stderr, "ExperimentsCommand: aborted, no confirmation on stdin"); + return 1; + } + + // Freeze this load's batch identity before anything is queued, then + // prepare the weekly experiments index + alias (see LoadCommand for the + // seed-not-workers doctrine). + const sweep::BatchStamp batch = sweep::currentBatchStamp(); + backtest_log::logLine("ExperimentsCommand: batch {} (execution {})", + batch.label, batch.executionTs); + prepareWeeklyExperimentsIndex(batch.label); + + const auto redisHost = env::getOr("REDIS_HOST", "127.0.0.1"); + + // One loader = one persistent Redis connection shared by every run below. + RedisLoader loader(redisHost, 6379); + + // Fan out: every resolved symbol group becomes its own run (its own + // RUN_ID, experiment list and run descriptor on the EXPERIMENT_RUN + // queue). A comma-separated entry is one run over multiple instruments. + for (const auto& group : symbolGroups) { + const std::string symbols = sweep::cleanSymbols(group); + + const auto runId = + boost::uuids::to_string(boost::uuids::random_generator()()); + + backtest_log::logLine( + "ExperimentsCommand: sweeping {} experiment(s) for RUN_ID={} symbols={}", + withThousands(combinationCount), runId, symbols); + + // Stream every experiment into Redis BEFORE the run descriptor + // (payloads first, then names — RedisLoader keeps that order per + // chunk), so the full set is present the moment the run is visible. + ExperimentChunkSource source(spec.generator, spec.factory, runId, + combinationCount); + if (const auto payloadStatus = loader.loadKeyedPayloadStream( + queue_keys::experimentKey(runId), source, + queue_keys::PAYLOAD_TTL_SECONDS); + payloadStatus != 0) + { + return payloadStatus; + } + + // Now advertise the run so analysis workers can pick it up. The + // descriptor carries the sweep's own tick window (per-sweep + // LAST_MONTHS/OFFSET_MONTHS — see ExperimentSweepSpec). + const nlohmann::json runJson = experiments::ExperimentRunConfiguration{ + .RUN_ID = runId, + .SYMBOLS = symbols, + .BATCH = batch.label, + .EXECUTION_TS = batch.executionTs, + .LAST_MONTHS = spec.lastMonths, + .OFFSET_MONTHS = spec.offsetMonths, + }; + if (const auto runStatus = + loader.loadPayload(queue_keys::EXPERIMENT_RUN, runJson.dump()); + runStatus != 0) + { + return runStatus; + } + } + + return 0; +} diff --git a/source/ingest/ingestCommand.cppm b/source/ingest/ingestCommand.cppm index 6110ff3..c6c58b2 100644 --- a/source/ingest/ingestCommand.cppm +++ b/source/ingest/ingestCommand.cppm @@ -14,18 +14,17 @@ module; -#include // POSIX gmtime_r (not exported by `import std`) - #include "ingest/questdbIngestClient.hpp" -#include "ingest/udpPorts.hpp" -#include "ingest/udpReceiver.hpp" +#include "shared/net/udpPorts.hpp" +#include "shared/net/udpReceiver.hpp" #include "shared/utilities/env.hpp" export module ingestCommand; import std; +import backtestLog; // backtest_log::logLine — timestamped, flushed stdout import priceData; // PriceData -import tickPacket; // ingest::decodeTick +import tickPacket; // tick_packet::decodeTick export class IngestCommand { public: @@ -45,36 +44,11 @@ std::uint16_t parsePort(std::string_view text, std::uint16_t fallback) { return static_cast(value); } -// UTC wall-clock prefix, e.g. "[2026-06-28 15:04:20.785]", to millisecond -// precision. gmtime_r is POSIX (hence the include in the global module -// fragment) — the same approach tradeManager.cppm uses for UTC tick times. -std::string timestamp() { - const auto now = std::chrono::system_clock::now(); - const std::time_t t = std::chrono::system_clock::to_time_t(now); - const auto millis = std::chrono::duration_cast( - now.time_since_epoch()) - .count() % - 1000; - std::tm utc{}; - gmtime_r(&t, &utc); - return std::format("[{:04}-{:02}-{:02} {:02}:{:02}:{:02}.{:03}]", - utc.tm_year + 1900, utc.tm_mon + 1, utc.tm_mday, - utc.tm_hour, utc.tm_min, utc.tm_sec, millis); -} - -// Like std::println to stdout, but timestamped and flushed immediately. stdout -// is fully buffered when redirected (pipe/file/service log), so without the -// flush the once-a-minute reports below would sit in the buffer and only appear -// in a burst when the process exits. -template -void logLine(std::format_string fmt, Args&&... args) { - std::println("{} {}", timestamp(), std::format(fmt, std::forward(args)...)); - std::fflush(stdout); -} - } // namespace int IngestCommand::run(const int argc, const char* argv[]) { + using backtest_log::logLine; + const std::string questHost = env::getOr("QUESTDB_HOST", "127.0.0.1"); // QuestDB serves ILP-over-HTTP (POST /write) on its main HTTP port, 9000 by // default. Override with $QUESTDB_ILP_PORT. @@ -92,9 +66,9 @@ int IngestCommand::run(const int argc, const char* argv[]) { std::atomic received{0}; std::atomic dropped{0}; - ingest::UdpReceiver receiver( + net::UdpReceiver receiver( bindAddr, bindPort, [&](std::span bytes) { - const auto tick = ingest::decodeTick(bytes); + const auto tick = tick_packet::decodeTick(bytes); if (!tick) { dropped.fetch_add(1, std::memory_order_relaxed); return; diff --git a/source/ingest/udpPorts.hpp b/source/ingest/udpPorts.hpp deleted file mode 100644 index 931753d..0000000 --- a/source/ingest/udpPorts.hpp +++ /dev/null @@ -1,20 +0,0 @@ -// Backtesting Engine in C++ -// -// (c) 2026 Ryan McCaffery | https://mccaffers.com -// This code is licensed under MIT license (see LICENSE.txt for details) -// --------------------------------------- - -#pragma once - -#include - -// The C# streamer (vortex/shared/UDPPorts.cs) fans each serialized tick out to -// several UDP sinks. This engine only consumes the persistence stream, so the -// ingest cares about exactly one port: PortSave. Keep kSave in lockstep with -// UDPPorts.PortSave. The bind port is also overridable at runtime -// (CLI arg / $INGEST_UDP_PORT). -namespace udp_ports { - -inline constexpr std::uint16_t kSave = 11111; // == UDPPorts.PortSave - -} // namespace udp_ports diff --git a/source/live/broker/igMarkets.cppm b/source/live/broker/igMarkets.cppm new file mode 100644 index 0000000..a5ade74 --- /dev/null +++ b/source/live/broker/igMarkets.cppm @@ -0,0 +1,286 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// igMarkets — the IG broker wire shapes and the order seams, mirroring the +// C# engine's TradeOpenObj / TradeCloseObj / IGPositionResponseObject / +// MarketElement models. Everything here is either a serialisation target +// (field spelling matches the IG REST API on purpose) or a pure codec over +// one — the HTTP/gating machinery lives in igRequests, the flow logic in +// orderChannel. +// +// IG's OTC dealing model, for reference: POST /positions/otc opens; closing +// is the SAME endpoint with an extra "_method: DELETE" header (Version 1). +// Both answer with { dealReference } only — the dealId exists once the deal +// confirms, and the external position producer reconciles it into the PL# +// book from the broker's own position list. + +module; + +#include + +export module igMarkets; + +import std; // replaces , , , , + +export namespace ig { + +// POST /positions/otc request body — the C# TradeOpenObj. Distances are in +// pips; a 0 distance means that leg is disarmed and encodeTradeOpen omits +// the field (IG rejects a literal zero distance). size is decimal at the +// broker (mini contracts), so the engine's integer TRADING_SIZE arrives +// here already multiplied by the market's sizeModifier. +struct TradeOpenObj { + std::string currencyCode; + std::string epic; + std::string expiry = "-"; + std::string direction; // "BUY" | "SELL" + double size{}; + bool forceOpen = true; + bool guaranteedStop = false; + std::string orderType = "MARKET"; + std::int32_t stopDistance{}; // pips; 0 = no stop leg + std::int32_t limitDistance{}; // pips; 0 = no limit leg + // Client-generated idempotency token ([A-Za-z0-9_-], max 30 chars); IG + // echoes it in the response and the confirm stream so fills can be + // matched to requests. Omitted from the JSON when empty. + std::string dealReference; +}; + +// POST /positions/otc + "_method: DELETE" request body — the C# TradeCloseObj. +// direction is the CLOSING direction (opposite of the open position's). +struct TradeCloseObj { + std::string orderType = "MARKET"; + std::string direction; // "BUY" | "SELL" + std::string dealId; + double size{}; +}; + +// The open/close response body — the C# IGPositionResponseObject. errorCode +// is null on success. +struct IGPositionResponseObject { + std::string dealReference; + std::optional errorCode; +}; + +// GET /confirms/{dealReference} response (Version 1) — the deal's fate. +// dealStatus is definitive once present: "ACCEPTED" or "REJECTED"; anything +// else (or a 404 while the confirm propagates) reads as still pending. +struct DealConfirmation { + std::string dealId; + std::string dealReference; + std::string dealStatus; // "ACCEPTED" | "REJECTED" | pending/unknown + std::string reason; // rejection reason; "SUCCESS" on accepts +}; + +// GET /markets market payload — the C# MarketElement. Parsed by the REST +// client when the positions read path lands; decimals arrive as double at +// this boundary. +struct MarketElement { + std::string instrumentName; + std::string expiry; + std::string epic; + std::string instrumentType; + double lotSize{}; + double high{}; + double low{}; + double percentageChange{}; + double netChange{}; + double bid{}; + double offer{}; + std::string updateTime; + int delayTime{}; + bool streamingPricesAvailable{}; + std::string marketStatus; + int scalingFactor{}; +}; + +// One deal inside GET /positions — the C# PositionElement. Nullable broker +// fields stay optional: absent in the JSON means absent here. +struct PositionElement { + double contractSize{}; + std::string createdDate; + std::string dealId; + double dealSize{}; + std::string dealReference; + std::string direction; // "BUY" | "SELL" + std::optional limitLevel; + double openLevel{}; + std::string currency; + bool controlledRisk{}; + std::optional stopLevel; + std::optional trailingStep; + std::optional trailingStopDistance; + std::optional limitedRiskPremium; +}; + +// GET /positions pairs each deal with its market snapshot. +struct Position { + std::optional position; + std::optional market; +}; + +struct AccountPositions { + std::vector positions; +}; + +// ---- codecs (pure, unit-tested without a broker) ---- + +// The request bodies, field names exactly as IG (and the C# serializer) +// spell them. +std::string encodeTradeOpen(const TradeOpenObj& order); +std::string encodeTradeClose(const TradeCloseObj& close); + +// Parse an open/close response body. nullopt when the body is not a JSON +// object (HTML error pages, truncation) — the C# "Failed to parse response" +// branch. A missing dealReference parses as empty (the caller decides what +// that means). +std::optional parsePositionResponse( + const std::string& body); + +// Parse a confirms body. nullopt for non-objects; missing fields parse as +// empty (the caller treats an empty dealStatus as still pending). +std::optional parseDealConfirmation(const std::string& body); + +// Closing a position trades the OPPOSITE side: BUY position -> SELL order. +std::string closingDirection(std::string_view openBrokerDirection); + +// ---- order seams ---- + +// Everything about the originating decision the broker call layer needs +// beyond the wire payload: the request-gate duplicate key is +// strategyUuid + openDirection (C# $"{strategyId}{reqObj.direction}" — note +// the OPEN direction even on a close request, so a close within the open's +// 30s suppression window is refused, exactly as in C#), and the live-trades +// audit trail wants symbol/strategy. +struct OrderContext { + std::string strategyUuid; + std::string strategyName; + std::string symbol; + std::string openDirection; // broker vocabulary: "BUY" | "SELL" +}; + +// Outcome of one placement attempt, split the way the order channel reacts: +// Accepted — IG took the request; dealReference tracks the deal until the +// confirm/producer supplies the dealId. +// Rejected — a definitive broker NO: safe to release the trade lock and +// re-enter early. (The HTTP path currently maps nothing here — +// a non-2xx could mean the order half-exists, so it fails +// conservatively; the confirms endpoint will populate this.) +// Failed — no definitive answer (transport error, non-2xx, unparseable +// body): the C# "trade didn't complete" path; the lock is +// extended because the order MAY still be live at the broker. +enum class OpenStatus { Accepted, Rejected, Failed }; + +struct OpenResult { + OpenStatus status{OpenStatus::Failed}; + std::string dealReference; // Accepted only (IG's echo, not ours) + // Broker deal id from the confirms poll; EMPTY when the confirm never + // resolved (the deal is booked anyway and cannot be strategy-closed + // until an id appears — the close path's blank-dealId guard). + std::string dealId; + std::string reason; // Rejected / Failed diagnostics +}; + +// Outcome of one close attempt: +// Ok — IG accepted the close request. +// Gone — the request layer produced no response at all; the C# engine +// treats this as "missing from IG" and deletes the book entry so +// a phantom position cannot haunt the strategy logic forever. +// Failed — IG answered but not OK (or unparseably): the position still +// exists as far as anyone knows; keep it on the book. +enum class CloseStatus { Ok, Gone, Failed }; + +struct CloseResult { + CloseStatus status{CloseStatus::Failed}; + std::string reason; +}; + +using PlaceOrder = + std::function; +using PlaceClose = + std::function; + +} // namespace ig + +namespace ig { + +std::string encodeTradeOpen(const TradeOpenObj& order) { + nlohmann::json body{ + {"currencyCode", order.currencyCode}, + {"epic", order.epic}, + {"expiry", order.expiry}, + {"direction", order.direction}, + {"size", order.size}, + {"forceOpen", order.forceOpen}, + {"guaranteedStop", order.guaranteedStop}, + {"orderType", order.orderType}, + }; + if (order.stopDistance > 0) { + body["stopDistance"] = order.stopDistance; + } + if (order.limitDistance > 0) { + body["limitDistance"] = order.limitDistance; + } + if (!order.dealReference.empty()) { + body["dealReference"] = order.dealReference; + } + return body.dump(); +} + +std::string encodeTradeClose(const TradeCloseObj& close) { + const nlohmann::json body{ + {"orderType", close.orderType}, + {"direction", close.direction}, + {"dealId", close.dealId}, + {"size", close.size}, + }; + return body.dump(); +} + +std::optional parsePositionResponse( + const std::string& body) { + const nlohmann::json parsed = + nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (!parsed.is_object()) { + return std::nullopt; + } + IGPositionResponseObject response; + if (const auto it = parsed.find("dealReference"); + it != parsed.end() && it->is_string()) { + response.dealReference = it->get(); + } + if (const auto it = parsed.find("errorCode"); + it != parsed.end() && it->is_string()) { + response.errorCode = it->get(); + } + return response; +} + +std::optional parseDealConfirmation(const std::string& body) { + const nlohmann::json parsed = + nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (!parsed.is_object()) { + return std::nullopt; + } + DealConfirmation confirmation; + const auto readString = [&parsed](const char* key, std::string& out) { + if (const auto it = parsed.find(key); + it != parsed.end() && it->is_string()) { + out = it->get(); + } + }; + readString("dealId", confirmation.dealId); + readString("dealReference", confirmation.dealReference); + readString("dealStatus", confirmation.dealStatus); + readString("reason", confirmation.reason); + return confirmation; +} + +std::string closingDirection(const std::string_view openBrokerDirection) { + return openBrokerDirection == "BUY" ? "SELL" : "BUY"; +} + +} // namespace ig diff --git a/source/live/broker/igRequests.cppm b/source/live/broker/igRequests.cppm new file mode 100644 index 0000000..be13f7c --- /dev/null +++ b/source/live/broker/igRequests.cppm @@ -0,0 +1,595 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// igRequests — the C# engine's IGMarketRequests, split into its two halves: +// +// IGMarketRequests the guarded request path every broker call funnels +// through: duplicate-deal suppression (API#, +// 30s) -> shared rate budget (REQ#, 30/min, +// shared with the C# engine via Redis) -> record both -> +// session credentials -> the retried HTTPS exchange +// (shared/ig/igRestClient). OPENS fail closed: an +// unknown counter sends nothing — a skipped entry is +// recoverable, a duplicate or unaccountable one is not. +// CLOSES are risk-reducing and fail OPEN on gate +// uncertainty (and bypass the soft budget): a skipped +// close leaves live exposure on the book, which is the +// one outcome worse than an extra request — a doubled +// close is rejected safely by IG. Only a definite +// duplicate marker or missing session refuses a close. +// +// IGMarketCalls the open/close calls over that path. The mapping +// cores (makeOpen/makeClose over an injected RequestFn) +// are pure response interpretation, unit-tested without +// Redis or a network; makeLiveOpen/makeLiveClose bind +// them to a per-worker-thread IGMarketRequests (same +// thread_local pattern as RedisTradeGate — the Redis +// budget is shared server-side, so per-thread instances +// don't weaken it) and add the live-trades Elasticsearch +// audit line on success. +// +// The GMF includes are all Asio-free headers (curl and Boost stay behind +// the .cpp implementations). + +module; + +#include + +#include "run/reporting/elasticPublisher.hpp" +#include "shared/aws/dynamoAuth.hpp" +#include "shared/ig/igRestClient.hpp" +#include "shared/redis/apiRequestGate.hpp" + +export module igRequests; + +import std; // replaces , , , , +import igMarkets; // ig::TradeOpenObj/TradeCloseObj, OpenResult, CloseResult, codecs +import backtestLog; // backtest_log::logLine +import liveTrace; // live_trace::emit/tradingEnv — live-traces documents + +export namespace ig { + +// Session credential source. Injectable for tests; live uses the DynamoDB +// store the external login service refreshes (CST/X-SECURITY-TOKEN expire). +using AuthProvider = std::function()>; + +// The C# Auth.PullWithRetry: MarketDataLive item "Auth#" +// ("live" / "demo"), pulled FRESH per broker request so a token the login +// service just rotated is picked up immediately. AWS credentials/region +// come from the process environment (SDK default chain). +AuthProvider dynamoAuthProvider(std::string environment); + +// Per-request gate/transport configuration, set by the market call that +// owns the request's semantics (open vs close vs confirm poll). +struct RequestOptions { + // Duplicate-suppression key (API#, 30s). "" = no dedup marker + // (the confirms GET). Opens key on strategyUuid+direction; closes key on + // "close#" — DISTINCT namespaces, so an open's marker can never + // refuse the close that follows it. + std::string dealKey; + // True for closes: gate uncertainty (Redis unreachable) and the soft + // budget must not refuse a risk-reducing order. Opens stay fail-closed. + bool riskReducing = false; + // Transport retries inside ig_rest::execute. 0 for the NON-IDEMPOTENT + // open POST (a blind re-send after a lost ACK can double a position — + // makeOpen resolves ambiguity through /confirms instead). + int transportRetries = 2; +}; + +// What actually happened to a guarded request. The old conflation of +// "refused by the gate" with "sent but transport failed" into one nullopt +// is exactly what let a refused close masquerade as a missing position. +enum class RequestFate { + Refused, // never sent: gate refusal or no session credentials + TransportFailed, // sent (or mid-send): no HTTP exchange completed + Responded, // an HTTP response arrived — response is engaged +}; + +struct RequestOutcome { + RequestFate fate{RequestFate::Refused}; + std::optional response; // Responded only + std::string detail; // refusal reason / transport note for logs & traces +}; + +// The seam between the market calls and the guarded request path — the C# +// IGMarketRequests.Request signature, carrying the outcome's fate. +using RequestFn = std::function; + +// Why the gate refused (nullopt from gatePolicy = proceed). Distinct values +// keep the live-trace `reason` fields diagnosable. +enum class GateRefusal { + DuplicateDeal, // marker definitely present + DuplicateUnknown, // Redis unreachable, fail-closed (opens only) + BudgetUnknown, // Redis unreachable, fail-closed (opens only) + RateLimited, // budget known and exhausted (opens only) +}; + +// The pure gate-policy core, unit-tested without Redis (same doctrine as +// the makeOpen/makeClose mapping cores). `duplicate` nullopt = unknown +// (Redis unreachable); `requestsMade` nullopt likewise. Risk-reducing +// requests proceed on any uncertainty and past the soft budget — only a +// DEFINITE duplicate refuses them. +std::optional gatePolicy(std::optional duplicate, + std::optional requestsMade, + bool riskReducing); + +class IGMarketRequests { +public: + // The C# MAX_REQUESTS_PER_MINUTE — a soft brake shared with the other + // engine through Redis; IG enforces the hard one. + static constexpr int kMaxRequestsPerMinute = 30; + + IGMarketRequests(const std::string& redisHost, int redisPort, + AuthProvider auth); + + // The guarded request path (see the header comment for the gate order). + RequestOutcome request(const std::string& path, const std::string& method, + const std::string& jsonBody, + const ig_rest::Headers& extraHeaders, + const RequestOptions& options); + +private: + redis_api::ApiRequestGate gate_; + AuthProvider auth_; +}; + +class IGMarketCalls { +public: + // Mapping cores — pure response interpretation over the injected + // request path. Open: POST /positions/otc; HTTP 200 with a parseable, + // error-free body is provisionally Accepted (IG's dealReference echo), + // then the confirm poll (GET /confirms/{dealReference}, Version 1) + // resolves the deal's fate: ACCEPTED fills the broker dealId, REJECTED + // maps to OpenStatus::Rejected (the channel's early-lock-release path). + // A confirm that never resolves keeps Accepted with an EMPTY dealId: + // the POST succeeded so IG has the order — Rejected here would release + // the lock (re-entry -> possible doubled exposure) and skip booking a + // possibly-live deal, while an empty dealId merely fails closed at + // close time. Anything murky before the echo (non-2xx, unparseable, + // errorCode on 200) stays Failed — booking a phantom deal is worse + // than a spurious 2-minute brake. + // confirmAttempts/confirmDelay bound the poll (tests pass 0ms; note + // each attempt spends the shared REQ# minute budget, so an accepted + // open costs up to 1 + confirmAttempts of the 30/min). + static PlaceOrder makeOpen( + RequestFn request, int confirmAttempts = 3, + std::chrono::milliseconds confirmDelay = std::chrono::milliseconds{300}); + + // Close: POST /positions/otc with "_method: DELETE" (Version 1). HTTP + // 200 + parseable body -> Ok; NO response from the request layer -> + // Gone (the C# "missing from IG" branch — the caller deletes the book + // entry); anything else -> Failed (the position may still exist). + static PlaceClose makeClose(RequestFn request); + + // Live bindings: one IGMarketRequests per calling worker thread, plus + // the live-trades Elasticsearch audit document on success (C# writes + // one on open; close is audited here too for a symmetric trail). + static PlaceOrder makeLiveOpen(std::string redisHost, int redisPort, + AuthProvider auth); + static PlaceClose makeLiveClose(std::string redisHost, int redisPort, + AuthProvider auth); +}; + +} // namespace ig + +namespace ig { + +namespace { + +int currentMinuteOfHour() { + const auto sinceEpoch = std::chrono::system_clock::now().time_since_epoch(); + return static_cast( + std::chrono::duration_cast(sinceEpoch).count() + % 60); +} + +// One IGMarketRequests per calling thread, lazily built — shared by the +// open and close bindings on that thread (same Redis budget either way). +RequestFn threadLocalRequestFn(std::string redisHost, const int redisPort, + AuthProvider auth) { + return [redisHost = std::move(redisHost), redisPort, + auth = std::move(auth)]( + const std::string& path, const std::string& method, + const std::string& jsonBody, + const ig_rest::Headers& extraHeaders, + const RequestOptions& options) { + thread_local std::unique_ptr requests; + if (!requests) { + requests = std::make_unique(redisHost, redisPort, + auth); + } + return requests->request(path, method, jsonBody, extraHeaders, + options); + }; +} + +// Polls GET /confirms/{confirmReference} until the deal's fate resolves. +// ACCEPTED maps to Accepted (reporting `reportReference` — the happy path +// preserves IG's echo, the ambiguity path reports the reference WE sent); +// REJECTED maps to Rejected. nullopt = the confirm never resolved within +// `attempts` — the CALLER decides what unresolved means (Accepted-with-empty- +// dealId after a clean 200, Failed after a transport-ambiguous POST). +std::optional pollConfirm(const RequestFn& request, + const std::string& confirmReference, + const std::string& reportReference, + const int attempts, + const std::chrono::milliseconds delay) { + for (int attempt = 0; attempt < attempts; ++attempt) { + if (attempt > 0 && delay.count() > 0) { + // The confirm is usually ready immediately; the delay only + // paces the retries while it propagates. + std::this_thread::sleep_for(delay); + } + // EMPTY dealKey on purpose: the open POST just recorded its own + // marker for 30s — reusing it would refuse this very confirm as a + // duplicate. Explicit Version 1 (confirms is a v1 endpoint; ig_rest + // lets a caller-supplied Version replace the versionFor default). + const RequestOutcome confirm = + request("/confirms/" + confirmReference, "GET", "", + {{"Version", "1"}}, RequestOptions{}); + if (confirm.fate != RequestFate::Responded + || confirm.response->status != 200) { + continue; // 404 while it propagates, gate refusal, 5xx + } + const auto confirmation = + parseDealConfirmation(confirm.response->body); + if (!confirmation) { + continue; + } + if (confirmation->dealStatus == "REJECTED") { + return OpenResult{.status = OpenStatus::Rejected, + .dealReference = confirmReference, + .reason = confirmation->reason.empty() + ? "REJECTED" + : confirmation->reason}; + } + if (confirmation->dealStatus == "ACCEPTED") { + return OpenResult{.status = OpenStatus::Accepted, + .dealReference = reportReference, + .dealId = confirmation->dealId}; + } + // Any other status = still pending — keep polling. + } + return std::nullopt; +} + +void auditTrade(const std::string& action, const OrderContext& context, + const std::string& dealReference, + const std::string& dealId = "") { + // The C# ElasticTradeLogs document shape, index "live-trades". Queued for + // the publisher's background flusher, so an Elastic outage can never stall + // an order worker mid-retry; delivery (with retry + dead-letter) happens + // in periodic _bulk batches. Disable via ELASTIC_ENABLED=0. + nlohmann::json doc{ + {"date", elastic::nowIsoUtc()}, + {"env", std::string(live_trace::tradingEnv())}, + {"symbol", context.symbol}, + {"action", action}, + {"strategy", context.strategyUuid}, + {"dealReference", dealReference}, + }; + if (!dealId.empty()) { + doc["dealId"] = dealId; + } + elastic::enqueueDocument("live-trades", doc.dump()); +} + +} // namespace + +AuthProvider dynamoAuthProvider(std::string environment) { + return [environment = std::move(environment)] { + return aws_auth::pullAuthWithRetry(environment); + }; +} + +IGMarketRequests::IGMarketRequests(const std::string& redisHost, + const int redisPort, AuthProvider auth) + : gate_(redisHost, redisPort), auth_(std::move(auth)) {} + +std::optional gatePolicy(const std::optional duplicate, + const std::optional requestsMade, + const bool riskReducing) { + // A DEFINITE duplicate refuses both classes: for a close this is pacing + // (its own close# marker), and a refused close now maps to + // Failed — the book entry survives and the sync loop retries after the + // 30s window. + if (duplicate.has_value() && *duplicate) { + return GateRefusal::DuplicateDeal; + } + if (riskReducing) { + // Risk-reducing (close): uncertainty and the soft budget never + // refuse — an unsent close leaves live exposure, which is worse + // than any doubled or over-budget request IG rejects safely. + return std::nullopt; + } + if (!duplicate.has_value()) { + return GateRefusal::DuplicateUnknown; + } + if (!requestsMade.has_value()) { + return GateRefusal::BudgetUnknown; + } + if (*requestsMade > IGMarketRequests::kMaxRequestsPerMinute) { + return GateRefusal::RateLimited; + } + return std::nullopt; +} + +namespace { + +std::string_view refusalReason(const GateRefusal refusal) { + switch (refusal) { + case GateRefusal::DuplicateDeal: return "duplicateDeal"; + case GateRefusal::DuplicateUnknown: return "duplicateUnknown"; + case GateRefusal::BudgetUnknown: return "budgetUnknown"; + case GateRefusal::RateLimited: return "rateLimit"; + } + return "unknown"; +} + +} // namespace + +RequestOutcome IGMarketRequests::request(const std::string& path, + const std::string& method, + const std::string& jsonBody, + const ig_rest::Headers& extraHeaders, + const RequestOptions& options) { + using backtest_log::logLine; + + // Gate reads first, then the pure policy. An empty dealKey has nothing + // to deduplicate (definite false, C# behaviour) — only a real key pays + // the Redis read. + const std::optional duplicate = + options.dealKey.empty() ? std::optional{false} + : gate_.isDuplicateDeal(options.dealKey); + const int minute = currentMinuteOfHour(); + const std::optional made = gate_.requestsMade(minute); + + if (const std::optional refusal = + gatePolicy(duplicate, made, options.riskReducing)) { + const std::string reason{refusalReason(*refusal)}; + logLine("IGMarketRequests: {} — refusing {} {} (dealKey={})", + reason, method, path, options.dealKey); + if (live_trace::enabled()) { + if (*refusal == GateRefusal::RateLimited) { + live_trace::emit("igRefused", {}, + {{"reason", reason}, + {"method", method}, + {"path", path}, + {"dealKey", options.dealKey}, + {"requestsThisMinute", + static_cast(*made)}}); + } else { + live_trace::emit("igRefused", {}, + {{"reason", reason}, + {"method", method}, + {"path", path}, + {"dealKey", options.dealKey}}); + } + } + return RequestOutcome{.fate = RequestFate::Refused, + .detail = reason}; + } + + // Record before sending, like the C# order: the budget must count an + // attempt even when the exchange itself then fails. Best-effort — a + // risk-reducing request proceeding through a Redis outage records + // nothing, which the policy above already priced in. + gate_.recordRequest(minute); + gate_.recordDealRequest(options.dealKey); + + const std::optional auth = auth_(); + if (!auth) { + logLine("IGMarketRequests: no IG session credentials — refusing {} {}" + " (is the login service writing Auth# to DynamoDB, and " + "are AWS credentials in the environment?)", + method, path); + if (live_trace::enabled()) { + live_trace::emit("igRefused", {}, + {{"reason", "noAuthSession"}, + {"method", method}, + {"path", path}, + {"dealKey", options.dealKey}}); + } + return RequestOutcome{.fate = RequestFate::Refused, + .detail = "noAuthSession"}; + } + + std::optional response = + ig_rest::execute(*auth, path, method, jsonBody, extraHeaders, + options.transportRetries); + if (!response) { + return RequestOutcome{.fate = RequestFate::TransportFailed, + .detail = "transport failed"}; + } + return RequestOutcome{.fate = RequestFate::Responded, + .response = std::move(response)}; +} + +PlaceOrder IGMarketCalls::makeOpen(RequestFn request, + const int confirmAttempts, + const std::chrono::milliseconds confirmDelay) { + return [request = std::move(request), confirmAttempts, confirmDelay]( + const TradeOpenObj& order, + const OrderContext& context) -> OpenResult { + // transportRetries = 0: the open POST is NOT idempotent, so a lost + // ACK is never resolved by re-sending — ambiguity goes through the + // confirms poll below, keyed on the dealReference WE minted (it is + // in the POST body, so IG can name the deal whether or not the + // response reached us). + const auto outcome = request( + "/positions/otc", "POST", encodeTradeOpen(order), {}, + RequestOptions{.dealKey = context.strategyUuid + + context.openDirection, + .riskReducing = false, + .transportRetries = 0}); + if (outcome.fate == RequestFate::Refused) { + return OpenResult{.status = OpenStatus::Failed, + .reason = "refused: " + outcome.detail}; + } + + // Transport failure, 5xx and 408 all leave the same ambiguity: IG + // may or may not hold the order. Ask /confirms instead of guessing. + const bool ambiguous = + outcome.fate == RequestFate::TransportFailed + || outcome.response->status >= 500 + || outcome.response->status == 408; + if (ambiguous) { + const std::string how = + outcome.fate == RequestFate::TransportFailed + ? outcome.detail + : "HTTP " + std::to_string(outcome.response->status); + if (order.dealReference.empty()) { + // Nothing to poll under — refuse to guess. + return OpenResult{.status = OpenStatus::Failed, + .reason = "ambiguous open (" + how + + ") with no dealReference to " + "confirm under — not re-sent"}; + } + backtest_log::logLine( + "IGMarketCalls: open POST ambiguous ({}) — resolving via " + "confirms for {}", + how, order.dealReference); + if (const std::optional resolved = pollConfirm( + request, order.dealReference, order.dealReference, + confirmAttempts, confirmDelay)) { + return *resolved; + } + // No confirm after an ambiguous POST: treat as not placed and do + // NOT re-send. The order-channel failure TTL brakes re-entry, + // and if the order DID land the producer's book sync seeds it + // within a refresh. + return OpenResult{.status = OpenStatus::Failed, + .reason = "unconfirmed after ambiguous open (" + + how + ") — not re-sent"}; + } + + const ig_rest::HttpResponse& response = *outcome.response; + if (response.status != 200) { + return OpenResult{.status = OpenStatus::Failed, + .reason = "HTTP " + + std::to_string(response.status) + + ": " + response.body}; + } + const auto parsed = parsePositionResponse(response.body); + if (!parsed) { + return OpenResult{.status = OpenStatus::Failed, + .reason = "failed to parse response: " + + response.body}; + } + if (parsed->errorCode && !parsed->errorCode->empty()) { + return OpenResult{.status = OpenStatus::Failed, + .reason = "errorCode: " + *parsed->errorCode}; + } + + // Confirm poll. IG's echoed reference names the deal; fall back to + // the reference WE sent (same fallback the channel books under). + // The Accepted result keeps reporting the raw echo, empty or not — + // the channel compensates (existing behaviour, unchanged here). + const std::string reference = parsed->dealReference.empty() + ? order.dealReference + : parsed->dealReference; + if (const std::optional resolved = + pollConfirm(request, reference, parsed->dealReference, + confirmAttempts, confirmDelay)) { + return *resolved; + } + backtest_log::logLine( + "IGMarketCalls: confirm for {} never resolved after {} attempts " + "— booking with an empty dealId (strategy closes blocked until " + "an id appears)", + reference, confirmAttempts); + return OpenResult{.status = OpenStatus::Accepted, + .dealReference = parsed->dealReference}; + }; +} + +PlaceClose IGMarketCalls::makeClose(RequestFn request) { + // The context is deliberately unused: the close is keyed on the dealId, + // never the open's uuid+direction — sharing that key let a fresh open's + // 30s marker refuse the close that followed it. + return [request = std::move(request)]( + const TradeCloseObj& close, + const OrderContext&) -> CloseResult { + // Risk-reducing: gate uncertainty (Redis down) and the soft budget + // never refuse a close; only its own close# marker paces + // repeats. Transport retries stay on — re-sending a close of the + // same dealId is safe (IG rejects the second), unlike the open. + const auto outcome = request( + "/positions/otc", "POST", encodeTradeClose(close), + {{"_method", "DELETE"}}, + RequestOptions{.dealKey = "close#" + close.dealId, + .riskReducing = true}); + if (outcome.fate == RequestFate::Refused) { + // Never sent — the position is exactly where it was. Failed + // keeps the book entry so the sync loop retries the close; + // the old mapping (Gone) deleted the very Redis entry that + // retry depends on. + return CloseResult{.status = CloseStatus::Failed, + .reason = "refused: " + outcome.detail}; + } + if (outcome.fate == RequestFate::TransportFailed) { + // The C# null-response branch, now reserved for a REAL lost + // exchange: as far as anyone can tell the position is missing + // from IG — the caller deletes the book entry so a phantom + // cannot haunt the strategy logic; the producer restores it + // from the broker book if it does still exist. + return CloseResult{.status = CloseStatus::Gone, + .reason = "no response after send (" + + outcome.detail + ")"}; + } + const ig_rest::HttpResponse& response = *outcome.response; + if (response.status != 200) { + return CloseResult{.status = CloseStatus::Failed, + .reason = "HTTP " + + std::to_string(response.status) + + ": " + response.body}; + } + if (!parsePositionResponse(response.body)) { + return CloseResult{.status = CloseStatus::Failed, + .reason = "failed to parse response: " + + response.body}; + } + return CloseResult{.status = CloseStatus::Ok}; + }; +} + +PlaceOrder IGMarketCalls::makeLiveOpen(std::string redisHost, + const int redisPort, + AuthProvider auth) { + PlaceOrder core = makeOpen(threadLocalRequestFn(std::move(redisHost), + redisPort, + std::move(auth))); + return [core = std::move(core)](const TradeOpenObj& order, + const OrderContext& context) { + const OpenResult result = core(order, context); + if (result.status == OpenStatus::Accepted) { + auditTrade("Open Trade", context, result.dealReference, + result.dealId); + } + return result; + }; +} + +PlaceClose IGMarketCalls::makeLiveClose(std::string redisHost, + const int redisPort, + AuthProvider auth) { + PlaceClose core = makeClose(threadLocalRequestFn(std::move(redisHost), + redisPort, + std::move(auth))); + return [core = std::move(core)](const TradeCloseObj& close, + const OrderContext& context) { + const CloseResult result = core(close, context); + if (result.status == CloseStatus::Ok) { + auditTrade("Close Trade", context, close.dealId); + } + return result; + }; +} + +} // namespace ig diff --git a/source/live/broker/marketDefinitions.cppm b/source/live/broker/marketDefinitions.cppm new file mode 100644 index 0000000..b00bc37 --- /dev/null +++ b/source/live/broker/marketDefinitions.cppm @@ -0,0 +1,189 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// marketDefinitions — engine symbol -> broker market mapping, the C# +// engine's MarketDescriptions ported verbatim from its PRODUCTION config +// (every epic below is what the C# engine trades today — including the +// quirks: XAUUSD is denominated in GBP on this account, indices and +// commodities mostly settle in GBP, and GBRIDXGBP's mini epic differs from +// its CFD epic). +// +// Two fields drive live trading, everything else is model fidelity: +// epicMini (IGMarketIdentiferMini) — the market actually traded; +// the order channel always deals MINI contracts +// tradeSizeModifier (TradeSizeModifier) — multiplies the strategy's +// TRADING_SIZE; 0 encodes the C# null = no scaling +// +// The C# model's `strategies` list is deliberately absent: strategy +// selection here comes from the winners system (liveWinners), not from +// per-market config. A symbol MISSING from this table means "do not trade +// it live" (the order channel logs and drops the signal), so the table +// doubles as the live trading allowlist. +// +// Same table discipline as symbolScale: constexpr, sorted, binary-searched, +// compile-time-checked — and the two tables cover the SAME 29 symbols +// (pinned by test), so anything priced can be traded and vice versa. + +export module marketDefinitions; + +import std; // replaces , , + +export namespace live { + +// The C# MarketType values in use. +enum class MarketType { Forex, Indice }; + +// The C# MarketDescriptions (field spellings modernised; the originals are +// noted where they differ). +struct MarketDefinition { + std::string_view symbol; + std::string_view igMarketId; // IGMarketID (== symbol throughout) + std::string_view epicCfd; // IGMarketIdentifer — full CFD contract + std::string_view epicMini; // IGMarketIdentiferMini — WHAT WE TRADE + std::string_view currency; + std::string_view polygonIdentifier; // PolygonIdentifer; "" = none + double polygonScale; // PolygonScale; 0 = none + MarketType type; + double tradeSizeModifier; // TradeSizeModifier; 0 = C# null + + // The C# call-site contract: `if (TradeSizeModifier is not null) + // reqObj.size *= modifier` — absent means unscaled. + [[nodiscard]] constexpr double sizeModifier() const noexcept { + return tradeSizeModifier > 0.0 ? tradeSizeModifier : 1.0; + } +}; + +// MUST stay sorted ascending by symbol — enforced below, binary search +// depends on it. Columns: {symbol, IGMarketID, CFD epic, MINI epic, +// currency, polygon id, polygon scale, type, size modifier}. +inline constexpr std::array kMarkets{{ + {"AUDNZD", "AUDNZD", "CS.D.AUDNZD.CFD.IP", "CS.D.AUDNZD.MINI.IP", "NZD", + "C.AUD/NZD", 0, MarketType::Forex, 0}, + {"AUDUSD", "AUDUSD", "CS.D.AUDUSD.CFD.IP", "CS.D.AUDUSD.MINI.IP", "USD", + "C.AUD/USD", 0, MarketType::Forex, 0}, + {"AUSIDXAUD", "AUSIDXAUD", "IX.D.ASX.IFS.IP", "IX.D.ASX.IFS.IP", "GBP", + "", 0, MarketType::Indice, 0}, + {"BRENTCMDUSD", "BRENTCMDUSD", "CC.D.LCO.UMP.IP", "CC.D.LCO.UMP.IP", + "GBP", "", 0, MarketType::Indice, 0}, + {"COPPERCMDUSD", "COPPERCMDUSD", "CC.D.HG.UMP.IP", "CC.D.HG.UMP.IP", + "GBP", "", 0, MarketType::Indice, 0}, + {"DEUIDXEUR", "DEUIDXEUR", "IX.D.DAX.IFS.IP", "IX.D.DAX.IFS.IP", "GBP", + "", 0, MarketType::Indice, 0}, + {"EURAUD", "EURAUD", "CS.D.EURAUD.CFD.IP", "CS.D.EURAUD.MINI.IP", "AUD", + "C.EUR/AUD", 0, MarketType::Forex, 0}, + {"EURCHF", "EURCHF", "CS.D.EURCHF.CFD.IP", "CS.D.EURCHF.MINI.IP", "CHF", + "C.EUR/CHF", 0, MarketType::Forex, 0}, + {"EURGBP", "EURGBP", "CS.D.EURGBP.CFD.IP", "CS.D.EURGBP.MINI.IP", "GBP", + "C.EUR/GBP", 0, MarketType::Forex, 0}, + {"EURJPY", "EURJPY", "CS.D.EURJPY.CFD.IP", "CS.D.EURJPY.MINI.IP", "JPY", + "C.EUR/JPY", 0, MarketType::Forex, 0}, + {"EURNOK", "EURNOK", "CS.D.EURNOK.CFD.IP", "CS.D.EURNOK.MINI.IP", "NOK", + "C.EUR/NOK", 0, MarketType::Forex, 0}, + {"EURUSD", "EURUSD", "CS.D.EURUSD.CFD.IP", "CS.D.EURUSD.MINI.IP", "USD", + "C.EUR/USD", 0, MarketType::Forex, 0}, + {"FRAIDXEUR", "FRAIDXEUR", "IX.D.CAC.IFS.IP", "IX.D.CAC.IFS.IP", "GBP", + "", 0, MarketType::Indice, 0}, + {"GBPJPY", "GBPJPY", "CS.D.GBPJPY.CFD.IP", "CS.D.GBPJPY.MINI.IP", "JPY", + "C.GBP/JPY", 0, MarketType::Forex, 0}, + {"GBPUSD", "GBPUSD", "CS.D.GBPUSD.CFD.IP", "CS.D.GBPUSD.MINI.IP", "USD", + "C.GBP/USD", 0, MarketType::Forex, 0}, + // The one market whose mini contract is a different epic family from + // its CFD — trading the CFD epic by mistake would 2x the exposure the + // 0.5 modifier is there to halve. + {"GBRIDXGBP", "GBRIDXGBP", "IX.D.FTSE.CFD.IP", "IX.D.FTSE.IFM.IP", "GBP", + "", 0, MarketType::Indice, 0.5}, + {"HKGIDXHKD", "HKGIDXHKD", "IX.D.HANGSENG.IFU.IP", "IX.D.HANGSENG.IFU.IP", + "USD", "", 0, MarketType::Indice, 0}, + {"JPNIDXJPY", "JPNIDXJPY", "IX.D.NIKKEI.IFM.IP", "IX.D.NIKKEI.IFM.IP", + "USD", "", 0, MarketType::Indice, 0}, + {"LIGHTCMDUSD", "LIGHTCMDUSD", "CC.D.CL.UMP.IP", "CC.D.CL.UMP.IP", "GBP", + "", 0, MarketType::Indice, 0}, + {"NZDUSD", "NZDUSD", "CS.D.NZDUSD.CFD.IP", "CS.D.NZDUSD.MINI.IP", "USD", + "C.NZD/USD", 0, MarketType::Forex, 0}, + {"USA30IDXUSD", "USA30IDXUSD", "IX.D.DOW.IFS.IP", "IX.D.DOW.IFS.IP", + "GBP", "", 0, MarketType::Indice, 0}, + {"USA500IDXUSD", "USA500IDXUSD", "IX.D.SPTRD.IFS.IP", "IX.D.SPTRD.IFS.IP", + "GBP", "", 0, MarketType::Indice, 0}, + {"USATECHIDXUSD", "USATECHIDXUSD", "IX.D.NASDAQ.IFS.IP", + "IX.D.NASDAQ.IFS.IP", "GBP", "", 0, MarketType::Indice, 0}, + {"USDCAD", "USDCAD", "CS.D.USDCAD.CFD.IP", "CS.D.USDCAD.MINI.IP", "CAD", + "C.USD/CAD", 0, MarketType::Forex, 0}, + {"USDCHF", "USDCHF", "CS.D.USDCHF.CFD.IP", "CS.D.USDCHF.MINI.IP", "CHF", + "C.USD/CHF", 0, MarketType::Forex, 0}, + {"USDJPY", "USDJPY", "CS.D.USDJPY.CFD.IP", "CS.D.USDJPY.MINI.IP", "JPY", + "C.USD/JPY", 0, MarketType::Forex, 0}, + {"USDSEK", "USDSEK", "CS.D.USDSEK.CFD.IP", "CS.D.USDSEK.MINI.IP", "SEK", + "C.USD/SEK", 0, MarketType::Forex, 0}, + {"XAGUSD", "XAGUSD", "CS.D.CFDSILVER.CFM.IP", "CS.D.CFDSILVER.CFM.IP", + "USD", "C.XAG/USD", 100, MarketType::Forex, 0.2}, + // XAUUSD modifier raised from the C# 0.5: IG rejects size 0.5 on this + // epic as below the market minimum (every 2026-07-10 XAUUSD open was + // REJECTED), so TRADING_SIZE 1 must reach the broker as 1.0. + {"XAUUSD", "XAUUSD", "CS.D.CFPGOLD.CFP.IP", "CS.D.CFPGOLD.CFP.IP", "GBP", + "C.XAU/USD", 0, MarketType::Forex, 1.0}, +}}; + +static_assert([] { + for (std::size_t i = 1; i < kMarkets.size(); ++i) { + if (!(kMarkets[i - 1].symbol < kMarkets[i].symbol)) return false; + } + return true; +}(), "live::kMarkets must be sorted ascending by symbol — binary search depends on it"); + +// The symbols live trading may book, in table (= sorted) order. The winners +// fetch fans out one query per (strategy, symbol) over exactly this list — a +// winner on any other symbol could only book a worker whose orders the +// channel drops, so the allowlist doubles as the fetch universe. +inline constexpr auto kTradableSymbols = [] { + std::array symbols{}; + for (std::size_t i = 0; i < kMarkets.size(); ++i) { + symbols[i] = kMarkets[i].symbol; + } + return symbols; +}(); + +// The matching definition, or nullptr when the symbol is not tradable live. +// Cold path (per order, not per tick) but the sorted table makes it cheap +// anyway. +[[nodiscard]] constexpr const MarketDefinition* findMarket( + std::string_view symbol) noexcept { + std::size_t lo = 0; + std::size_t hi = kMarkets.size(); + while (lo < hi) { + const std::size_t mid = lo + ((hi - lo) >> 1); + const auto& entry = kMarkets[mid]; + if (entry.symbol < symbol) { + lo = mid + 1; + } else if (symbol < entry.symbol) { + hi = mid; + } else { + return &entry; + } + } + return nullptr; +} + +// Reverse lookup for the position producer: the market whose MINI epic (the +// contract the order channel trades, and what IG's /positions book reports +// back) matches. The table is sorted by symbol, not epic, so this is a linear +// scan — 29 entries, once a minute. nullptr = an epic the engine does not +// trade (the producer skips it, like the C# SavePositions match). +[[nodiscard]] constexpr const MarketDefinition* findMarketByEpicMini( + std::string_view epic) noexcept { + for (const auto& entry : kMarkets) { + if (entry.epicMini == epic) { + return &entry; + } + } + return nullptr; +} + +// Lookup seam for the order channel: injectable so unit tests can supply +// their own definitions (and a missing symbol) without depending on what the +// real table currently lists — test the machinery, not the table. +using MarketLookup = std::function; + +} // namespace live diff --git a/source/live/config/liveSettings.cppm b/source/live/config/liveSettings.cppm new file mode 100644 index 0000000..9426f48 --- /dev/null +++ b/source/live/config/liveSettings.cppm @@ -0,0 +1,131 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// liveSettings — the live subcommand's runtime configuration, resolved once +// at startup from the environment and argv. Every knob parses forgivingly +// (garbage falls back to the default) so a stray env var can't crash startup. + +module; + +#include "shared/net/udpPorts.hpp" +#include "shared/utilities/env.hpp" + +export module liveSettings; + +import std; + +export namespace live { + +// Defaults in parens; see Settings::fromEnv for the exact env names. +struct Settings { + std::string bindAddr; // LIVE_BIND_ADDR (127.0.0.1) + std::uint16_t bindPort{}; // argv[2] > LIVE_UDP_PORT > kLive (11110) + double minScore{}; // LIVE_MIN_SCORE floor on + // results.performanceScore (20) + double maxDrawdownPercent{}; // LIVE_MAX_DRAWDOWN_PERCENT ceiling on + // results.maxDrawdownPercent (10) — a + // hard eligibility gate: the blended + // Calmar half of performanceScore lets + // a high-expectancy spiky run buy its + // way past the drawdown penalty + double minCalmarScore{}; // LIVE_MIN_CALMAR_SCORE floor on + // results.calmarScore (30) — Calmar + // ratio ~2 on resultsSummary's scale + // (ratio 3 == 50): growth must be ~2x + // the worst giveback. Complements the + // ceiling, does not replace it — a + // fast-growing run can hold Calmar 2 + // with a deep absolute drawdown, which + // the ceiling still refuses + std::chrono::seconds lockTtl{}; // LIVE_TRADE_LOCK_SECONDS (30) + std::string redisHost; // REDIS_HOST (127.0.0.1) + int redisPort{}; // fixed 6379, matching loadCommand + std::size_t topPerGroup{}; // winners kept per (symbol, strategy): 3 + // TRADING_ENVIRONMENT ("live" | "demo", lowercased; default demo — the + // safe account). Selects the IG session item the login service keeps in + // DynamoDB: Auth#. + std::string tradingEnv; + // OHLC_PREPOPULATE (default on; "0" disables — same gate string, same + // default, that ohlcBuilder itself re-reads at each symbol's first + // tick). Surfaced here only so startup can log the bar warm-up mode — + // the builder owns the behaviour, this field must never gate anything. + bool ohlcPrepopulate{}; + + static Settings fromEnv(int argc, const char* argv[]); +}; + +} // namespace live + +namespace { + +// Parse a port from a string, returning `fallback` on empty/garbage input. +std::uint16_t parsePort(std::string_view text, const std::uint16_t fallback) { + unsigned value = 0; + const auto [ptr, ec] = + std::from_chars(text.data(), text.data() + text.size(), value); + if (ec != std::errc{} || value == 0 || value > 65535) { + return fallback; + } + return static_cast(value); +} + +// stod (not from_chars) matches how this codebase parses doubles from strings +// (see readIntField in tradingVariables.hpp). +double parseScore(const std::string& text, const double fallback) { + try { + const double value = std::stod(text); + return std::isfinite(value) ? value : fallback; + } catch (const std::exception&) { + return fallback; + } +} + +// Positive whole seconds or the fallback. +std::chrono::seconds parseTtlSeconds(std::string_view text, const long fallback) { + long value = 0; + const auto [ptr, ec] = + std::from_chars(text.data(), text.data() + text.size(), value); + if (ec != std::errc{} || value <= 0) { + return std::chrono::seconds{fallback}; + } + return std::chrono::seconds{value}; +} + +} // namespace + +namespace live { + +Settings Settings::fromEnv(const int argc, const char* argv[]) { + Settings settings; + settings.bindAddr = env::getOr("LIVE_BIND_ADDR", "127.0.0.1"); + + // Bind port: CLI arg (argv[2]) overrides $LIVE_UDP_PORT overrides kLive. + settings.bindPort = + parsePort(env::getOr("LIVE_UDP_PORT", ""), udp_ports::kLive); + if (argc >= 3) { + settings.bindPort = parsePort(argv[2], settings.bindPort); + } + + settings.minScore = parseScore(env::getOr("LIVE_MIN_SCORE", ""), 20.0); + settings.maxDrawdownPercent = + parseScore(env::getOr("LIVE_MAX_DRAWDOWN_PERCENT", ""), 10.0); + settings.minCalmarScore = + parseScore(env::getOr("LIVE_MIN_CALMAR_SCORE", ""), 30.0); + settings.lockTtl = + parseTtlSeconds(env::getOr("LIVE_TRADE_LOCK_SECONDS", ""), 30); + settings.redisHost = env::getOr("REDIS_HOST", "127.0.0.1"); + settings.redisPort = 6379; + settings.topPerGroup = 3; + settings.ohlcPrepopulate = env::getOr("OHLC_PREPOPULATE", "1") == "1"; + settings.tradingEnv = env::getOr("TRADING_ENVIRONMENT", "demo"); + std::ranges::transform(settings.tradingEnv, settings.tradingEnv.begin(), + [](const unsigned char c) { + return static_cast(std::tolower(c)); + }); + return settings; +} + +} // namespace live diff --git a/source/live/execution/brokerOrderSink.cppm b/source/live/execution/brokerOrderSink.cppm new file mode 100644 index 0000000..eba2b49 --- /dev/null +++ b/source/live/execution/brokerOrderSink.cppm @@ -0,0 +1,246 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// brokerOrderSink — the live order handoff: adapts the strategy runner's +// OrderSink AND CloseSink seams onto one OrderChannel per worker thread +// (decision -> RequestObject -> IG, and strategy close -> close request), +// binding the real Redis-backed lock/position bookkeeping around the +// injected broker calls (ig::IGMarketCalls::makeLiveOpen / makeLiveClose in +// liveCommand). +// +// Same per-worker-thread pattern as RedisTradeGate / RedisPositionCounter: +// each worker lazily builds its own TradeLocks + PositionManager + channel +// on first use, so no worker's placement serialises behind another's Redis +// round trip. Both sinks route through threadChannel() below — one shared +// function-scope thread_local set — so the open and close paths on a worker +// share ONE channel (one recently-closed window, one set of Redis +// managers). The GMF only #includes Asio-free headers. + +module; + +#include "shared/redis/positionClustering.hpp" +#include "shared/redis/positionManager.hpp" +#include "shared/redis/tradeLocks.hpp" + +export module brokerOrderSink; + +import std; // replaces , , +import igMarkets; // ig::PlaceOrder, PlaceClose +import backtestLog; // backtest_log::logLine +import liveStrategyRunner; // live::OrderSink, CloseSink, OrderIntent, CloseIntent +import liveTrace; // live_trace::emit — live-traces documents +import marketDefinitions; // live::findMarket +import orderChannel; // live::OrderChannel, CloseRequest +import orderRequest; // live::makeOrderRequest +import redisPositionCounter; // live::RedisPositionCounter::invalidateCache +import trade; // Direction + +export namespace live { + +// The two halves of the broker handoff, built together so they share +// per-thread state. +struct BrokerSinks { + OrderSink order; + CloseSink close; +}; + +class BrokerOrderSink { +public: + // lockTtl should match the TTL the gate acquires with (liveSettings + // lockTtl) so the in-flight extension restarts the same window. + static BrokerSinks make(const std::string& redisHost, int redisPort, + std::chrono::seconds lockTtl, + ig::PlaceOrder placeOrder, + ig::PlaceClose placeClose); +}; + +} // namespace live + +namespace live { + +namespace { + +// The per-worker-thread channel both sinks share. Thread-local quartet, +// built on this worker's first use and destroyed at thread exit (when the +// runner joins its workers). The hooks capture raw pointers to the +// thread_local managers: all four live and die with this same thread, and +// the managers are constructed first / destroyed last. Function-scope thread_locals are +// one instance per thread across ALL callers of this function — exactly the +// sharing the open+close pair needs (the process only ever wires one broker +// configuration, same caveat as RedisTradeGate's per-thread locks). +OrderChannel& threadChannel(const std::string& redisHost, const int redisPort, + const std::chrono::seconds lockTtl, + const ig::PlaceOrder& placeOrder, + const ig::PlaceClose& placeClose) { + thread_local std::unique_ptr locks; + thread_local std::unique_ptr positions; + thread_local std::unique_ptr clusters; + thread_local std::unique_ptr channel; + if (!channel) { + locks = std::make_unique(redisHost, + redisPort); + positions = std::make_unique( + redisHost, redisPort); + clusters = std::make_unique( + redisHost, redisPort); + channel = std::make_unique( + MarketLookup{findMarket}, placeOrder, placeClose, + OrderChannel::Hooks{ + .clusterBlocked = + [c = clusters.get()](const std::string& symbol, + const std::string& strategyName) { + return c->isClusterBlocked(symbol, strategyName); + }, + .clusterOpened = + [c = clusters.get()](const std::string& symbol) { + c->markOpened(symbol); + }, + .extendLock = + [l = locks.get()](const std::string& uuid, + const std::string& direction, + const std::chrono::seconds ttl) { + return l->extendLock(uuid, direction, ttl); + }, + .releaseLock = + [l = locks.get()](const std::string& uuid, + const std::string& direction) { + return l->releaseLock(uuid, direction); + }, + .savePosition = + [p = positions.get()](const std::string& reference, + const std::string& payload) { + return p->savePosition(reference, payload); + }, + .addPosition = + [p = positions.get()](const std::string& uuid, + const std::string& reference) { + return p->addPosition(uuid, reference); + }, + .saveDealReceipt = + [p = positions.get()](const std::string& reference, + const std::string& symbol, + const std::string& payload) { + return p->saveDealReceipt(reference, symbol, + payload); + }, + .removePosition = + [p = positions.get()](const std::string& uuid, + const std::string& reference) { + return p->removePosition(uuid, reference); + }, + }, + lockTtl); + } + return *channel; +} + +} // namespace + +BrokerSinks BrokerOrderSink::make(const std::string& redisHost, + const int redisPort, + const std::chrono::seconds lockTtl, + ig::PlaceOrder placeOrder, + ig::PlaceClose placeClose) { + BrokerSinks sinks; + sinks.order = [redisHost, redisPort, lockTtl, placeOrder, + placeClose](const OrderIntent& intent) { + const std::optional request = makeOrderRequest(intent); + if (!request) { + // A winner trading a symbol symbolScale doesn't know shouldn't + // exist (the backtest scaled its prices with the same table) — + // fail loud, drop the order. + backtest_log::logLine( + "BrokerOrderSink: {} - no symbolScale entry; dropping {} " + "order from {}", + intent.symbol, + intent.direction == Direction::LONG ? "LONG" : "SHORT", + intent.strategyName); + if (live_trace::enabled()) { + live_trace::emit( + "orderDropped", + {.strategyUuid = intent.strategyUuid, + .strategyName = intent.strategyName, + .symbol = intent.symbol}, + {{"reason", "noSymbolScale"}, + {"direction", intent.direction == Direction::LONG + ? "LONG" + : "SHORT"}}); + } + return; + } + if (live_trace::enabled()) { + // The intent, traced AFTER the request build so it carries the + // freshly minted dealReference every later event correlates on. + live_trace::emit( + "orderIntent", + {.strategyUuid = request->strategyUuid, + .strategyName = request->strategyName, + .symbol = request->symbol, + .dealReference = request->dealReference}, + {{"direction", request->direction == Direction::LONG + ? "LONG" + : "SHORT"}, + {"size", static_cast(request->size)}, + {"stopDistancePips", + static_cast(request->stopDistancePips)}, + {"limitDistancePips", + static_cast(request->limitDistancePips)}, + {"bid", static_cast(request->bid)}, + {"ask", static_cast(request->ask)}, + {"level", static_cast(request->level)}, + {"stopLevel", static_cast(request->stopLevel)}, + {"limitLevel", + static_cast(request->limitLevel)}}); + } + const bool accepted = + threadChannel(redisHost, redisPort, lockTtl, placeOrder, + placeClose) + .request(*request); + if (accepted) { + // The channel's addPosition just moved PL# on this very thread; + // drop the cached count so the next MAX_OPEN_TRADES check reads + // the true value instead of the pre-open one (a stale read here + // let a second direction through the cap inside the cache TTL). + RedisPositionCounter::invalidateCache(request->strategyUuid); + } + }; + sinks.close = [redisHost, redisPort, lockTtl, placeOrder, + placeClose](const CloseIntent& intent) { + if (live_trace::enabled()) { + live_trace::emit( + "closeIntent", + {.strategyUuid = intent.strategyUuid, + .strategyName = intent.strategyName, + .symbol = intent.symbol, + .dealReference = intent.dealReference, + .dealId = intent.dealId}, + {{"direction", intent.direction == Direction::LONG + ? "LONG" + : "SHORT"}, + {"brokerSize", intent.brokerSize}}); + } + const bool closed = + threadChannel(redisHost, redisPort, lockTtl, placeOrder, + placeClose) + .closePosition(CloseRequest{ + .strategyUuid = intent.strategyUuid, + .strategyName = intent.strategyName, + .symbol = intent.symbol, + .direction = intent.direction, + .size = intent.brokerSize, + .dealId = intent.dealId, + .dealReference = intent.dealReference, + }); + if (closed) { + // Mirror of the open path: removePosition just shrank PL#, so a + // freed cap slot is visible immediately instead of after the TTL. + RedisPositionCounter::invalidateCache(intent.strategyUuid); + } + }; + return sinks; +} + +} // namespace live diff --git a/source/live/execution/liveStrategyRunner.cppm b/source/live/execution/liveStrategyRunner.cppm new file mode 100644 index 0000000..31e0637 --- /dev/null +++ b/source/live/execution/liveStrategyRunner.cppm @@ -0,0 +1,790 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// liveStrategyRunner — executes the cached winning strategies against the +// live tick stream. +// +// Threading model: one dedicated worker thread per live strategy instance, +// each with its own tick queue. The UdpReceiver thread only routes (enqueue + +// notify), so a slow strategy can never stall the socket; and because a +// strategy instance is only ever touched by its own worker, the stateful +// strategies (OhlcBreakout's per-symbol bars, Random's RNG) need no locking +// and see ticks in arrival order. The shared ThreadPool is deliberately NOT +// used here — pooled submits could run the same strategy instance +// concurrently for back-to-back ticks. +// +// Per tick a worker runs only the live-relevant slice of trading::runTicks +// (runLoop.cppm): book sync (broker positions -> TradeManager, throttled to +// bookSyncInterval) -> decide -> caps (MAX_TRADES_PER_MINUTE sliding window, +// MAX_OPEN_TRADES against the broker position count) -> gate -> emit order, +// then during, then the close diff (a strategy that closed a book trade in +// during() emits a CloseIntent). No markToMarket, no reviewStopAndLimit, no +// closeAllTrades — the broker owns exits; the book mirrors the broker. + +module; + +#include "shared/tradingDefinitions/variables/tradingVariables.hpp" +#include "shared/utilities/backtestLog.hpp" + +export module liveStrategyRunner; + +import std; // replaces , , , + // , , , , + // +import barStore; // bars::BarStore — the worker's shared bar pipeline +import entryConditions; // conditions::check — pre-decide ATR gate +import rangeBarBuilder; // rangebar::RangeBarSpec — range-bar registrations +import liveTrace; // live_trace::emit — live-traces documents +import marketHours; // market_hours::tradePermitted +import priceData; // PriceData +import strategy; // IStrategy +import trade; // Direction +import tradeManager; // TradeManager + +export namespace live { + +// One order signal, exactly what phase 1 logs and what a phase-2 broker +// channel will consume. Prices are the scaled fixed-point points the decoder +// produced (see tickPacket / symbolScale). +struct OrderIntent { + std::string strategyName; + std::string strategyUuid; + std::string symbol; + Direction direction; + std::int32_t size; + std::int32_t stopDistancePips; + std::int32_t limitDistancePips; + std::int32_t bid; + std::int32_t ask; + std::chrono::system_clock::time_point timestamp; // tick timestamp +}; + +// Entry gate: true = trading may proceed. Injected as a functor so unit tests +// need no Redis and phase 2 can layer more gates behind one seam. Called on a +// worker thread, only when a strategy signals. +using TradeGate = std::function; +using OrderSink = std::function; + +// Broker position count for one strategy UUID (the PL# list a separate +// producer refreshes from the broker every ~2 minutes — see +// redis_positions::PositionManager). nullopt = UNKNOWN (Redis unreachable): +// the MAX_OPEN_TRADES check then fails closed, same doctrine as the trade +// lock. Called on a worker thread, only when a strategy signals AND its spec +// caps open trades. +using PositionCounter = + std::function(const std::string& strategyUuid)>; + +// One broker deal for one worker's symbol, decoded from the PL#/PO# book +// (see redisPositionFeed). The side-map entry the close path addresses. +struct BookedPosition { + std::string dealId; // EMPTY until the deal confirmed — a close + // cannot be sent without it + std::string dealReference; // the PO#/PL# book key + Direction direction{Direction::LONG}; + double brokerSize{}; // exact broker units — what a close must send + std::int32_t engineSize{}; // brokerSize / sizeModifier, for the book Trade + std::int32_t level{}; // booked entry price, scaled INT32 points + std::chrono::system_clock::time_point openedAt; +}; + +// Broker positions for (strategy UUID, symbol). nullopt = UNKNOWN (Redis +// unreachable): the book keeps its last known state — FAIL-OPEN, unlike the +// entry gates, because a stale book can at worst cause a redundant close +// (absorbed by the channel's blank-dealId guard, its recently-closed window +// and the broker itself) while entries stay fail-closed via the caps/locks. +// Called on a worker thread, at most once per bookSyncInterval. +using PositionFeed = std::function>( + const std::string& strategyUuid, const std::string& symbol)>; + +// Mirror of OrderIntent for strategy-initiated closes: emitted when a +// strategy's during() closed a book trade, carrying the broker identity the +// order channel needs. +struct CloseIntent { + std::string strategyName; + std::string strategyUuid; + std::string symbol; + Direction direction{Direction::LONG}; // the OPEN position's direction + double brokerSize{}; + std::string dealId; + std::string dealReference; + std::chrono::system_clock::time_point timestamp; // closing tick +}; +using CloseSink = std::function; + +// One live strategy instance bound to ONE symbol (multi-symbol backtest +// configs were split per symbol during winner selection). Instances sharing a +// UUID also share their lock keys per direction — matching the C# TradeLocks +// semantics. +struct WorkerSpec { + std::string symbol; + std::string strategyName; + std::string strategyUuid; + tradingDefinitions::TradingVariables vars; // size / stop / limit for orders + // Risk caps from the winning run's config; defaults mirror + // tradingDefinitions::RunConfiguration (<= 0 disables a cap, and the + // per-minute brake is ON by default there too). Both are per strategy + // UUID: a UUID trading several symbols shares one budget, exactly like + // MAX_OPEN_TRADES / MAX_TRADES_PER_MINUTE cap a whole backtest run. + int maxOpenTrades{0}; + int maxTradesPerMinute{60}; + // Peak-market-hours entry filter (market_hours::tradePermitted), from + // the winning run's config like the caps above: out-of-session ticks + // skip decide() entirely. Book sync and during()/close-diff are never + // gated — the broker owns exits. + bool peakHoursOnly{false}; + // Bar series to register on the worker's BarStore: the strategy's OHLC + // timeframes (StrategyCache skips the {0,0} "no bars" sentinels) so a + // bar-reading strategy's decide() has history to work from. + std::vector barSeries; + // Range-bar series, same contract as barSeries (StrategyCache skips the + // all-zeros sentinels). Default-empty: pre-range winners and test specs + // register nothing and behave exactly as before. + std::vector rangeSeries; + // The ATR entry conditions' series (conditions::gateSeriesFor). Engaged + // in production by StrategyCache; tests that script caps/gate/sink in + // isolation leave it disengaged, and the vars then flow into OrderIntent + // as literal pip distances. + std::optional gateSeries; + std::unique_ptr strategy; +}; + +// Counters are cumulative since start(); routed counts per-worker deliveries +// (one tick fanned out to two workers counts twice). +struct RunnerStats { + std::uint64_t routed{}; + std::uint64_t ignoredSymbol{}; // decoded ticks with no worker for their symbol + std::uint64_t queueDropped{}; // oldest-tick drops on queue overflow + std::uint64_t sessionSkipped{}; // peakHoursOnly: out-of-session ticks + // (decide() skipped) + std::uint64_t conditionsSkipped{}; // ATR entry conditions failed: gate + // series not warm, spread too wide vs + // ATR, or volatility below the floors + // (decide() skipped) + std::uint64_t signals{}; // decide() returned a direction + std::uint64_t rateBlocked{}; // MAX_TRADES_PER_MINUTE window was full + std::uint64_t positionBlocked{}; // at MAX_OPEN_TRADES, or count unknown + std::uint64_t lockBlocked{}; // gate refused (lock held, or fail-closed) + std::uint64_t ordersLogged{}; // OrderIntents handed to the sink + std::uint64_t bookSeeded{}; // broker deals seeded into a worker's book + std::uint64_t bookRemoved{}; // book trades dropped (closed at broker) + std::uint64_t bookSyncFailed{}; // feed returned UNKNOWN (book kept as-is) + std::uint64_t strategyCloses{}; // CloseIntents handed to the close sink + std::uint64_t closeDropped{}; // strategy closes without a broker dealId +}; + +class StrategyRunner { +public: + // `positions` may be empty when no spec caps open trades (unit tests, a + // deployment without the position producer); a spec that DOES cap them + // then fails closed — every entry blocks, warned once at start(). + // `feed` + `closeSink` come as a pair (warned at start() if only one is + // wired): the feed mirrors broker deals into each worker's book, the + // close sink carries strategy-initiated closes back to the broker. + // bookSyncInterval throttles the feed per worker (checked per tick). + StrategyRunner(std::vector specs, TradeGate gate, OrderSink sink, + PositionCounter positions = {}, PositionFeed feed = {}, + CloseSink closeSink = {}, + std::chrono::seconds bookSyncInterval = + std::chrono::seconds{15}); + ~StrategyRunner(); // stops and joins if still running + + StrategyRunner(const StrategyRunner&) = delete; + StrategyRunner& operator=(const StrategyRunner&) = delete; + + // Spawns one plain std::thread per spec. Plain thread + condition_variable + // on purpose — std::jthread's stop_token wait (condition_variable_any) + // doesn't link under `import std` here (see module-migration notes). + void start(); + + // UdpReceiver thread: route the tick to every worker cached for its + // symbol. Enqueue + notify only — never blocks on strategy work. + void onTick(const PriceData& tick); + + // Workers drain their queues, then exit and are joined — shutdown is + // lossless and deterministic under test. Safe to call twice. + void stop(); + + [[nodiscard]] RunnerStats stats() const; + +private: + struct Worker { + explicit Worker(WorkerSpec s) : spec(std::move(s)) { + // Register every declared timeframe before the first tick. A + // malformed spec throws here — at runner construction, loudly — + // not on a worker thread mid-stream. + for (const bars::SeriesSpec& series : spec.barSeries) { + barStore.registerSeries(series.minutes, series.count); + } + for (const rangebar::RangeBarSpec& series : spec.rangeSeries) { + barStore.registerRangeSeries(series); + } + if (spec.gateSeries) { + barStore.registerSeries(spec.gateSeries->minutes, + spec.gateSeries->count); + } + } + WorkerSpec spec; + // The worker's shared bar pipeline, updated once per tick at the top + // of processTick, before the gates and decide() — and its first + // update prepopulates from QuestDB (see barStore). Worker-thread-only + // state like recentOpens: no locking needed. + bars::BarStore barStore; + // The worker's mirror of the BROKER book: syncBook seeds/removes + // trades from the PL#/PO# feed so a strategy's during() sees (and + // may close) real positions. Empty when no feed is wired. The + // broker still owns exits — no markToMarket/reviewStopAndLimit runs + // here; closes flow out through the close diff, never in. + TradeManager tradeManager; + // Broker identity for the booked trade(s), keyed by symbol like the + // TradeManager itself (at most one entry — one-symbol specs). The + // side map is the authority for dealId/dealReference/broker size: + // Trade cannot carry them (no dealId field, no setters). + std::unordered_map sideMap; + // Next steady-clock instant the feed may be consulted; the epoch + // default makes the first tick sync immediately. + std::chrono::steady_clock::time_point nextSyncAt{}; + // workerException trace throttle: a persistently-throwing strategy + // would otherwise emit one trace document per tick. At most one doc + // per 30s per worker, carrying the count it stands for. + // Worker-thread-only state: no locking needed. + std::chrono::steady_clock::time_point nextExceptionTraceAt{}; + std::uint64_t exceptionsSinceTrace{0}; + // Timestamps of orders emitted inside the sliding one-minute window, + // in TICK time (same clock the backtest cap uses; live tick time is + // wall clock anyway). Only pushed to while the cap is active, so it + // holds at most maxTradesPerMinute entries. Worker-thread-only state: + // no locking needed. + std::deque recentOpens; + std::mutex m; + std::condition_variable cv; + std::deque queue; + std::thread thread; + }; + + void workerLoop(Worker& worker); + void processTick(Worker& worker, const PriceData& tick); + static void traceSignalBlocked(const Worker& worker, + std::string_view reason, + std::string_view direction); + static void traceWorkerException(Worker& worker, std::string_view detail); + bool belowTradeRateCap(Worker& worker, + std::chrono::system_clock::time_point now); + bool belowOpenTradeCap(const Worker& worker); + void syncBookIfDue(Worker& worker, const PriceData& tick); + void emitClose(Worker& worker, const Trade& closed, const PriceData& tick); + + // Bounds a worker's backlog through a stall. Drop-OLDEST: for trading the + // newest price is the one that matters, and a bounded queue keeps memory + // flat if a strategy wedges (e.g. the gate's Redis timeout per signal). + static constexpr std::size_t kMaxQueuedTicks = 8192; + + std::vector> workers_; + std::unordered_map> bySymbol_; + TradeGate gate_; + OrderSink sink_; + PositionCounter positions_; + PositionFeed feed_; + CloseSink closeSink_; + std::chrono::seconds bookSyncInterval_; + std::atomic stopping_{false}; + bool started_{false}; + + std::atomic routed_{0}; + std::atomic ignoredSymbol_{0}; + std::atomic queueDropped_{0}; + std::atomic sessionSkipped_{0}; + std::atomic conditionsSkipped_{0}; + std::atomic signals_{0}; + std::atomic rateBlocked_{0}; + std::atomic positionBlocked_{0}; + std::atomic lockBlocked_{0}; + std::atomic ordersLogged_{0}; + std::atomic bookSeeded_{0}; + std::atomic bookRemoved_{0}; + std::atomic bookSyncFailed_{0}; + std::atomic strategyCloses_{0}; + std::atomic closeDropped_{0}; +}; + +} // namespace live + +namespace live { + +namespace { + +std::string_view directionName(const Direction direction) { + return direction == Direction::LONG ? "LONG" : "SHORT"; +} + +} // namespace + +StrategyRunner::StrategyRunner(std::vector specs, TradeGate gate, + OrderSink sink, PositionCounter positions, + PositionFeed feed, CloseSink closeSink, + const std::chrono::seconds bookSyncInterval) + : gate_(std::move(gate)), sink_(std::move(sink)), + positions_(std::move(positions)), feed_(std::move(feed)), + closeSink_(std::move(closeSink)), bookSyncInterval_(bookSyncInterval) { + workers_.reserve(specs.size()); + for (WorkerSpec& spec : specs) { + workers_.push_back(std::make_unique(std::move(spec))); + Worker* worker = workers_.back().get(); + bySymbol_[worker->spec.symbol].push_back(worker); + } +} + +StrategyRunner::~StrategyRunner() { stop(); } + +void StrategyRunner::start() { + if (started_) { + return; + } + // A previous stop() leaves the flag raised; without this reset a + // stop()-then-start() sequence would spawn workers that drain whatever is + // queued and exit immediately — a runner that still accepts ticks but + // silently never processes them. + stopping_.store(false, std::memory_order_seq_cst); + started_ = true; + // Fail-closed is silent per tick, so a missing counter next to an active + // open-trades cap — a wiring bug or a producer not deployed — must be + // loud here, or the runner looks alive while blocking every entry. + if (!positions_) { + for (const auto& worker : workers_) { + if (worker->spec.maxOpenTrades > 0) { + backtest_log::error( + "StrategyRunner: no position counter wired but " + + worker->spec.strategyName + " (" + + worker->spec.strategyUuid + ") caps open trades — its " + "entries will all fail closed"); + } + } + } + // The feed and the close sink only make sense as a pair: a fed book + // whose strategy closes go nowhere silently drops them, and a close + // sink over an empty book can never fire. A wiring bug, not fatal — + // loud once here. + if (static_cast(feed_) != static_cast(closeSink_)) { + backtest_log::error( + feed_ ? "StrategyRunner: position feed wired without a close " + "sink — strategy closes will be dropped (and counted)" + : "StrategyRunner: close sink wired without a position " + "feed — the book stays empty and no close can fire"); + } + for (auto& worker : workers_) { + worker->thread = std::thread([this, w = worker.get()] { workerLoop(*w); }); + } +} + +void StrategyRunner::onTick(const PriceData& tick) { + const auto it = bySymbol_.find(tick.symbol); + if (it == bySymbol_.end()) { + ignoredSymbol_.fetch_add(1, std::memory_order_relaxed); + return; + } + for (Worker* worker : it->second) { + { + const std::lock_guard lock{worker->m}; + worker->queue.push_back(tick); + if (worker->queue.size() > kMaxQueuedTicks) { + worker->queue.pop_front(); + queueDropped_.fetch_add(1, std::memory_order_relaxed); + } + } + worker->cv.notify_one(); + routed_.fetch_add(1, std::memory_order_relaxed); + } +} + +void StrategyRunner::stop() { + if (!started_) { + return; + } + stopping_.store(true, std::memory_order_seq_cst); + for (auto& worker : workers_) { + // Touch the mutex between flag and notify so a worker mid-predicate + // cannot miss the wakeup (the classic lost-notify race). + { const std::lock_guard lock{worker->m}; } + worker->cv.notify_one(); + } + for (auto& worker : workers_) { + if (worker->thread.joinable()) { + worker->thread.join(); + } + } + started_ = false; +} + +void StrategyRunner::workerLoop(Worker& worker) { + for (;;) { + PriceData tick; + { + std::unique_lock lock{worker.m}; + worker.cv.wait(lock, [&] { + return stopping_.load(std::memory_order_relaxed) || + !worker.queue.empty(); + }); + if (worker.queue.empty()) { + return; // stopping and drained + } + tick = std::move(worker.queue.front()); + worker.queue.pop_front(); + } + // A throwing strategy (or gate/sink) must not take the live process + // down — log and move on to the next tick. catch (...) as well: a + // non-std exception escaping the thread function would std::terminate + // the whole process. + try { + processTick(worker, tick); + } catch (const std::exception& e) { + // Suppress the live-logs ship: a wedged strategy throws on EVERY + // tick, and shipping one document per tick is the exact storm the + // workerException trace's 30s throttle exists to prevent — the + // (throttled) trace carries the text off-box; stderr keeps the + // per-tick line. + const backtest_log::SinkSuppression suppression; + backtest_log::error("StrategyRunner: " + worker.spec.strategyName + + " (" + worker.spec.strategyUuid + + ") threw on tick: " + e.what()); + traceWorkerException(worker, e.what()); + } catch (...) { + const backtest_log::SinkSuppression suppression; + backtest_log::error("StrategyRunner: " + worker.spec.strategyName + + " (" + worker.spec.strategyUuid + + ") threw a non-std exception on tick"); + traceWorkerException(worker, "non-std exception"); + } + } +} + +void StrategyRunner::processTick(Worker& worker, const PriceData& tick) { + // Book first, so broker-side removals happen BEFORE the close-diff + // snapshot below — a deal the broker closed must never read as a + // strategy close — and fresh seeds are visible to this tick's + // decide/during. + syncBookIfDue(worker, tick); + + // Shared bar pipeline, EVERY tick, ungated (a gap would corrupt the ATR + // and the strategies' histories) and BEFORE the gates: the ATR + // conditions and decide() judge this tick against bar state that + // already includes it — same ordering as the backtest loop, so the two + // paths cannot diverge. The first update prepopulates from QuestDB + // (see barStore), so a warm feed can trade from its very first tick. + worker.barStore.update(tick); + + // Peak-hours entry gate, before decide(): out-of-session ticks open + // nothing (skipped, never deferred — same doctrine as the caps). The + // book sync above and the during()/close-diff below are NEVER gated. + std::optional distances; + if (worker.spec.peakHoursOnly && + !market_hours::tradePermitted(tick.symbol, tick.timestamp)) { + sessionSkipped_.fetch_add(1, std::memory_order_relaxed); + } else if (worker.spec.gateSeries.has_value() && + !(distances = conditions::check( + worker.barStore, *worker.spec.gateSeries, tick, + worker.spec.vars.STOP_DISTANCE_IN_ATR, + worker.spec.vars.LIMIT_DISTANCE_IN_ATR))) { + // ATR entry conditions failed: gate series not warm, spread too wide + // vs ATR, or volatility below the pip floors — skip decide() like the + // session gate above. Same conditions::check the backtest loop runs, + // so backtest and live can never diverge here. + conditionsSkipped_.fetch_add(1, std::memory_order_relaxed); + } else if (const auto signal = + worker.spec.strategy->decide(tick, worker.barStore)) { + signals_.fetch_add(1, std::memory_order_relaxed); + const std::string direction{directionName(*signal)}; + // Cheapest check first: the rate window is in-memory, the position + // count and the trade lock are Redis round trips. A blocked signal is + // skipped, never deferred — same as the backtest caps. + if (!belowTradeRateCap(worker, tick.timestamp)) { + rateBlocked_.fetch_add(1, std::memory_order_relaxed); + traceSignalBlocked(worker, "rateCap", direction); + } else if (!belowOpenTradeCap(worker)) { + positionBlocked_.fetch_add(1, std::memory_order_relaxed); + traceSignalBlocked(worker, "openTradeCap", direction); + } else if (gate_(worker.spec.strategyUuid, direction)) { + sink_(OrderIntent{ + .strategyName = worker.spec.strategyName, + .strategyUuid = worker.spec.strategyUuid, + .symbol = worker.spec.symbol, + .direction = *signal, + .size = worker.spec.vars.TRADING_SIZE, + // Dynamic ATR-derived pip distances when the gate is + // engaged; the raw variables (test-only path) otherwise. + .stopDistancePips = distances + ? distances->stopPips + : worker.spec.vars.STOP_DISTANCE_IN_ATR, + .limitDistancePips = + distances ? distances->limitPips + : worker.spec.vars.LIMIT_DISTANCE_IN_ATR, + .bid = tick.bid, + .ask = tick.ask, + .timestamp = tick.timestamp, + }); + if (worker.spec.maxTradesPerMinute > 0) { + worker.recentOpens.push_back(tick.timestamp); + } + ordersLogged_.fetch_add(1, std::memory_order_relaxed); + } else { + lockBlocked_.fetch_add(1, std::memory_order_relaxed); + traceSignalBlocked(worker, "tradeLock", direction); + } + } + // TradeManager has no close callback, so a strategy close inside + // during() is detected by diffing the closed-trades log around the call + // (closedTrades is append-only; syncBook's own removals happened above). + const std::size_t closedBefore = + worker.tradeManager.getClosedTrades().size(); + worker.spec.strategy->during(tick, worker.barStore, worker.tradeManager); + const auto& closedTrades = worker.tradeManager.getClosedTrades(); + for (std::size_t i = closedBefore; i < closedTrades.size(); ++i) { + emitClose(worker, closedTrades[i], tick); + } +} + +void StrategyRunner::syncBookIfDue(Worker& worker, const PriceData& tick) { + if (!feed_) { + return; + } + const auto now = std::chrono::steady_clock::now(); + if (now < worker.nextSyncAt) { + return; + } + worker.nextSyncAt = now + bookSyncInterval_; + + const auto fetched = feed_(worker.spec.strategyUuid, worker.spec.symbol); + if (!fetched) { + // FAIL-OPEN for book state (see PositionFeed): keep the last known + // book rather than tearing it down on a Redis blip. + bookSyncFailed_.fetch_add(1, std::memory_order_relaxed); + if (live_trace::enabled()) { + live_trace::emit("bookSyncFailed", + {.strategyUuid = worker.spec.strategyUuid, + .strategyName = worker.spec.strategyName, + .symbol = worker.spec.symbol}); + } + return; + } + + const std::string& symbol = worker.spec.symbol; + if (const auto booked = worker.sideMap.find(symbol); + booked != worker.sideMap.end()) { + const auto match = std::ranges::find_if( + *fetched, [&](const BookedPosition& position) { + return position.dealReference == booked->second.dealReference; + }); + if (match != fetched->end()) { + // Still open at the broker. Refresh the broker identity — this + // is how a confirm-resolved (or producer-reconciled) dealId + // reaches the side map after the open booked without one. + booked->second.dealId = match->dealId; + booked->second.brokerSize = match->brokerSize; + } else { + // Gone from the feed: closed at the broker, pruned by the + // producer, or its PO# expired unconfirmed — either way it no + // longer exists as far as anyone can tell. Remove from the book + // WITHOUT a CloseIntent (nothing to close). closeTrade logs one + // "Trade Closed" line; its book-keeping PnL against the last + // mark is cosmetic — the broker owns the real PnL. + const Trade* trade = worker.tradeManager.findActiveTrade(symbol); + const std::int32_t closePrice = + trade != nullptr ? trade->lastMarkPrice : tick.bid; + // Copied, not referenced: the erase below invalidates `booked` + // and the trace still needs the broker identity. + const BookedPosition removed = booked->second; + worker.tradeManager.closeTrade(symbol, closePrice, tick); + worker.sideMap.erase(symbol); + bookRemoved_.fetch_add(1, std::memory_order_relaxed); + if (live_trace::enabled()) { + live_trace::emit( + "bookRemoved", + {.strategyUuid = worker.spec.strategyUuid, + .strategyName = worker.spec.strategyName, + .symbol = symbol, + .dealReference = removed.dealReference, + .dealId = removed.dealId}, + {{"direction", directionName(removed.direction)}, + {"closePrice", static_cast(closePrice)}}); + } + } + } + + if (!worker.sideMap.contains(symbol) && !fetched->empty() && + !worker.tradeManager.hasActiveTradeForSymbol(symbol)) { + // Seed the OLDEST deal (PL# append order): the book holds at most + // one trade per symbol by TradeManager design, so surplus deals + // (maxOpenTrades > 1, or LONG+SHORT after a lock lapse) are + // acknowledged in the log but not booked — the close path only ever + // addresses the booked one. + const BookedPosition& first = fetched->front(); + const PriceData synthetic{first.level, first.level, first.openedAt, + symbol}; + worker.tradeManager.openTrade(synthetic, first.engineSize, + first.direction, 0, 0); + worker.sideMap.emplace(symbol, first); + bookSeeded_.fetch_add(1, std::memory_order_relaxed); + if (live_trace::enabled()) { + live_trace::emit( + "bookSeeded", + {.strategyUuid = worker.spec.strategyUuid, + .strategyName = worker.spec.strategyName, + .symbol = symbol, + .dealReference = first.dealReference, + .dealId = first.dealId}, + {{"direction", directionName(first.direction)}, + {"brokerSize", first.brokerSize}, + {"engineSize", static_cast(first.engineSize)}, + {"level", static_cast(first.level)}, + {"brokerDeals", fetched->size()}}); + } + if (fetched->size() > 1) { + backtest_log::error( + "StrategyRunner: " + worker.spec.strategyName + " (" + + worker.spec.strategyUuid + ") has " + + std::to_string(fetched->size()) + " broker deals for " + + symbol + " — booking only the oldest (" + + first.dealReference + ")"); + } + } +} + +void StrategyRunner::emitClose(Worker& worker, const Trade& closed, + const PriceData& tick) { + const auto booked = worker.sideMap.find(closed.symbol); + if (!closeSink_ || booked == worker.sideMap.end() || + booked->second.dealId.empty()) { + // No broker identity to address — the deal never confirmed (or the + // book drifted). Count + log; the deal is still in Redis, so the + // next sync re-seeds the book and the strategy may try again once + // an id has appeared. + closeDropped_.fetch_add(1, std::memory_order_relaxed); + backtest_log::error( + "StrategyRunner: dropping strategy close of " + closed.symbol + + " for " + worker.spec.strategyName + " (" + + worker.spec.strategyUuid + ") — " + + (closeSink_ ? "no confirmed broker dealId" + : "no close sink wired (warned at start)")); + if (live_trace::enabled()) { + live_trace::emit( + "closeDropped", + {.strategyUuid = worker.spec.strategyUuid, + .strategyName = worker.spec.strategyName, + .symbol = closed.symbol, + .dealReference = + booked != worker.sideMap.end() + ? std::string_view{booked->second.dealReference} + : std::string_view{}}, + {{"reason", closeSink_ ? "noDealId" : "noCloseSink"}}); + } + if (booked != worker.sideMap.end()) { + worker.sideMap.erase(booked); + } + return; + } + closeSink_(CloseIntent{ + .strategyName = worker.spec.strategyName, + .strategyUuid = worker.spec.strategyUuid, + .symbol = closed.symbol, + .direction = booked->second.direction, + .brokerSize = booked->second.brokerSize, + .dealId = booked->second.dealId, + .dealReference = booked->second.dealReference, + .timestamp = tick.timestamp, + }); + strategyCloses_.fetch_add(1, std::memory_order_relaxed); + // Erase either way: if the broker close succeeded the deal leaves Redis + // too; if it failed, the deal is still there and the next sync re-seeds + // book + side map — a natural, sync-interval-throttled retry. + worker.sideMap.erase(booked); +} + +void StrategyRunner::traceSignalBlocked(const Worker& worker, + const std::string_view reason, + const std::string_view direction) { + if (!live_trace::enabled()) { + return; + } + live_trace::emit("signalBlocked", + {.strategyUuid = worker.spec.strategyUuid, + .strategyName = worker.spec.strategyName, + .symbol = worker.spec.symbol}, + {{"reason", reason}, {"direction", direction}}); +} + +void StrategyRunner::traceWorkerException(Worker& worker, + const std::string_view detail) { + if (!live_trace::enabled()) { + return; + } + ++worker.exceptionsSinceTrace; + const auto now = std::chrono::steady_clock::now(); + if (now < worker.nextExceptionTraceAt) { + return; // counted; the next emitted doc carries the total + } + worker.nextExceptionTraceAt = now + std::chrono::seconds{30}; + live_trace::emit("workerException", + {.strategyUuid = worker.spec.strategyUuid, + .strategyName = worker.spec.strategyName, + .symbol = worker.spec.symbol}, + {{"detail", detail}, + {"occurrences", worker.exceptionsSinceTrace}}); + worker.exceptionsSinceTrace = 0; +} + +bool StrategyRunner::belowTradeRateCap( + Worker& worker, const std::chrono::system_clock::time_point now) { + if (worker.spec.maxTradesPerMinute <= 0) { + return true; + } + // Same half-open window as the backtest cap (runLoop.cppm): an order + // exactly 60 seconds old has aged out and frees its slot on this tick. + // Broker ticks arrive wall-clock-ordered; one arriving with an older + // timestamp just evicts nothing, which can only over-block, never let an + // extra order through. + while (!worker.recentOpens.empty() && + now - worker.recentOpens.front() >= std::chrono::minutes{1}) { + worker.recentOpens.pop_front(); + } + return worker.recentOpens.size() < + static_cast(worker.spec.maxTradesPerMinute); +} + +bool StrategyRunner::belowOpenTradeCap(const Worker& worker) { + if (worker.spec.maxOpenTrades <= 0) { + return true; + } + // FAIL-CLOSED, same doctrine as the trade lock: no counter wired (warned + // at start()) or an unknown count (Redis unreachable — already logged + // down in the client) blocks the entry, because a missed entry is + // recoverable and a position over the cap is not. + if (!positions_) { + return false; + } + const std::optional open = positions_(worker.spec.strategyUuid); + return open && *open < worker.spec.maxOpenTrades; +} + +RunnerStats StrategyRunner::stats() const { + return RunnerStats{ + .routed = routed_.load(std::memory_order_relaxed), + .ignoredSymbol = ignoredSymbol_.load(std::memory_order_relaxed), + .queueDropped = queueDropped_.load(std::memory_order_relaxed), + .sessionSkipped = sessionSkipped_.load(std::memory_order_relaxed), + .conditionsSkipped = conditionsSkipped_.load(std::memory_order_relaxed), + .signals = signals_.load(std::memory_order_relaxed), + .rateBlocked = rateBlocked_.load(std::memory_order_relaxed), + .positionBlocked = positionBlocked_.load(std::memory_order_relaxed), + .lockBlocked = lockBlocked_.load(std::memory_order_relaxed), + .ordersLogged = ordersLogged_.load(std::memory_order_relaxed), + .bookSeeded = bookSeeded_.load(std::memory_order_relaxed), + .bookRemoved = bookRemoved_.load(std::memory_order_relaxed), + .bookSyncFailed = bookSyncFailed_.load(std::memory_order_relaxed), + .strategyCloses = strategyCloses_.load(std::memory_order_relaxed), + .closeDropped = closeDropped_.load(std::memory_order_relaxed), + }; +} + +} // namespace live diff --git a/source/live/execution/orderChannel.cppm b/source/live/execution/orderChannel.cppm new file mode 100644 index 0000000..008e812 --- /dev/null +++ b/source/live/execution/orderChannel.cppm @@ -0,0 +1,552 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// orderChannel — the decision-to-market request layer, porting the C# +// engine's RequestOpenTrade.Request() and IGMarketCalls.ClosePosition / +// SavePosition / DeletePosition flows around the broker call seams. +// +// OPEN (one OrderRequest in, one placement attempt out): +// missing market definition -> log, no broker call, signal dropped (the +// gate's lock simply runs out its TTL — C# semantics; the repeat log +// is rate-limited by that same TTL) +// before placing -> extendLock, so the lock cannot lapse while +// the order is in flight +// Accepted -> book the deal under IG's echoed +// dealReference (the dealId does not exist until the deal confirms; +// the external producer reconciles it later): savePosition (PO#, +// short TTL) + addPosition (PL# list) + saveDealReceipt (DealId#..., +// 60 days). The lock is left to expire, preserving the throttle. +// Rejected -> releaseLock, the strategy may re-enter early +// Failed / placeOrder threw -> extendLock for the failure TTL (the C# +// "trade didn't complete — updating trade lock for 2 minutes") +// +// CLOSE (a booked position in, one close attempt out): +// blank dealId -> refused (nothing addressable at the broker) +// recently closed -> refused (the C# dealId#CLOSE 5-minute +// cache: while a close is in flight, repeat close signals for the +// same deal must not stack further market orders) +// Ok -> removePosition (PO# -> PH# archive + PL# +// prune) and the deal enters the recently-closed window +// Gone (no response at all) -> removePosition too — the C# branch: it is +// missing from IG as far as anyone can tell, and a phantom book entry +// would haunt the strategy logic indefinitely (the producer restores +// it from the broker book if it actually still exists) +// Failed -> keep the book entry; the position still +// exists as far as anyone knows +// +// By the time an open reaches this channel the runner has already run the +// caps and the lock gate (decide -> caps -> gate -> sink) — the gate +// ACQUIRED the lock this channel extends or releases. +// +// Everything side-effectful is injected (market lookup, broker calls, the +// lock/position hooks), so the flow is unit-testable without Redis or a +// broker; brokerOrderSink binds the real implementations. + +export module orderChannel; + +import std; // replaces , , , , +import igMarkets; // ig::TradeOpenObj/TradeCloseObj, results, seams +import backtestLog; // backtest_log::logLine +import liveTrace; // live_trace::emit — live-traces documents +import marketDefinitions; // live::MarketDefinition, MarketLookup +import orderRequest; // live::OrderRequest +import trade; // Direction + +export namespace live { + +// Everything a close needs from the position book. size is in BROKER units +// (as opened — TRADING_SIZE already multiplied by the market's +// sizeModifier); direction is the OPEN position's engine direction. +struct CloseRequest { + std::string strategyUuid; + std::string strategyName; + std::string symbol; + Direction direction{Direction::LONG}; + double size{}; + std::string dealId; // broker deal id — required + std::string dealReference; // the PO#/PL# book key +}; + +// The PO# payload written the moment the broker accepts a deal: the open +// request as JSON under the booked reference. The external position +// producer overwrites it from the broker's own book on its ~2-minute +// refresh, so this only needs to describe the open — exported so tests can +// pin the fields the C# tooling greps for. dealId comes from the confirms +// poll and is empty when the confirm never resolved. +[[nodiscard]] std::string buildPositionPayload(const OrderRequest& request, + const ig::TradeOpenObj& order, + const std::string& dealReference, + const std::string& dealId); + +// The DealId## receipt payload — the C# DealReceipt shape. +// Dated from the decision tick (deterministic under test; within a second +// of the C# DateTime.UtcNow it replaces). +[[nodiscard]] std::string buildDealReceipt(const OrderRequest& request, + const std::string& dealReference, + const std::string& dealId); + +class OrderChannel { +public: + // Lock/position bookkeeping seams. Directions passed to the lock hooks + // are the engine's "LONG"/"SHORT" — the SAME strings the runner's gate + // used to acquire (lock keys are LOCK##); only the + // broker payloads speak "BUY"/"SELL". + struct Hooks { + // The C# PositionClustering.CheckForCluster: TRUE = block the open + // (cluster at capacity, same-(symbol, strategy) already open, or a + // strict cluster losing strategy diversity). Consulted after the + // gate's lock but before anything touches the broker. + std::function + clusterBlocked; + // Called on the Accepted branch only (the open is live at IG): + // re-arms the symbol's CLUSTER_LOCK cooldowns across the CG# + // staleness window — the producer's next sync is what makes the new + // deal visible to clusterBlocked, minutes from now. Optional so + // tests scripting the other hooks need not wire it. + std::function clusterOpened; + std::function + extendLock; + std::function + releaseLock; + std::function + savePosition; + std::function + addPosition; + std::function + saveDealReceipt; + std::function + removePosition; + }; + + // inFlightTtl restarts the gate's lock just before placing (normally the + // same TTL the gate acquired with); failureTtl is the C# two-minute + // brake after a placement that didn't complete; closedTtl is the C# + // 5-minute dealId#CLOSE window suppressing repeat closes of one deal. + OrderChannel(MarketLookup lookup, ig::PlaceOrder placeOrder, + ig::PlaceClose placeClose, Hooks hooks, + std::chrono::seconds inFlightTtl = std::chrono::seconds{30}, + std::chrono::seconds failureTtl = std::chrono::seconds{120}, + std::chrono::seconds closedTtl = std::chrono::minutes{5}); + + // True only when the broker accepted the open. All outcomes are logged; + // bookkeeping failures (Redis down while recording an ACCEPTED deal) are + // logged loudly but still return true — the deal is live regardless, and + // the external producer rebuilds PL#/PO# from the broker's book within + // its refresh cadence. + bool request(const OrderRequest& request); + + // True only when the broker accepted the close. NOT thread-safe with + // itself (the recently-closed window is instance state) — same + // one-instance-per-worker-thread discipline as everything else here. + bool closePosition(const CloseRequest& request); + +private: + MarketLookup lookup_; + ig::PlaceOrder placeOrder_; + ig::PlaceClose placeClose_; + Hooks hooks_; + std::chrono::seconds inFlightTtl_; + std::chrono::seconds failureTtl_; + std::chrono::seconds closedTtl_; + // dealId -> when its recently-closed suppression window ends. In-memory + // like the C# TtlCacheService (per-process there, per-worker here — a + // strategy's deals are only ever touched by its own worker). + std::map recentCloses_; +}; + +} // namespace live + +namespace live { + +namespace { + +std::string_view lockDirection(const Direction direction) { + return direction == Direction::LONG ? "LONG" : "SHORT"; +} + +std::string_view brokerDirection(const Direction direction) { + return direction == Direction::LONG ? "BUY" : "SELL"; +} + +std::string isoUtcSeconds(const std::chrono::system_clock::time_point tp) { + return std::format("{:%FT%TZ}", + std::chrono::floor(tp)); +} + +} // namespace + +std::string buildPositionPayload(const OrderRequest& request, + const ig::TradeOpenObj& order, + const std::string& dealReference, + const std::string& dealId) { + // Hand-rolled on purpose: every field is an internal identifier or a + // number (no user text to escape), matching the flat shape the C# engine + // stores. openedAt is the decision tick in epoch microseconds. + return std::format( + R"({{"dealId":"{}","dealReference":"{}","symbol":"{}","epic":"{}",)" + R"("direction":"{}","size":{},"level":{},"stopLevel":{},)" + R"("limitLevel":{},"strategyId":"{}","strategyName":"{}",)" + R"("openedAt":{}}})", + dealId, dealReference, request.symbol, order.epic, order.direction, + order.size, request.level, request.stopLevel, request.limitLevel, + request.strategyUuid, request.strategyName, + std::chrono::duration_cast( + request.timestamp.time_since_epoch()) + .count()); +} + +std::string buildDealReceipt(const OrderRequest& request, + const std::string& dealReference, + const std::string& dealId) { + // The C# DealReceipt: id/sort mirror the key parts. + return std::format( + R"({{"id":"DealId#{}","sort":"{}","date":"{}","dealReference":"{}",)" + R"("strategyId":"{}","dealId":"{}","strategyName":"{}"}})", + dealReference, request.symbol, isoUtcSeconds(request.timestamp), + dealReference, request.strategyUuid, dealId, request.strategyName); +} + +OrderChannel::OrderChannel(MarketLookup lookup, ig::PlaceOrder placeOrder, + ig::PlaceClose placeClose, Hooks hooks, + const std::chrono::seconds inFlightTtl, + const std::chrono::seconds failureTtl, + const std::chrono::seconds closedTtl) + : lookup_(std::move(lookup)), placeOrder_(std::move(placeOrder)), + placeClose_(std::move(placeClose)), hooks_(std::move(hooks)), + inFlightTtl_(inFlightTtl), failureTtl_(failureTtl), + closedTtl_(closedTtl) {} + +bool OrderChannel::request(const OrderRequest& request) { + using backtest_log::logLine; + + const MarketDefinition* market = lookup_(request.symbol); + if (market == nullptr) { + logLine("OrderChannel: {} - missing from marketDefinitions; dropping " + "{} {} signal", + request.symbol, request.strategyName, + lockDirection(request.direction)); + if (live_trace::enabled()) { + live_trace::emit( + "orderDropped", + {.strategyUuid = request.strategyUuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = request.dealReference}, + {{"reason", "missingMarketDefinition"}, + {"direction", lockDirection(request.direction)}}); + } + return false; + } + + const std::string uuid = request.strategyUuid; + const std::string direction{lockDirection(request.direction)}; + + // Portfolio cluster gate (the C# CheckForCluster), after the trade lock + // and before the broker: correlated-asset capacity, same-symbol + // stacking and strict-cluster strategy diversity, read from the + // producer-maintained CG# sets. A block leaves the gate's lock to run + // out its own TTL — same as the C#, where the lock keeps throttling + // re-signals while the cluster stays busy. + if (hooks_.clusterBlocked(request.symbol, request.strategyName)) { + logLine("OrderChannel: {} - cluster gate blocked {} {} — leaving " + "the trade lock to expire", + request.symbol, request.strategyName, direction); + if (live_trace::enabled()) { + live_trace::emit("orderBlocked", + {.strategyUuid = uuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = request.dealReference}, + {{"reason", "cluster"}, + {"direction", direction}}); + } + return false; + } + + // Keep the gate's lock alive while the order is in flight. A false here + // (Redis unreachable) is logged by the lock layer and NOT fatal: the + // gate acquired the lock moments ago, so its original TTL comfortably + // covers one placement — same as the C# ExtendLock fire-and-forget. + hooks_.extendLock(uuid, direction, inFlightTtl_); + + // Always the MINI contract (IGMarketIdentiferMini), and the C# + // TradeSizeModifier semantics: size scales only when the market sets one. + ig::TradeOpenObj order{ + .currencyCode = std::string{market->currency}, + .epic = std::string{market->epicMini}, + .direction = std::string{brokerDirection(request.direction)}, + .size = static_cast(request.size) * market->sizeModifier(), + .stopDistance = request.stopDistancePips, + .limitDistance = request.limitDistancePips, + .dealReference = request.dealReference, + }; + const ig::OrderContext context{ + .strategyUuid = request.strategyUuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .openDirection = order.direction, + }; + + // A throwing broker call is indistinguishable from a transport failure: + // the order may or may not have reached IG, so it takes the Failed path + // (and must not bubble up past the sink into the worker's tick guard). + ig::OpenResult result; + try { + result = placeOrder_(order, context); + } catch (const std::exception& e) { + result = ig::OpenResult{.status = ig::OpenStatus::Failed, + .reason = e.what()}; + } catch (...) { + result = ig::OpenResult{.status = ig::OpenStatus::Failed, + .reason = "non-std exception from placeOrder"}; + } + + switch (result.status) { + case ig::OpenStatus::Accepted: { + // Book under IG's echoed reference; when the echo is missing fall + // back to the reference WE sent — it names the same deal (the C# + // TEMP-guid fallback, minus the fresh guid nobody can correlate). + std::string reference = result.dealReference; + if (reference.empty()) { + reference = request.dealReference; + logLine("OrderChannel: {} - accepted open echoed no " + "dealReference; booking under ours ({})", + request.symbol, reference); + } + logLine("OrderChannel: TRADE OPEN | {} | Strategy: {} | Deal: {} " + "(dealId={}) | {}", + request.symbol, request.strategyUuid, reference, + result.dealId.empty() ? "unconfirmed" : result.dealId, + order.size); + // Before the book-keeping: the deal exists at IG from this instant, + // and every Redis write below can fail without changing that. + if (hooks_.clusterOpened) { + hooks_.clusterOpened(request.symbol); + } + const bool saved = hooks_.savePosition( + reference, + buildPositionPayload(request, order, reference, result.dealId)); + const bool listed = hooks_.addPosition(uuid, reference); + const bool receipt = hooks_.saveDealReceipt( + reference, request.symbol, + buildDealReceipt(request, reference, result.dealId)); + if (!saved || !listed || !receipt) { + logLine("OrderChannel: {} - deal {} is LIVE but recording it " + "failed (PO# saved={}, PL# listed={}, receipt={}); the " + "position producer will rebuild from the broker book", + request.symbol, reference, saved, listed, receipt); + } + if (live_trace::enabled()) { + // confirmed=false is the never-resolved-confirm path: the POST + // succeeded but no dealId arrived (see igRequests makeOpen). + live_trace::emit( + "orderAccepted", + {.strategyUuid = uuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = reference, + .dealId = result.dealId}, + {{"direction", direction}, + {"confirmed", !result.dealId.empty()}, + {"brokerSize", order.size}, + {"size", static_cast(request.size)}, + {"level", static_cast(request.level)}, + {"stopLevel", static_cast(request.stopLevel)}, + {"limitLevel", + static_cast(request.limitLevel)}, + {"epic", order.epic}, + {"bookkeepingOk", saved && listed && receipt}}); + } + return true; + } + case ig::OpenStatus::Rejected: + logLine("OrderChannel: {} - broker REJECTED {} ({}): {} — releasing " + "trade lock for early re-entry", + request.symbol, order.direction, request.strategyName, + result.reason); + hooks_.releaseLock(uuid, direction); + if (live_trace::enabled()) { + live_trace::emit("orderRejected", + {.strategyUuid = uuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = request.dealReference}, + {{"direction", direction}, + {"detail", result.reason}}); + } + return false; + case ig::OpenStatus::Failed: + default: + logLine("OrderChannel: {} - the trade didn't complete ({}): {} — " + "updating trade lock for {} seconds", + request.symbol, request.strategyName, result.reason, + failureTtl_.count()); + hooks_.extendLock(uuid, direction, failureTtl_); + if (live_trace::enabled()) { + live_trace::emit("orderFailed", + {.strategyUuid = uuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = request.dealReference}, + {{"direction", direction}, + {"detail", result.reason}, + {"failureTtlSeconds", failureTtl_.count()}}); + } + return false; + } +} + +bool OrderChannel::closePosition(const CloseRequest& request) { + using backtest_log::logLine; + + if (request.dealId.empty()) { + // Nothing addressable at the broker — the C# missing-deal guard. + logLine("OrderChannel: {} - close refused: missing dealId for {} ({})", + request.symbol, request.strategyName, request.strategyUuid); + if (live_trace::enabled()) { + live_trace::emit("closeRefused", + {.strategyUuid = request.strategyUuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = request.dealReference}, + {{"reason", "missingDealId"}}); + } + return false; + } + + const auto now = std::chrono::steady_clock::now(); + if (const auto it = recentCloses_.find(request.dealId); + it != recentCloses_.end()) { + if (now < it->second) { + logLine("OrderChannel: {} - close of {} suppressed (a close " + "was accepted within the last {}s)", + request.symbol, request.dealId, closedTtl_.count()); + if (live_trace::enabled()) { + live_trace::emit("closeRefused", + {.strategyUuid = request.strategyUuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = request.dealReference, + .dealId = request.dealId}, + {{"reason", "recentlyClosed"}}); + } + return false; + } + recentCloses_.erase(it); + } + + // Flip the direction: closing a BUY position sells it back, and vice + // versa. + const ig::TradeCloseObj close{ + .direction = ig::closingDirection( + std::string{brokerDirection(request.direction)}), + .dealId = request.dealId, + .size = request.size, + }; + const ig::OrderContext context{ + .strategyUuid = request.strategyUuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .openDirection = std::string{brokerDirection(request.direction)}, + }; + logLine("OrderChannel: {} - requesting close {} dealId={} size={}", + request.symbol, close.direction, close.dealId, close.size); + + ig::CloseResult result; + try { + result = placeClose_(close, context); + } catch (const std::exception& e) { + result = ig::CloseResult{.status = ig::CloseStatus::Failed, + .reason = e.what()}; + } catch (...) { + result = ig::CloseResult{.status = ig::CloseStatus::Failed, + .reason = "non-std exception from placeClose"}; + } + + switch (result.status) { + case ig::CloseStatus::Ok: { + recentCloses_[request.dealId] = now + closedTtl_; + logLine("OrderChannel: TRADE CLOSE | {} | Strategy: {} | Deal: {} | {}", + request.symbol, request.strategyUuid, request.dealReference, + request.size); + bool bookRemoved = false; + if (request.dealReference.empty()) { + // The C# DeletePosition guard: without the book key there is + // nothing to remove — log and leave the PO# TTL / producer to + // groom the book. + logLine("OrderChannel: {} - closed {} but its book reference is " + "missing; leaving the book to the producer", + request.symbol, request.dealId); + } else if (hooks_.removePosition(request.strategyUuid, + request.dealReference)) { + bookRemoved = true; + } else { + logLine("OrderChannel: {} - closed {} but removing book entry {} " + "failed; the position producer will prune it", + request.symbol, request.dealId, request.dealReference); + } + if (live_trace::enabled()) { + live_trace::emit("closeOk", + {.strategyUuid = request.strategyUuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = request.dealReference, + .dealId = request.dealId}, + {{"brokerSize", request.size}, + {"bookRemoved", bookRemoved}}); + } + return true; + } + case ig::CloseStatus::Gone: + // No response at all: as far as anyone can tell the position is + // missing from IG — drop the book entry so a phantom cannot pin the + // strategy's open-trade count forever (C# null-response branch). If + // it DOES still exist, the producer restores it from the broker + // book on its next refresh. + logLine("OrderChannel: {} - close of {} got no response ({}); " + "treating as missing from IG and removing book entry", + request.symbol, request.dealId, result.reason); + if (!request.dealReference.empty()) { + hooks_.removePosition(request.strategyUuid, request.dealReference); + } + if (live_trace::enabled()) { + live_trace::emit("closeGone", + {.strategyUuid = request.strategyUuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = request.dealReference, + .dealId = request.dealId}, + {{"detail", result.reason}}); + } + return false; + case ig::CloseStatus::Failed: + default: + logLine("OrderChannel: {} - close of {} didn't complete ({}); " + "keeping the book entry", + request.symbol, request.dealId, result.reason); + if (live_trace::enabled()) { + live_trace::emit("closeFailed", + {.strategyUuid = request.strategyUuid, + .strategyName = request.strategyName, + .symbol = request.symbol, + .dealReference = request.dealReference, + .dealId = request.dealId}, + {{"detail", result.reason}}); + } + return false; + } +} + +} // namespace live diff --git a/source/live/execution/orderRequest.cppm b/source/live/execution/orderRequest.cppm new file mode 100644 index 0000000..3f1c14c --- /dev/null +++ b/source/live/execution/orderRequest.cppm @@ -0,0 +1,139 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// orderRequest — the C# engine's RequestObject, translated to this engine's +// integer price model: everything a placement needs, snapshotted at decision +// time, with the entry/stop/limit levels precomputed from the tick. +// +// The C# constructor divides pip distances by a scalingFactor to get price +// units; here prices are scaled INT32 points (see priceData) so the same +// conversion is `pips * pointsPerPip` (symbolScale). The levels are +// bookkeeping — the IG order itself carries pip DISTANCES (TradeOpenObj), +// but the levels go into the PO# position payload so the book can be +// inspected and marked without re-deriving them. +// +// Built from the runner's OrderIntent by makeOrderRequest, which also mints +// the deal reference — the client-generated idempotency token IG echoes back +// in the confirm stream. One reference per placement attempt; the trade lock +// already guarantees at most one attempt per (strategy, direction) per TTL +// window, so uuid-prefix + direction + tick milliseconds is collision-free. + +export module orderRequest; + +import std; // replaces , , , +import liveStrategyRunner; // live::OrderIntent +import symbolScale; // symbol_scale::get +import trade; // Direction + +export namespace live { + +struct OrderRequest { + // Identity / audit. + std::string symbol; + std::string strategyUuid; + std::string strategyName; + std::string dealReference; + Direction direction{Direction::LONG}; + + // Tick snapshot (scaled INT32 points, as decoded — see priceData). + std::int32_t bid{}; + std::int32_t ask{}; + std::chrono::system_clock::time_point timestamp; + + // Order parameters from the winning run's config. + std::int32_t size{}; + std::int32_t stopDistancePips{}; // 0 = leg disarmed + std::int32_t limitDistancePips{}; // 0 = leg disarmed + + // Derived at construction (C# RequestObject constructor semantics). + int pointsPerPip{}; // symbolScale points-per-pip + std::int32_t level{}; // entry side: ask for LONG, bid for SHORT + std::int32_t stopLevel{}; // meaningful only when stopDistancePips > 0 + std::int32_t limitLevel{}; // meaningful only when limitDistancePips > 0 + std::int32_t spreadPoints{}; // ask - bid, in points +}; + +// IG constrains deal references to [A-Za-z0-9_-] and 30 chars, so the UUID +// is reduced to its first 8 alphanumerics (no '#'/'-' vocabulary leaks in): +// "-" = 8 + 1 + 1 + 13 = 23 +// chars. Deterministic from its inputs, so tests can pin the format. +[[nodiscard]] std::string makeDealReference( + std::string_view strategyUuid, Direction direction, + std::chrono::system_clock::time_point timestamp); + +// OrderIntent -> OrderRequest. nullopt when the symbol has no entry in +// symbolScale (pointsPerPip unknown means the stop/limit levels — and the +// pip distances themselves — are meaningless): the caller logs and drops, +// same fail-loud doctrine as the ingest path scaling by kUnknown. +[[nodiscard]] std::optional makeOrderRequest( + const OrderIntent& intent); + +} // namespace live + +namespace live { + +std::string makeDealReference( + const std::string_view strategyUuid, const Direction direction, + const std::chrono::system_clock::time_point timestamp) { + std::string prefix; + prefix.reserve(8); + for (const char c : strategyUuid) { + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z')) { + prefix.push_back(c); + if (prefix.size() == 8) { + break; + } + } + } + const auto millis = std::chrono::duration_cast( + timestamp.time_since_epoch()) + .count(); + return std::format("{}-{}{}", prefix, + direction == Direction::LONG ? 'L' : 'S', millis); +} + +std::optional makeOrderRequest(const OrderIntent& intent) { + const int pointsPerPip = symbol_scale::get(intent.symbol); + if (pointsPerPip == symbol_scale::kUnknown) { + return std::nullopt; + } + + OrderRequest request{ + .symbol = intent.symbol, + .strategyUuid = intent.strategyUuid, + .strategyName = intent.strategyName, + .dealReference = makeDealReference(intent.strategyUuid, + intent.direction, intent.timestamp), + .direction = intent.direction, + .bid = intent.bid, + .ask = intent.ask, + .timestamp = intent.timestamp, + .size = intent.size, + .stopDistancePips = intent.stopDistancePips, + .limitDistancePips = intent.limitDistancePips, + .pointsPerPip = pointsPerPip, + }; + + const std::int32_t stopPoints = intent.stopDistancePips * pointsPerPip; + const std::int32_t limitPoints = intent.limitDistancePips * pointsPerPip; + + // Entry at the side the broker fills a market order on; stop is adverse, + // limit is favourable — the C# RequestObject constructor, in points. + if (intent.direction == Direction::LONG) { + request.level = intent.ask; + request.stopLevel = request.level - stopPoints; + request.limitLevel = request.level + limitPoints; + } else { + request.level = intent.bid; + request.stopLevel = request.level + stopPoints; + request.limitLevel = request.level - limitPoints; + } + request.spreadPoints = intent.ask - intent.bid; + return request; +} + +} // namespace live diff --git a/source/live/execution/redisPositionCounter.cppm b/source/live/execution/redisPositionCounter.cppm new file mode 100644 index 0000000..b900e54 --- /dev/null +++ b/source/live/execution/redisPositionCounter.cppm @@ -0,0 +1,136 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// redisPositionCounter — adapts the Redis broker position store +// (shared/redis/positionManager, the PL#/PO# keys an external producer +// refreshes from the broker every ~2 minutes) into the strategy runner's +// PositionCounter seam, for the MAX_OPEN_TRADES entry cap. Counts are cached +// per thread for a short period: a per-tick signaller (RandomStrategy) would +// otherwise pay a Redis GET per tick, and anything fresher than the +// producer's own 2-minute cadence is precision the data doesn't have. +// +// The one event that DOES move the count between producer refreshes is this +// worker's own accepted open (the channel's addPosition lands in PL# +// immediately) — so brokerOrderSink invalidates the cache after each +// successful open/close via invalidateCache(). Without that, a LONG open +// followed by a SHORT signal inside the cache TTL read the stale pre-open +// count and breached MAX_OPEN_TRADES (the trade lock is per-direction, so it +// never serialised the two). + +module; + +#include "shared/redis/positionManager.hpp" + +export module redisPositionCounter; + +import std; +import liveStrategyRunner; // live::PositionCounter + +export namespace live { + +// The per-uuid count cache, pure and clock-free (`now`/`freshUntil` are +// passed in) so tests drive expiry without waiting. Exported for unit tests; +// the runtime instance is a per-worker-thread thread_local below. +class PositionCountCache { +public: + // The cached count, or nullopt when absent or no longer fresh at `now`. + [[nodiscard]] std::optional get( + const std::string& strategyUuid, + const std::chrono::steady_clock::time_point now) const { + const auto it = cache_.find(strategyUuid); + if (it == cache_.end() || now >= it->second.freshUntil) { + return std::nullopt; + } + return it->second.count; + } + + void put(const std::string& strategyUuid, const int count, + const std::chrono::steady_clock::time_point freshUntil) { + cache_[strategyUuid] = CachedCount{count, freshUntil}; + } + + void invalidate(const std::string& strategyUuid) { + cache_.erase(strategyUuid); + } + +private: + struct CachedCount { + int count; + std::chrono::steady_clock::time_point freshUntil; + }; + std::map cache_; +}; + +class RedisPositionCounter { +public: + // Returns a PositionCounter whose callable creates one PositionManager + // per calling worker thread (lazily, on the thread's first capped + // signal), each destroyed at that thread's exit — the same per-thread + // pattern, for the same serialisation reasons, as RedisTradeGate. + static PositionCounter make( + const std::string& redisHost, int redisPort, + std::chrono::seconds cacheTtl = std::chrono::seconds{15}); + + // Drops the CALLING thread's cached count for the strategy, forcing the + // next cap check to re-read PL#. Called by brokerOrderSink right after + // an accepted open / successful close — same worker thread as the cap + // check, so the invalidation always hits the cache that matters. A + // thread that never cached the uuid is a no-op. + static void invalidateCache(const std::string& strategyUuid); +}; + +} // namespace live + +namespace live { + +namespace { + +// One cache per thread, shared by every counter make() returns and by +// invalidateCache(). The pre-extraction code kept this as a thread_local +// inside make()'s lambda, which is the same sharing (one instance per +// thread across all closures) — it just had no invalidation handle. +PositionCountCache& threadCache() { + thread_local PositionCountCache cache; + return cache; +} + +} // namespace + +PositionCounter RedisPositionCounter::make(const std::string& redisHost, + const int redisPort, + const std::chrono::seconds cacheTtl) { + return [redisHost, redisPort, + cacheTtl](const std::string& strategyUuid) -> std::optional { + thread_local std::unique_ptr manager; + if (!manager) { + manager = std::make_unique( + redisHost, redisPort); + } + // Only SUCCESSFUL reads are cached — an unknown count must stay + // unknown (fail closed) rather than serve a stale number, and the + // client's circuit breaker already fail-fasts repeat probes while + // Redis is down. Staleness up to cacheTtl cannot over-open on its + // own: this worker's own opens invalidate (see invalidateCache), + // and any other writer republishes on the producer cadence anyway. + const auto now = std::chrono::steady_clock::now(); + if (const std::optional cached = + threadCache().get(strategyUuid, now)) { + return cached; + } + const std::optional count = + manager->getPositionCount(strategyUuid); + if (count) { + threadCache().put(strategyUuid, *count, now + cacheTtl); + } + return count; + }; +} + +void RedisPositionCounter::invalidateCache(const std::string& strategyUuid) { + threadCache().invalidate(strategyUuid); +} + +} // namespace live diff --git a/source/live/execution/redisPositionFeed.cppm b/source/live/execution/redisPositionFeed.cppm new file mode 100644 index 0000000..527e4e2 --- /dev/null +++ b/source/live/execution/redisPositionFeed.cppm @@ -0,0 +1,132 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// redisPositionFeed — adapts the Redis broker position store (PL#/PO#, see +// shared/redis/positionManager) into the strategy runner's PositionFeed +// seam: the deals one (strategy UUID, symbol) worker should mirror into its +// TradeManager book. Same per-worker-thread pattern as RedisPositionCounter, +// but NO cache here — the runner's bookSyncInterval already bounds the call +// rate to one fetch per worker per interval. +// +// Payloads that fail to decode are logged and SKIPPED: to the sync that +// reads as closed-at-broker, which is safe — a removal sends no order, and +// re-entry stays guarded by the caps and the trade lock. + +module; + +#include "shared/redis/positionManager.hpp" + +export module redisPositionFeed; + +import std; // replaces , , , , +import backtestLog; // backtest_log::logLine +import liveStrategyRunner; // live::PositionFeed, BookedPosition +import marketDefinitions; // live::findMarket +import trade; // Direction + +export namespace live { + +// Pure record -> book-entry mapping, exported for tests. nullopt when the +// direction is neither BUY nor SELL (an unaddressable deal must not enter +// the book). `market` may be null (symbol since dropped from the table): +// the broker size then passes through unscaled, loudly. +[[nodiscard]] std::optional toBookedPosition( + const redis_positions::PositionRecord& record, + const MarketDefinition* market); + +class RedisPositionFeed { +public: + // Returns a PositionFeed whose callable creates one PositionManager per + // calling worker thread (lazily, on the thread's first sync), each + // destroyed at that thread's exit — the same per-thread pattern, for + // the same serialisation reasons, as RedisTradeGate. + static PositionFeed make(const std::string& redisHost, int redisPort); +}; + +} // namespace live + +namespace live { + +std::optional toBookedPosition( + const redis_positions::PositionRecord& record, + const MarketDefinition* market) { + Direction direction{}; + if (record.direction == "BUY") { + direction = Direction::LONG; + } else if (record.direction == "SELL") { + direction = Direction::SHORT; + } else { + return std::nullopt; + } + // Engine lots from the broker size by REVERSING the trade-size modifier + // — truthful to the actual position (a deal opened under an older + // config with a different TRADING_SIZE must not be misbooked from the + // current spec). The broker size itself stays verbatim: it is what a + // close must send. + const double modifier = market != nullptr ? market->sizeModifier() : 1.0; + if (market == nullptr) { + backtest_log::logLine( + "RedisPositionFeed: {} missing from marketDefinitions — booking " + "{} with its broker size unscaled", + record.symbol, record.dealReference); + } + return BookedPosition{ + .dealId = record.dealId, + .dealReference = record.dealReference, + .direction = direction, + .brokerSize = record.size, + .engineSize = static_cast( + std::llround(record.size / modifier)), + .level = record.level, + .openedAt = std::chrono::system_clock::time_point{ + std::chrono::microseconds{record.openedAtMicros}}, + }; +} + +PositionFeed RedisPositionFeed::make(const std::string& redisHost, + const int redisPort) { + return [redisHost, redisPort](const std::string& strategyUuid, + const std::string& symbol) + -> std::optional> { + thread_local std::unique_ptr manager; + if (!manager) { + manager = std::make_unique( + redisHost, redisPort); + } + const auto payloads = manager->getPositionPayloads(strategyUuid); + if (!payloads) { + return std::nullopt; // UNKNOWN — the runner keeps its book + } + std::vector positions; + positions.reserve(payloads->size()); + for (const auto& [reference, payload] : *payloads) { + const auto record = + redis_positions::decodePositionRecord(payload); + if (!record) { + backtest_log::logLine( + "RedisPositionFeed: undecodable PO# payload for {} " + "({} bytes) — skipping (reads as closed-at-broker)", + reference, payload.size()); + continue; + } + // A UUID may trade several symbols across workers — each + // worker's book mirrors only its own symbol's deals. + if (record->symbol != symbol) { + continue; + } + if (auto position = toBookedPosition(*record, findMarket(symbol))) { + positions.push_back(std::move(*position)); + } else { + backtest_log::logLine( + "RedisPositionFeed: {} has unusable direction '{}' — " + "skipping", reference, record->direction); + } + } + return positions; + }; +} + +} // namespace live diff --git a/source/live/execution/redisTradeGate.cppm b/source/live/execution/redisTradeGate.cppm new file mode 100644 index 0000000..6df6afd --- /dev/null +++ b/source/live/execution/redisTradeGate.cppm @@ -0,0 +1,60 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// redisTradeGate — adapts the Redis trade lock (shared/redis/tradeLocks) into +// the strategy runner's TradeGate seam: one lock per (strategy UUID, +// direction), acquired atomically with a TTL, fail-closed on any Redis +// failure. This is the live entry LOCK gate; the runner checks the risk caps +// first (MAX_TRADES_PER_MINUTE in-memory window, MAX_OPEN_TRADES against the +// broker position count — see redisPositionCounter) so a capped signal never +// acquires a lock it cannot use. + +module; + +#include "shared/redis/tradeLocks.hpp" + +export module redisTradeGate; + +import std; +import liveStrategyRunner; // live::TradeGate + +export namespace live { + +class RedisTradeGate { +public: + // Returns a TradeGate whose callable creates one TradeLocks per calling + // worker thread (lazily, on the thread's first signal), each destroyed at + // that thread's exit — i.e. when the runner joins its workers. + static TradeGate make(const std::string& redisHost, int redisPort, + std::chrono::seconds lockTtl); +}; + +} // namespace live + +namespace live { + +TradeGate RedisTradeGate::make(const std::string& redisHost, + const int redisPort, + const std::chrono::seconds lockTtl) { + return [redisHost, redisPort, lockTtl](const std::string& strategyUuid, + const std::string& direction) { + // Per-thread instance: a single shared TradeLocks would serialise + // EVERY worker's gate check behind one mutex and one synchronous + // pump — with Redis down that is the full op deadline paid one + // worker at a time. SET NX is atomic server-side, so per-thread + // connections don't weaken the lock. thread_local also means all + // gates made by this function share one instance per thread; the + // process only ever makes one gate, and only worker threads call it. + thread_local std::unique_ptr locks; + if (!locks) { + locks = std::make_unique(redisHost, + redisPort); + } + return !locks->isThereATradeLock(strategyUuid, direction, lockTtl); + }; +} + +} // namespace live diff --git a/source/live/liveCommand.cppm b/source/live/liveCommand.cppm new file mode 100644 index 0000000..1590058 --- /dev/null +++ b/source/live/liveCommand.cppm @@ -0,0 +1,231 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// liveCommand — the `live` subcommand, now a thin orchestrator over the +// live/ modules: +// +// config/liveSettings env + argv -> Settings +// winners/liveWinners backtesting-winners-current _search -> ranked +// Winners +// winners/liveStrategyCache Winners -> runnable WorkerSpecs (via +// strategyFactory) +// execution/liveStrategyRunner one worker thread per strategy; +// decide -> caps -> gate -> order, during every tick +// execution/redisTradeGate entry gate: Redis lock per (UUID, direction) +// execution/redisPositionCounter broker position count (PL# store) for +// the MAX_OPEN_TRADES cap +// execution/redisPositionFeed broker positions (PL#/PO#) mirrored into +// each worker's TradeManager book; strategy +// closes flow back through the close sink +// execution/brokerOrderSink the order handoff: OrderIntent -> +// orderRequest -> orderChannel -> igRequests +// (gated, retried IG REST calls). Without IG +// session credentials in the environment every +// placement fails and extends its trade lock — +// loud, safe, and effectively a dry run. +// monitoring/liveReporter once-a-minute stats + final summary +// +// Timestamped flushed stdout lines come from the shared backtestLog module +// (shared/utilities), the same logLine every other subcommand uses. +// +// Ticks arrive as UDP datagrams (the same wire format ingest consumes, on +// the live stream's port). The broker owns positions and exits — nothing is +// persisted to QuestDB and no TradeManager exit logic runs here. Reachable +// Elasticsearch with at least one qualifying run is REQUIRED (no winners +// means nothing to trade, exit 1); env knobs are documented in liveSettings. +// +// The GMF only #includes Asio-free headers (the receiver hides Asio behind a +// pimpl), so it is safe to `import std` here. + +module; + +#include "run/reporting/elasticPublisher.hpp" +#include "shared/net/udpReceiver.hpp" + +export module liveCommand; + +import std; +import priceData; // PriceData +import tickPacket; // tick_packet::decodeTick +import backtestLog; // backtest_log::logLine +import liveSettings; // live::Settings +import liveWinners; // live::fetchWinners +import liveStrategyCache; // live::StrategyCache +import liveStrategyRunner; // live::StrategyRunner +import redisTradeGate; // live::RedisTradeGate +import redisPositionCounter; // live::RedisPositionCounter +import redisPositionFeed; // live::RedisPositionFeed +import brokerOrderSink; // live::BrokerOrderSink, BrokerSinks +import igRequests; // ig::IGMarketCalls, dynamoAuthProvider +import liveReporter; // live::LiveReporter +import liveTrace; // live_trace::init/emit — live-traces documents +import marketDefinitions; // live::kTradableSymbols — the winners fan-out +import strategyFactory; // strategies::kActiveStrategies + +export class LiveCommand { +public: + static int run(int argc, const char* argv[]); +}; + +int LiveCommand::run(const int argc, const char* argv[]) { + using backtest_log::logLine; + + const live::Settings settings = live::Settings::fromEnv(argc, argv); + + // Arm the trace channel before any thread exists (init writes its env/ + // hostname strings unsynchronised on purpose — see liveTrace). Only this + // subcommand ever arms it, so every other path's emits stay no-ops. + live_trace::init(settings.tradingEnv); + + // The winners are fetched once, at startup — restart to pick up newer + // backtest results. Zero winners is fatal on purpose: with no refresh, an + // idling zero-strategy process is a permanent silent no-op that *looks* + // alive; failing fast surfaces the real problem (Elastic down, wrong + // host, no qualifying runs) to the operator/supervisor. + logLine("LiveCommand: fetching winners from backtesting-winners-current " + "(performanceScore > {}, maxDrawdownPercent <= {}, " + "calmarScore >= {}, top {} per symbol+strategy, one query per " + "pair: {} strategies x {} symbols)", + settings.minScore, settings.maxDrawdownPercent, + settings.minCalmarScore, settings.topPerGroup, + strategies::kActiveStrategies.size(), + live::kTradableSymbols.size()); + + const std::vector winners = live::fetchWinners( + settings.minScore, settings.maxDrawdownPercent, + settings.minCalmarScore, settings.topPerGroup, + strategies::kActiveStrategies, live::kTradableSymbols); + + if (winners.empty()) { + logLine("LiveCommand: no winning strategies available — nothing to " + "trade; exiting (check ELASTIC_HOST and backtest results)"); + live_trace::emit("startupAborted", {}, {{"reason", "noWinners"}}); + elastic::flushQueuedDocuments(); + return 1; + } + + std::vector specs = live::StrategyCache::build(winners); + if (specs.empty()) { + logLine("LiveCommand: every winner failed to instantiate; exiting"); + live_trace::emit("startupAborted", {}, {{"reason", "noSpecs"}}); + elastic::flushQueuedDocuments(); + return 1; + } + + const std::size_t workerCount = specs.size(); + // Fail loud at startup, not per signal: with no session the channel + // refuses every placement (each burns its 2-minute failure brake) — the + // operator should know before the first order, not after. One probe + // pull of Auth# from DynamoDB (AWS creds from the environment). + const ig::AuthProvider auth = ig::dynamoAuthProvider(settings.tradingEnv); + const bool haveAuth = auth().has_value(); + if (!haveAuth) { + logLine("LiveCommand: WARNING — could not pull IG session " + "Auth#{} from DynamoDB ({} table); every order will fail " + "and extend its trade lock (check AWS credentials in the " + "environment and the login service)", + settings.tradingEnv, "MarketDataLive"); + } + live::BrokerSinks sinks = live::BrokerOrderSink::make( + settings.redisHost, settings.redisPort, settings.lockTtl, + ig::IGMarketCalls::makeLiveOpen(settings.redisHost, + settings.redisPort, auth), + ig::IGMarketCalls::makeLiveClose(settings.redisHost, + settings.redisPort, auth)); + live::StrategyRunner runner( + std::move(specs), + live::RedisTradeGate::make(settings.redisHost, settings.redisPort, + settings.lockTtl), + std::move(sinks.order), + live::RedisPositionCounter::make(settings.redisHost, + settings.redisPort), + live::RedisPositionFeed::make(settings.redisHost, settings.redisPort), + std::move(sinks.close)); + runner.start(); + + std::atomic received{0}; + std::atomic dropped{0}; + + net::UdpReceiver receiver( + settings.bindAddr, settings.bindPort, + [&](std::span bytes) { + const auto tick = tick_packet::decodeTick(bytes); + if (!tick) { + dropped.fetch_add(1, std::memory_order_relaxed); + return; + } + runner.onTick(*tick); + received.fetch_add(1, std::memory_order_relaxed); + }); + + logLine("LiveCommand: starting; binding udp://{}:{} " + "({} live strategies, lockTtl={}s, IG order channel [{}]{})", + settings.bindAddr, settings.bindPort, workerCount, + settings.lockTtl.count(), settings.tradingEnv, + haveAuth ? "" : " WITHOUT a session"); + if (live_trace::enabled()) { + live_trace::emit( + "startup", {}, + {{"winners", winners.size()}, + {"workers", workerCount}, + {"haveAuth", haveAuth}, + {"bindAddr", settings.bindAddr}, + {"bindPort", static_cast(settings.bindPort)}, + {"lockTtlSeconds", settings.lockTtl.count()}, + {"minScore", settings.minScore}, + {"maxDrawdownPercent", settings.maxDrawdownPercent}, + {"minCalmarScore", settings.minCalmarScore}, + {"ohlcPrepopulate", settings.ohlcPrepopulate}}); + } + // The book is now fed from PL#/PO# (one sync per worker per ~15s) and + // strategy closes flow back out — but two operational caveats remain + // worth a line at every startup. + logLine("LiveCommand: note — each worker mirrors its broker positions " + "from Redis (PL#/PO#, ~15s cadence); deals whose confirm never " + "resolved have no dealId and cannot be strategy-closed until " + "one appears, and WITHOUT the external position producer PO# " + "entries expire ~5min after open, fading them from the book"); + // Bar warm-up mode, loud at startup either way: bar-based strategies + // (OhlcBreakout) otherwise build their windows tick by tick, so a + // daily-bar window would trade nothing for weeks. The seeding itself + // lives in ohlcBuilder / rangeBarBuilder (shared with run) and fires at + // each symbol's first tick; per-seed lines land on stderr as bars load. + // One switch covers both bar types — sweeps that silence warm-up DB + // traffic silence range bars too. + if (settings.ohlcPrepopulate) { + logLine("LiveCommand: OHLC/range prepopulate on (the default) — bar " + "histories seed from QuestDB (QUESTDB_HOST/QUESTDB_PORT) at " + "each symbol's first tick; a failed seed logs and falls " + "back to a cold start"); + } else { + logLine("LiveCommand: OHLC/range prepopulate OFF (OHLC_PREPOPULATE=0) " + "— bar strategies warm up from the live stream alone (a " + "big-bar window can take days/weeks before decide() fires); " + "unset it, or set 1, to seed bar histories from QuestDB"); + } + + // Declared after the counters and runner it borrows, so it is destroyed + // (and its thread joined) before they are. + live::LiveReporter reporter(received, dropped, runner); + reporter.start(); + + const bool ok = receiver.run(); // binds the socket, then blocks until SIGINT/SIGTERM + + reporter.stop(); + runner.stop(); // workers drain their queues, then join + + if (!ok) { + live_trace::emit("startupAborted", {}, {{"reason", "bindFailed"}}); + elastic::flushQueuedDocuments(); + return 1; // bind failed (bad address / port in use) — logged + } + + reporter.logFinalSummary(); + // Deliver the shutdown/stats tail now, deterministically, rather than + // leaving it to the publisher's exit-time flush. + elastic::flushQueuedDocuments(); + return 0; +} diff --git a/source/live/monitoring/liveReporter.cppm b/source/live/monitoring/liveReporter.cppm new file mode 100644 index 0000000..27d7076 --- /dev/null +++ b/source/live/monitoring/liveReporter.cppm @@ -0,0 +1,169 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// liveReporter — the live subcommand's once-a-minute throughput report, on a +// background thread (receiver.run() blocks the main thread). Borrows the +// command's receive/drop counters and the runner by reference: the reporter +// must be stopped (or destroyed — the destructor stops it) before they go +// away, which the command guarantees by declaring it after them. + +export module liveReporter; + +import std; +import backtestLog; // backtest_log::logLine +import liveStrategyRunner; // live::StrategyRunner, RunnerStats +import liveTrace; // live_trace::emit — stats/shutdown documents + +export namespace live { + +class LiveReporter { +public: + LiveReporter(const std::atomic& received, + const std::atomic& decodeDropped, + const StrategyRunner& runner) + : received_(received), decodeDropped_(decodeDropped), runner_(runner) {} + + ~LiveReporter() { stop(); } + + LiveReporter(const LiveReporter&) = delete; + LiveReporter& operator=(const LiveReporter&) = delete; + + // Spawns the reporting thread. It wakes every second to notice stop() + // promptly. Plain std::thread + atomic flag on purpose — std::jthread's + // stop_token wait (condition_variable_any) doesn't link under + // `import std` here (see module-migration notes). + void start(); + + // Joins the reporting thread. Safe to call twice. + void stop(); + + // The end-of-run summary line ("shutting down"); call after the runner + // has stopped so the printed stats are final. + void logFinalSummary() const; + +private: + void loop(); + + const std::atomic& received_; + const std::atomic& decodeDropped_; + const StrategyRunner& runner_; + std::atomic running_{false}; + std::thread thread_; +}; + +} // namespace live + +namespace live { + +void LiveReporter::start() { + if (running_.exchange(true, std::memory_order_relaxed)) { + return; + } + thread_ = std::thread([this] { loop(); }); +} + +void LiveReporter::stop() { + running_.store(false, std::memory_order_relaxed); + if (thread_.joinable()) { + thread_.join(); + } +} + +void LiveReporter::loop() { + using clock = std::chrono::steady_clock; + constexpr auto interval = std::chrono::minutes(1); + std::uint64_t lastReceived = 0; + auto nextReport = clock::now() + interval; + while (running_.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + if (clock::now() < nextReport) { + continue; + } + nextReport += interval; + const auto total = received_.load(std::memory_order_relaxed); + const auto delta = total - lastReceived; + lastReceived = total; + const RunnerStats stats = runner_.stats(); + backtest_log::logLine( + "LiveCommand: received={} (+{}, ~{}/s); decode-dropped={}; " + "routed={} ignoredSymbol={} queueDropped={} sessionSkipped={} " + "conditionsSkipped={} signals={} rateBlocked={} " + "positionBlocked={} lockBlocked={} orders={} bookSeeded={} " + "bookRemoved={} bookSyncFailed={} strategyCloses={} " + "closeDropped={}", + total, delta, delta / 60, + decodeDropped_.load(std::memory_order_relaxed), stats.routed, + stats.ignoredSymbol, stats.queueDropped, stats.sessionSkipped, + stats.conditionsSkipped, stats.signals, stats.rateBlocked, + stats.positionBlocked, stats.lockBlocked, stats.ordersLogged, + stats.bookSeeded, stats.bookRemoved, stats.bookSyncFailed, + stats.strategyCloses, stats.closeDropped); + if (live_trace::enabled()) { + live_trace::emit( + "stats", {}, + {{"received", total}, + {"receivedDelta", delta}, + {"decodeDropped", + decodeDropped_.load(std::memory_order_relaxed)}, + {"routed", stats.routed}, + {"ignoredSymbol", stats.ignoredSymbol}, + {"queueDropped", stats.queueDropped}, + {"sessionSkipped", stats.sessionSkipped}, + {"conditionsSkipped", stats.conditionsSkipped}, + {"signals", stats.signals}, + {"rateBlocked", stats.rateBlocked}, + {"positionBlocked", stats.positionBlocked}, + {"lockBlocked", stats.lockBlocked}, + {"orders", stats.ordersLogged}, + {"bookSeeded", stats.bookSeeded}, + {"bookRemoved", stats.bookRemoved}, + {"bookSyncFailed", stats.bookSyncFailed}, + {"strategyCloses", stats.strategyCloses}, + {"closeDropped", stats.closeDropped}}); + } + } +} + +void LiveReporter::logFinalSummary() const { + // decode-dropped: bad size / unknown symbol / corrupt fields. + const RunnerStats stats = runner_.stats(); + backtest_log::logLine( + "LiveCommand: shutting down (received={}, decode-dropped={}, " + "routed={}, ignoredSymbol={}, queueDropped={}, sessionSkipped={}, " + "conditionsSkipped={}, signals={}, rateBlocked={}, " + "positionBlocked={}, lockBlocked={}, orders={}, bookSeeded={}, " + "bookRemoved={}, bookSyncFailed={}, strategyCloses={}, " + "closeDropped={})", + received_.load(), decodeDropped_.load(), stats.routed, + stats.ignoredSymbol, stats.queueDropped, stats.sessionSkipped, + stats.conditionsSkipped, stats.signals, stats.rateBlocked, + stats.positionBlocked, stats.lockBlocked, stats.ordersLogged, + stats.bookSeeded, stats.bookRemoved, stats.bookSyncFailed, + stats.strategyCloses, stats.closeDropped); + if (live_trace::enabled()) { + live_trace::emit( + "shutdown", {}, + {{"received", received_.load()}, + {"decodeDropped", decodeDropped_.load()}, + {"routed", stats.routed}, + {"ignoredSymbol", stats.ignoredSymbol}, + {"queueDropped", stats.queueDropped}, + {"sessionSkipped", stats.sessionSkipped}, + {"conditionsSkipped", stats.conditionsSkipped}, + {"signals", stats.signals}, + {"rateBlocked", stats.rateBlocked}, + {"positionBlocked", stats.positionBlocked}, + {"lockBlocked", stats.lockBlocked}, + {"orders", stats.ordersLogged}, + {"bookSeeded", stats.bookSeeded}, + {"bookRemoved", stats.bookRemoved}, + {"bookSyncFailed", stats.bookSyncFailed}, + {"strategyCloses", stats.strategyCloses}, + {"closeDropped", stats.closeDropped}}); + } +} + +} // namespace live diff --git a/source/live/monitoring/liveTrace.cppm b/source/live/monitoring/liveTrace.cppm new file mode 100644 index 0000000..022c331 --- /dev/null +++ b/source/live/monitoring/liveTrace.cppm @@ -0,0 +1,253 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// liveTrace — structured trace documents for live mode's important junctions +// (order/close lifecycle, gate blocks, book sync, IG guard refusals, +// startup/shutdown, minutely stats), delivered to the Elasticsearch index +// "live-traces" through the shared async batch publisher (elasticPublisher: +// background flusher, retry + dead-letter, never blocks the caller). +// +// Tracing is ARMED, not merely env-gated: every emit is a no-op until +// init() runs, and only LiveCommand::run calls init(). Unit tests construct +// OrderChannel / StrategyRunner / the sinks directly and never init, so +// their trace calls cost one relaxed atomic load — no batcher thread, no +// delivery attempt, no dead-letter file in the test dir. init() arms unless +// $LIVE_TRACE_ENABLED == "0" (default on), and records TRADING_ENVIRONMENT +// and the hostname regardless, so igRequests' auditTrade can stamp `env` +// on live-trades documents even with tracing itself switched off. +// +// Call sites pass typed Fields rather than JSON, so no caller needs +// in its GMF and nothing is serialised while disarmed. +// Convention: `reason` values are static keywords (Kibana-aggregatable); +// dynamic text (broker error bodies, exception messages) goes in `detail`. +// +// init() also registers the backtest_log sink (unless $LIVE_LOG_SHIP_ENABLED +// == "0"), shipping every error()/logLine() line as a "live-logs" document +// through the same batcher — so the human-readable narrative survives off +// the box alongside the structured traces. The publisher suppresses its own +// lines from the sink (backtestLog.hpp) so a delivery outage cannot feed +// itself. +// +// The GMF includes are all Asio-free headers, so `import std` is safe here. + +module; + +#include + +#include "run/reporting/elasticPublisher.hpp" // elastic::enqueueDocument +#include "run/reporting/tradeDocument.hpp" // TradeDocument::isoUtcMillis +#include "run/reporting/tradingResults.hpp" // TradeFinal::localHostname +#include "shared/utilities/backtestLog.hpp" // backtest_log::error +#include "shared/utilities/env.hpp" // env::getOr + +export module liveTrace; + +import std; + +export namespace live_trace { + +// Correlation ids common to most events; empty ones are omitted from the +// document. The views are copied during emit, so they only need to outlive +// the call. +struct Common { + std::string_view strategyUuid{}; + std::string_view strategyName{}; + std::string_view symbol{}; + std::string_view dealReference{}; + std::string_view dealId{}; +}; + +// One event-specific field. The variant covers every scalar the event +// taxonomy needs; string values land as JSON strings. +struct Field { + std::string_view key; + std::variant + value; +}; + +// Called once from LiveCommand::run, on the main thread, before any worker/ +// receiver/reporter thread exists (the env/hostname strings are written here +// and read-only afterwards — thread creation provides the happens-before). +// `tradingEnv` is Settings::fromEnv's already-lowercased TRADING_ENVIRONMENT, +// passed in rather than re-read so the doc field can never disagree with the +// Auth# credentials actually in use. +void init(std::string tradingEnv); + +// Switch tracing AND log shipping back off — the full reset for tests that +// exercise init(); production never disarms. +void disarm(); + +// One relaxed atomic load. Call sites wrap emits in `if (enabled())` so a +// disarmed trace also skips evaluating its field expressions. +[[nodiscard]] bool enabled(); + +// The cached environment name ("demo"/"live"); "unknown" before init(). +[[nodiscard]] std::string_view tradingEnv(); + +// Build the document body (exported for unit tests; deterministic apart from +// @timestamp and hostname). Envelope: @timestamp (UTC, milliseconds — the +// seconds-only stamps used elsewhere would collapse a placement and its +// confirm onto one instant), env, hostname, event, the non-empty Common ids, +// then the fields, all at top level. +[[nodiscard]] std::string buildDocumentJson(std::string_view event, + const Common& common, + std::initializer_list fields); + +// One live-logs document (exported for unit tests): @timestamp/env/hostname +// like the traces, level "error"|"info" (stderr vs stdout origin), and the +// line itself under `message`, capped so a rogue dump cannot bloat the +// index (the cap is generous — normal lines are a few hundred bytes). +[[nodiscard]] std::string buildLogDocumentJson(bool isError, + std::string_view message); + +// No-op unless armed; otherwise queue the document for "live-traces". Never +// blocks on the network and never throws (same doctrine as +// elastic::putEngineException): serialisation failures are logged to stderr +// and swallowed — a trace must never take down an order path. +void emit(std::string_view event, const Common& common, + std::initializer_list fields = {}) noexcept; + +} // namespace live_trace + +namespace live_trace { + +namespace { + +std::atomic& armedFlag() { + static std::atomic armed{false}; + return armed; +} + +std::string& envName() { + static std::string name = "unknown"; + return name; +} + +std::string& hostName() { + static std::string host = "unknown"; + return host; +} + +} // namespace + +namespace { + +// The backtest_log sink (capture-free — the atomic slot holds a plain +// function pointer). Same never-throw doctrine as emit(): a log line must +// never take down the code that logged it. +void shipLogLine(const bool isError, const std::string_view message) noexcept { + try { + elastic::enqueueDocument("live-logs", + buildLogDocumentJson(isError, message)); + } catch (...) { + // Deliberately NOT backtest_log::error — that is the very function + // whose sink just failed; stderr via the mutex-free path would still + // recurse through shipToSink. Swallow: the line already reached + // stdout/stderr. + } +} + +} // namespace + +void init(std::string tradingEnv) { + envName() = std::move(tradingEnv); + hostName() = TradeFinal::localHostname(); + armedFlag().store(env::getOr("LIVE_TRACE_ENABLED", "1") != "0", + std::memory_order_release); + // Independent of the trace flag: the narrative and the structured + // events are separate feeds with separate kill switches. The gate is + // authoritative in both directions — "off" also clears any sink a + // previous init registered, so a re-init cannot leave a stale sink + // shipping. + if (env::getOr("LIVE_LOG_SHIP_ENABLED", "1") != "0") { + backtest_log::setSink(&shipLogLine); + } else { + backtest_log::setSink(nullptr); + } +} + +void disarm() { + armedFlag().store(false, std::memory_order_release); + backtest_log::setSink(nullptr); +} + +bool enabled() { return armedFlag().load(std::memory_order_relaxed); } + +std::string_view tradingEnv() { return envName(); } + +std::string buildDocumentJson(const std::string_view event, + const Common& common, + const std::initializer_list fields) { + nlohmann::json doc{ + {"@timestamp", + TradeDocument::isoUtcMillis(std::chrono::system_clock::now())}, + {"env", envName()}, + {"hostname", hostName()}, + {"event", std::string(event)}, + }; + const auto putId = [&doc](const char* key, const std::string_view value) { + if (!value.empty()) { + doc[key] = std::string(value); + } + }; + putId("strategyUuid", common.strategyUuid); + putId("strategyName", common.strategyName); + putId("symbol", common.symbol); + putId("dealReference", common.dealReference); + putId("dealId", common.dealId); + for (const Field& field : fields) { + std::visit( + [&doc, &field](const auto& value) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + // Same convention as the Common ids: an empty string + // (blank dealKey, empty broker reason) is omitted, not + // indexed as "". + if (!value.empty()) { + doc[std::string(field.key)] = std::string(value); + } + } else { + doc[std::string(field.key)] = value; + } + }, + field.value); + } + return doc.dump(); +} + +std::string buildLogDocumentJson(const bool isError, + const std::string_view message) { + // Generous cap: protects the index from a pathological line (a dumped + // document, a runaway body) without touching normal traffic. + constexpr std::size_t kMaxMessageBytes = 4096; + const nlohmann::json doc{ + {"@timestamp", + TradeDocument::isoUtcMillis(std::chrono::system_clock::now())}, + {"env", envName()}, + {"hostname", hostName()}, + {"level", isError ? "error" : "info"}, + {"message", std::string(message.substr(0, kMaxMessageBytes))}, + }; + // error_handler_t::replace: a log line can carry arbitrary bytes (broker + // response bodies, exception text); U+FFFD beats throwing the doc away. + return doc.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); +} + +void emit(const std::string_view event, const Common& common, + const std::initializer_list fields) noexcept { + if (!enabled()) { + return; + } + try { + elastic::enqueueDocument("live-traces", + buildDocumentJson(event, common, fields)); + } catch (...) { + backtest_log::error("liveTrace: failed to build/queue a trace " + "document — dropping it"); + } +} + +} // namespace live_trace diff --git a/source/live/winners/liveStrategyCache.cppm b/source/live/winners/liveStrategyCache.cppm new file mode 100644 index 0000000..11aa0e0 --- /dev/null +++ b/source/live/winners/liveStrategyCache.cppm @@ -0,0 +1,124 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// liveStrategyCache — turns the winning backtest runs (liveWinners) into +// runnable worker specs for the strategy runner: instantiates each winner's +// StrategyConfig through the shared factory, logging one cache line per +// strategy taken live and a skip line (with the reason) for any winner that +// cannot be traded. One bad historical config must not kill live startup — +// the caller only bails when NOTHING could be instantiated. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module liveStrategyCache; + +import std; +import backtestLog; // backtest_log::logLine +import barStore; // bars::SeriesSpec — worker bar registrations +import entryConditions; // conditions::gateSeriesFor — the ATR gate's series +import liveWinners; // live::Winner +import liveStrategyRunner; // live::WorkerSpec +import rangeBarBuilder; // rangebar::RangeBarSpec — worker range registrations +import strategyFactory; // strategies::makeStrategy +import symbolScale; // symbol_scale::get — validates winner symbols + +export namespace live { + +class StrategyCache { +public: + // Winner -> WorkerSpec, skipping (with a logged reason) winners whose + // symbol is unknown to symbol_scale (the tick decoder drops such symbols, + // so the worker would look cached but never receive a tick), whose config + // has no UUID (the trade-lock identity — without one the lock key would + // collide across every UUID-less strategy) or whose strategy constructor + // rejects the config. + static std::vector build(const std::vector& winners); +}; + +} // namespace live + +namespace live { + +std::vector StrategyCache::build(const std::vector& winners) { + std::vector specs; + specs.reserve(winners.size()); + for (const Winner& winner : winners) { + if (symbol_scale::get(winner.symbol) == symbol_scale::kUnknown) { + backtest_log::logLine("StrategyCache: skipping winner strategy={} " + "symbol={} (runId={}) — symbol not in " + "symbol_scale; no tick could route to it", + winner.strategyName, winner.symbol, winner.runId); + continue; + } + if (winner.config.UUID.empty()) { + backtest_log::logLine("StrategyCache: skipping winner strategy={} " + "symbol={} (runId={}) — config has no UUID", + winner.strategyName, winner.symbol, winner.runId); + continue; + } + try { + // The worker's bar registrations: the strategy's OHLC timeframes + // — {0,0} entries are the documented "builds no bars" sentinel + // (RandomStrategy) and register nothing — plus the ATR entry + // gate's series, always engaged in production. + std::vector barSeries; + for (const auto& ohlcVars : winner.config.OHLC_VARIABLES) { + if (ohlcVars.OHLC_MINUTES >= 1 && ohlcVars.OHLC_COUNT >= 1) { + barSeries.push_back( + {std::chrono::minutes{ohlcVars.OHLC_MINUTES}, + ohlcVars.OHLC_COUNT}); + } + } + // Range-bar series, same sentinel convention as the OHLC loop. + // Winners written before RANGE_VARIABLES existed parse to an + // empty vector and register nothing. + std::vector rangeSeries; + for (const auto& rangeVars : winner.config.RANGE_VARIABLES) { + if (rangeVars.RANGE_ATR_TICK_WINDOW >= 1 && + rangeVars.RANGE_ATR_PERCENT >= 1 && + rangeVars.RANGE_COUNT >= 1) { + rangeSeries.push_back( + {.atrTickWindow = rangeVars.RANGE_ATR_TICK_WINDOW, + .atrPercent = rangeVars.RANGE_ATR_PERCENT, + .count = rangeVars.RANGE_COUNT}); + } + } + WorkerSpec spec{ + .symbol = winner.symbol, + .strategyName = winner.strategyName, + .strategyUuid = winner.config.UUID, + .vars = winner.config.TRADING_VARIABLES, + .maxOpenTrades = winner.maxOpenTrades, + .maxTradesPerMinute = winner.maxTradesPerMinute, + .peakHoursOnly = winner.peakHoursOnly, + .barSeries = std::move(barSeries), + .rangeSeries = std::move(rangeSeries), + .gateSeries = conditions::gateSeriesFor(winner.config), + .strategy = strategies::makeStrategy(winner.config), + }; + backtest_log::logLine( + "StrategyCache: cached strategy={} uuid={} symbol={} " + "score={:.2f} size={} stopAtr={} limitAtr={} " + "maxOpen={} maxPerMin={} peakHours={} (runId={})", + spec.strategyName, spec.strategyUuid, spec.symbol, + winner.performanceScore, spec.vars.TRADING_SIZE, + spec.vars.STOP_DISTANCE_IN_ATR, + spec.vars.LIMIT_DISTANCE_IN_ATR, spec.maxOpenTrades, + spec.maxTradesPerMinute, spec.peakHoursOnly, winner.runId); + specs.push_back(std::move(spec)); + } catch (const std::exception& e) { + backtest_log::logLine("StrategyCache: skipping winner strategy={} " + "uuid={} symbol={} (runId={}) — {}", + winner.strategyName, winner.config.UUID, + winner.symbol, winner.runId, e.what()); + } + } + return specs; +} + +} // namespace live diff --git a/source/live/winners/liveWinners.cppm b/source/live/winners/liveWinners.cppm new file mode 100644 index 0000000..99f3de6 --- /dev/null +++ b/source/live/winners/liveWinners.cppm @@ -0,0 +1,611 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// liveWinners — pulls the winning backtest runs out of Elasticsearch so the +// live subcommand can trade them. The read goes through the rolling +// backtesting-winners-current alias (outcome_index::currentAlias), which the +// weekly load repoints at that week's winners index — so live selects from +// the MOST RECENT weekly batch only, and a past winner keeps trading only by +// re-proving itself each week. A "winner" is a run whose +// results.performanceScore cleared the minimum, whose +// results.maxDrawdownPercent stayed at or under the ceiling (a HARD gate on +// peak-to-trough giveback — the Calmar half of the score only blends the +// drawdown in, so a high-expectancy spiky run can out-score it), whose +// results.calmarScore cleared its floor (growth per unit of drawdown — the +// two drawdown gates catch different shapes: the floor rejects smooth but +// stagnant runs, the ceiling rejects fast growers with deep absolute +// givebacks), AND that ran over the full-history window +// (rolling::kFullHistory — LAST_MONTHS = 9, OFFSET_MONTHS = 0): the +// ladder's shorter 3-month rungs are screening passes, not evidence a +// config should trade live. The window filters are +// kept even though the weekly winners index only holds full-history runs — +// during rollout the alias may be hand-parked on the mixed-rung legacy +// trading_results index (see QUICKSTART), and the filters cost nothing after. +// live keeps the top N runs per (symbol, strategy) pair, restricted to the +// caller's active strategy names. +// +// The fetch is one _search PER (active strategy, tradable symbol) pair, not +// one page per strategy: a score-sorted page shared across symbols is +// monopolised by whichever symbol sweeps hottest (observed 2026-07-12: +// FvgStrategy's 12.3k qualifying docs put its top-1000 cutoff at score ~51, +// starving 10 of its 15 winning symbols before grouping ever saw them — the +// same starvation the old one-global-page fetch inflicted across +// strategies). A multi-symbol document matches each of its symbols' queries +// and lands in the merged group map once per fetch; those copies are exact +// duplicates, so the behavioural-clone dedup below removes them. +// Selection within a group also enforces behavioural diversity — +// re-running `load` mints fresh UUIDs for byte-identical configs, and +// near-identical configs backtest near-identically, so a plain top-N books +// the same behaviour several times over (see sameResults / isDiverse below). +// +// Parsing deliberately touches only config.SYMBOLS, config.STRATEGY, RUN_ID, +// results.performanceScore/finalPnl/tradesClosed and the two integer risk +// caps (MAX_OPEN_TRADES, MAX_TRADES_PER_MINUTE) — never the whole +// Configuration. The reporting path +// re-types STARTING_BALANCE / MAX_LOSS_PERCENT as JSON numbers for Kibana +// (see reportConfigJson in tradingResults.cpp), which the shared +// string-encoded decimal from_json would reject. config.STRATEGY is safe both +// ways (readIntField accepts numbers or strings), and the caps are plain +// JSON ints in both encodings. + +module; + +#include + +#include "run/reporting/elasticPublisher.hpp" +#include "run/reporting/outcomeIndices.hpp" +#include "shared/tradingDefinitions/config/runConfiguration.hpp" +#include "shared/tradingDefinitions/strategyConfig.hpp" +#include "shared/utilities/backtestLog.hpp" + +export module liveWinners; + +import std; // replaces , , , , +import rollingWindow; // rolling::kFullHistory — the ladder's terminal full run +import symbolGroups; // sweep::splitSymbols — the same trim+split the sweeps use + +export namespace live { + +// One tradable winner: a past run's full StrategyConfig bound to a single +// symbol (multi-symbol runs are split into one Winner per symbol). +struct Winner { + std::string symbol; + std::string strategyName; // == config.TRADING_VARIABLES.STRATEGY + std::string runId; // provenance for logs + double performanceScore{0.0}; + // Behavioural provenance, doing double duty at selection time: + // the exact tuple (performanceScore, finalPnl, tradesClosed) is the + // clone key (re-running `load` mints fresh UUIDs for byte-identical + // configs, and near-identical configs can backtest byte-identically), + // and finalPnl/tradesClosed plus the long/short mix below are the + // diversity axes (see isDiverse). The strategy runner never reads any + // of them. -1 marks a document that predates the field, so an old doc + // and a genuine 0-trade run never collide on the fallback key, and + // isDiverse knows when a field is unavailable rather than zero. + double finalPnl{0.0}; + long long tradesClosed{-1}; + long long openedLong{-1}; + long long openedShort{-1}; + // Run-level risk caps the winning run backtested under, enforced live by + // the strategy runner. Defaults mirror tradingDefinitions:: + // RunConfiguration for documents written before the caps existed. + int maxOpenTrades{0}; + int maxTradesPerMinute{60}; + bool peakHoursOnly{false}; + tradingDefinitions::StrategyConfig config; +}; + +// Pure selection over an Elasticsearch _search response body (no I/O, unit +// tested with synthetic fixtures): keep hits whose strategy name is in +// `activeStrategies`, split comma-separated config.SYMBOLS into one candidate +// per symbol, and pick up to `topPerGroup` per (symbol, strategy) by +// performanceScore descending, subject to two constraints against every +// already-picked candidate in the group: not a behavioural clone (identical +// results tuple) and behaviourally diverse (the results sit apart on at +// least one axis — trade count, PnL, or long/short mix; see isDiverse). +// A group with fewer diverse survivors books fewer — a backfilled +// neighbour adds risk concentration, not coverage. A malformed hit is +// logged and skipped — one bad historical document must not kill live +// startup. Output is ordered by symbol, then strategy name, then score +// descending, so startup logs and tests are deterministic. +std::vector selectTopWinners( + const std::string& searchResponseBody, + std::span activeStrategies, + std::size_t topPerGroup); + +// The _search body fetchWinners issues for one (strategy, symbol) pair: the +// score floor, drawdown ceiling, calmar floor and full-history window +// filters, a term on the .keyword subfield of the strategy name — the base field is analyzed +// text, so a term there silently matches nothing and live would start with +// zero winners and no clue why — and a match on the ANALYZED config.SYMBOLS +// field, which is deliberately not the .keyword: a term there would silently +// drop legacy multi-symbol documents ("EURUSD,USDJPY"), where the analyzer's +// tokenisation still hits each symbol. Exported so a test can pin both +// paths. +std::string buildWinnersQueryBody(double minScore, + double maxDrawdownPercent, + double minCalmarScore, + std::string_view strategyName, + std::string_view symbol); + +// Startup fetch: one _search on the backtesting-winners-current alias PER +// (active strategy, symbol) pair, each filtered server-side to +// performanceScore > minScore AND maxDrawdownPercent <= maxDrawdownPercent +// AND calmarScore >= minCalmarScore AND the full-history window +// (rolling::kFullHistory), sorted by score descending, so if more than the +// request size qualify, the truncation keeps that pair's best. `symbols` +// is the live allowlist (live::kTradableSymbols) — a winner on any other +// symbol could only book a worker whose orders the channel drops. A pair +// whose query or parse fails is logged and skipped — trading the remaining +// pairs beats trading none, and a persistent outage fails every query so +// the result degenerates to the empty vector the caller already treats as +// fatal. +std::vector fetchWinners(double minScore, double maxDrawdownPercent, + double minCalmarScore, + std::size_t topPerGroup, + std::span activeStrategies, + std::span symbols); + +} // namespace live + +namespace { + +// Upper bound on hits fetched per (strategy, symbol) startup query. The page +// is sorted by score, so truncation keeps that pair's best 5000 — sized above +// the largest group observed so far (4,345 docs, FvgStrategy/DEUIDXEUR, +// 2026-07-12) and under Elasticsearch's default 10k max_result_window; if a +// sweep outgrows it, fetchWinners logs the truncation loudly per pair. +constexpr std::size_t kMaxHits = 5000; + +// LEGACY FALLBACK ONLY (see isDiverse): config-space diversity governs a +// pair only when either document predates results.tradesClosed. For those +// documents, a candidate whose OHLC_COUNT sits within this many bars of a +// picked one (on the same OHLC_MINUTES) is a parameter jiggle, not a +// distinct strategy: the count is derived from the real knob +// (LOOKBACK_BARS + 3, SMA period + 2, ...) for every strategy except +// ohlcBreakout — so the gap indirectly diversifies those drivers too. +constexpr int kOhlcCountDiversityGap = 10; + +// Behavioural neighbourhood: two candidates are the same behaviour unless +// their results sit apart on at least one axis (trade count, PnL, +// long/short mix). The band is relative — kBehaviourNeighbourFrac of the +// larger value — with absolute floors so relative maths cannot manufacture +// diversity out of noise: small trade counts are noisy in relative terms, +// and sub-floor PnL differences are spread/slippage noise, not behaviour +// (a 10-vs-20 PnL pair is one strategy twice, and so is +5 vs -5 — the +// floor subsumes the sign question, while +1000 vs -1000 clears both +// bounds and stays distinct). Distinct requires STRICTLY exceeding the +// bound, mirroring the legacy count-gap boundary doctrine. kMinPnlGap +// assumes account-currency PnL at the observed winner scale (9-month PnLs +// ~1-2k, 2026-07-12 batch); revisit if TRADING_SIZE or the account scale +// shifts. +constexpr double kBehaviourNeighbourFrac = 0.20; +constexpr long long kMinTradeGap = 3; +constexpr double kMinPnlGap = 500.0; + +// Shape-checked walk to hits.hits. Elasticsearch always returns that +// envelope, but a 2xx body from something else (a proxy's error JSON, a +// non-object top level, {"hits":null}) must take the empty-return path — a +// throwing accessor here would escape selectTopWinners' per-hit guard and +// take live startup down with an uncaught type_error. Returns a pointer into +// `response` (no copy of the hits tree). +const nlohmann::json* findHitsArray(const nlohmann::json& response) { + if (!response.is_object()) { + return nullptr; + } + const auto hits = response.find("hits"); + if (hits == response.end() || !hits->is_object()) { + return nullptr; + } + const auto list = hits->find("hits"); + if (list == hits->end() || !list->is_array()) { + return nullptr; + } + return &*list; +} + +} // namespace + +namespace live { + +// (symbol, strategyName) -> candidates, shared by the collect and pick +// phases so fetchWinners can merge its per-strategy responses into one map +// before picking — std::map iteration keeps the output ordered by symbol +// then strategy name, with no concatenate-then-resort pass. +using GroupMap = + std::map, std::vector>; + +// Behavioural-clone test: an identical results tuple means the market never +// noticed the difference between the two configs. Exact double equality is +// deliberate — clones are byte-identical documents, not merely close ones. +static bool sameResults(const Winner& a, const Winner& b) { + return std::tie(a.performanceScore, a.finalPnl, a.tradesClosed) + == std::tie(b.performanceScore, b.finalPnl, b.tradesClosed); +} + +// True when the two configs' OHLC series differ meaningfully: any element +// differs in OHLC_MINUTES (the axis that actually moves results), or (same +// minutes) the counts sit more than kOhlcCountDiversityGap apart. Different +// series counts are trivially diverse. Equal-and-empty is NOT diversity — +// legacyConfigDiverse below owns that judgement across both bar types. +static bool ohlcDiffers(const std::vector& x, + const std::vector& y) { + if (x.size() != y.size()) { + return true; + } + for (std::size_t i = 0; i < x.size(); ++i) { + if (x[i].OHLC_MINUTES != y[i].OHLC_MINUTES) { + return true; + } + if (std::abs(x[i].OHLC_COUNT - y[i].OHLC_COUNT) + > kOhlcCountDiversityGap) { + return true; + } + } + return false; +} + +// Range-bar analogue: TICK_WINDOW and PERCENT are the real knobs (the series +// identity); RANGE_COUNT is ignored because the makers derive it from the +// strategy's scan depths — comparing it would mistake a derivation artefact +// for diversity. +static bool rangeDiffers( + const std::vector& x, + const std::vector& y) { + if (x.size() != y.size()) { + return true; + } + for (std::size_t i = 0; i < x.size(); ++i) { + if (x[i].RANGE_ATR_TICK_WINDOW != y[i].RANGE_ATR_TICK_WINDOW || + x[i].RANGE_ATR_PERCENT != y[i].RANGE_ATR_PERCENT) { + return true; + } + } + return false; +} + +// Legacy config-space diversity, kept ONLY for documents that predate +// results.tradesClosed (isDiverse dispatches here when either side carries +// the -1 fallback): diverse when either bar-series axis differs +// meaningfully. Identical shapes on both axes are vacuously diverse ONLY +// when neither config builds any bars at all (RandomStrategy) — clone +// dedup is the sole guard there. +static bool legacyConfigDiverse(const Winner& a, const Winner& b) { + if (ohlcDiffers(a.config.OHLC_VARIABLES, b.config.OHLC_VARIABLES)) { + return true; + } + if (rangeDiffers(a.config.RANGE_VARIABLES, b.config.RANGE_VARIABLES)) { + return true; + } + return a.config.OHLC_VARIABLES.empty() && b.config.OHLC_VARIABLES.empty() && + a.config.RANGE_VARIABLES.empty() && b.config.RANGE_VARIABLES.empty(); +} + +// Trade-count axis: close when the difference sits inside the relative +// band or the absolute floor (see the kBehaviour* comment). +static bool tradesClose(const Winner& a, const Winner& b) { + const long long delta = a.tradesClosed >= b.tradesClosed + ? a.tradesClosed - b.tradesClosed + : b.tradesClosed - a.tradesClosed; + const double band = std::max( + static_cast(kMinTradeGap), + kBehaviourNeighbourFrac + * static_cast(std::max(a.tradesClosed, b.tradesClosed))); + return static_cast(delta) <= band; +} + +// PnL axis: same shape. The floor makes sign flips inside the noise band +// close (+5 vs -5) while materially opposite results stay distinct +// (+1000 vs -1000 has delta 2000, clearing both bounds). +static bool pnlClose(const Winner& a, const Winner& b) { + const double band = std::max( + kMinPnlGap, + kBehaviourNeighbourFrac + * std::max(std::abs(a.finalPnl), std::abs(b.finalPnl))); + return std::abs(a.finalPnl - b.finalPnl) <= band; +} + +// Long/short-mix axis: close when the long fractions sit within the band. +// Two parameterisations of one strategy can flip directional emphasis while +// landing on near-identical counts and PnL — partially hedging pairs the +// portfolio wants to keep, so a flipped mix is diversity. The axis cannot +// claim distinctness when either side lacks the fields (mid-vintage +// documents, -1) or opened nothing — it reads close and leaves the verdict +// to the other axes. +static bool directionClose(const Winner& a, const Winner& b) { + if (a.openedLong < 0 || a.openedShort < 0 || b.openedLong < 0 || + b.openedShort < 0) { + return true; + } + const long long aTotal = a.openedLong + a.openedShort; + const long long bTotal = b.openedLong + b.openedShort; + if (aTotal == 0 || bTotal == 0) { + return true; + } + const double aFrac = + static_cast(a.openedLong) / static_cast(aTotal); + const double bFrac = + static_cast(b.openedLong) / static_cast(bTotal); + return std::abs(aFrac - bFrac) <= kBehaviourNeighbourFrac; +} + +// Diverse when the two candidates' RESULTS sit apart on any axis — the +// backtest already measured whether the market distinguished them, which is +// what config-space identity used to approximate (and misjudged both ways, +// observed 2026-07-12: LSR configs whose VALID_BARS flip closed 41 vs 60 +// trades on one 60m series — distinct behaviour read as a jiggle — while +// two Fvg configs sharing a 15m scan series and differing only in HTF +// filter traded near-identical books yet read as diverse). Neighbour = +// close on ALL axes; the config check survives solely for documents too +// old to carry the behavioural fields. +static bool isDiverse(const Winner& a, const Winner& b) { + if (a.tradesClosed < 0 || b.tradesClosed < 0) { + return legacyConfigDiverse(a, b); + } + return !(tradesClose(a, b) && pnlClose(a, b) && directionClose(a, b)); +} + +// Parsing half of selection — module-private so fetchWinners (which also +// reads hits.total) parses each body exactly once, appending one Winner per +// (hit x symbol) into `groups` across calls. Not an exported overload of +// selectTopWinners: a string literal converts equally well to std::string +// and nlohmann::json, so an overload pair would be ambiguous at every +// literal call site. +static void collectCandidates( + const nlohmann::json& searchResponse, + std::span activeStrategies, GroupMap& groups) { + const nlohmann::json* hitsArray = findHitsArray(searchResponse); + if (hitsArray == nullptr) { + backtest_log::error( + "liveWinners: _search response is not an Elasticsearch hits " + "envelope; treating as no winners"); + return; + } + + // Fallbacks for documents predating the risk caps — the same defaults the + // backtest itself would have run those configs under. + const tradingDefinitions::RunConfiguration riskDefaults{}; + + for (const auto& hit : *hitsArray) { + try { + const nlohmann::json& source = hit.at("_source"); + const nlohmann::json& config = source.at("config"); + const nlohmann::json& results = source.at("results"); + + Winner base; + base.runId = source.value("RUN_ID", ""); + base.performanceScore = + results.at("performanceScore").get(); + // Clone-key / diversity-axis fields: absent (or non-numeric) in + // documents that predate them — fall back rather than let the + // per-hit guard discard the whole hit. + if (const auto it = results.find("finalPnl"); + it != results.end() && it->is_number()) { + base.finalPnl = it->get(); + } + if (const auto it = results.find("tradesClosed"); + it != results.end() && it->is_number()) { + base.tradesClosed = it->get(); + } + if (const auto it = results.find("openedLong"); + it != results.end() && it->is_number()) { + base.openedLong = it->get(); + } + if (const auto it = results.find("openedShort"); + it != results.end() && it->is_number()) { + base.openedShort = it->get(); + } + base.config = config.at("STRATEGY") + .get(); + base.strategyName = base.config.TRADING_VARIABLES.STRATEGY; + base.maxOpenTrades = + config.value("MAX_OPEN_TRADES", riskDefaults.MAX_OPEN_TRADES); + base.maxTradesPerMinute = config.value( + "MAX_TRADES_PER_MINUTE", riskDefaults.MAX_TRADES_PER_MINUTE); + base.peakHoursOnly = + config.value("PEAK_HOURS_ONLY", riskDefaults.PEAK_HOURS_ONLY); + + if (std::ranges::find(activeStrategies, base.strategyName) + == activeStrategies.end()) { + continue; + } + + // sweep::splitSymbols trims stray whitespace and drops empty + // fields, exactly like the sweep side — an untrimmed " USDJPY" + // would cache a worker no decoded tick can ever route to. The + // returned views point into symbolsField, hence the named local. + const auto symbolsField = config.at("SYMBOLS").get(); + for (const std::string_view symbol : + sweep::splitSymbols(symbolsField)) { + Winner w = base; + w.symbol = std::string{symbol}; + groups[{w.symbol, w.strategyName}].push_back(std::move(w)); + } + } catch (const std::exception& e) { + backtest_log::error(std::string("liveWinners: skipping malformed hit (") + + e.what() + "): " + hit.dump()); + } + } +} + +// Ranking half: per group, walk candidates by score descending and keep up +// to topPerGroup that are neither behavioural clones of (sameResults) nor +// parameter jiggles on (isDiverse) every already-kept candidate. +static std::vector pickWinners(GroupMap& groups, + const std::size_t topPerGroup) { + std::vector winners; + for (auto& [key, candidates] : groups) { + // stable_sort keeps document order for equal scores, so ties resolve + // the same way every startup. + std::ranges::stable_sort(candidates, [](const Winner& a, const Winner& b) { + return a.performanceScore > b.performanceScore; + }); + std::vector picked; + for (Winner& candidate : candidates) { + if (picked.size() == topPerGroup) { + break; + } + const bool clone = std::ranges::any_of( + picked, [&candidate](const Winner& p) { + return sameResults(candidate, p); + }); + const bool diverse = std::ranges::all_of( + picked, [&candidate](const Winner& p) { + return isDiverse(candidate, p); + }); + if (clone || !diverse) { + continue; + } + picked.push_back(std::move(candidate)); + } + // groups is a std::map, so iteration (and therefore the output) is + // already ordered by symbol then strategy name. + winners.insert(winners.end(), + std::make_move_iterator(picked.begin()), + std::make_move_iterator(picked.end())); + } + return winners; +} + +std::vector selectTopWinners( + const std::string& searchResponseBody, + std::span activeStrategies, + const std::size_t topPerGroup) { + nlohmann::json response; + try { + response = nlohmann::json::parse(searchResponseBody); + } catch (const std::exception& e) { + backtest_log::error(std::string("liveWinners: unparseable _search response: ") + + e.what()); + return {}; + } + GroupMap groups; + collectCandidates(response, activeStrategies, groups); + return pickWinners(groups, topPerGroup); +} + +std::string buildWinnersQueryBody(const double minScore, + const double maxDrawdownPercent, + const double minCalmarScore, + const std::string_view strategyName, + const std::string_view symbol) { + // Server-side eligibility: the score floor, the drawdown ceiling, the + // calmar floor and the full-history window, narrowed to one + // (strategy, symbol). LAST_MONTHS must equal the terminal rung's + // exactly, but OFFSET_MONTHS is excluded when beyond the rung's (> 0) + // instead of term-matched to it, because documents written before the + // field existed carry no OFFSET_MONTHS at all yet ran offset-0 — the + // same missing-means-default read RunConfiguration's from_json gives + // them. The ceiling uses lte on results.maxDrawdownPercent and the + // floor gte on results.calmarScore, so a document missing either field + // does NOT match — fail-closed, matching the trade-lock doctrine. + const nlohmann::json scoreFloor{ + {"range", {{"results.performanceScore", {{"gt", minScore}}}}}}; + const nlohmann::json drawdownCeiling{ + {"range", + {{"results.maxDrawdownPercent", {{"lte", maxDrawdownPercent}}}}}}; + const nlohmann::json calmarFloor{ + {"range", {{"results.calmarScore", {{"gte", minCalmarScore}}}}}}; + const nlohmann::json fullWindow{ + {"term", {{"config.LAST_MONTHS", rolling::kFullHistory.lastMonths}}}}; + const nlohmann::json strategyTerm{ + {"term", + {{"config.STRATEGY.TRADING_VARIABLES.STRATEGY.keyword", + std::string{strategyName}}}}}; + // match, not a .keyword term: the analyzer tokenises a legacy + // multi-symbol SYMBOLS ("EURUSD,USDJPY") so each symbol's query still + // finds it; a whole-field term would silently drop those documents. + const nlohmann::json symbolMatch{ + {"match", {{"config.SYMBOLS", std::string{symbol}}}}}; + const nlohmann::json offsetWindow{ + {"range", + {{"config.OFFSET_MONTHS", {{"gt", rolling::kFullHistory.offsetMonths}}}}}}; + const nlohmann::json fullRunQuery{ + {"bool", + {{"filter", + nlohmann::json::array({scoreFloor, drawdownCeiling, calmarFloor, + fullWindow, strategyTerm, symbolMatch})}, + {"must_not", nlohmann::json::array({offsetWindow})}}}}; + const nlohmann::json body{ + {"query", fullRunQuery}, + {"sort", nlohmann::json::array( + {{{"results.performanceScore", "desc"}}})}, + {"size", kMaxHits}, + }; + return body.dump(); +} + +std::vector fetchWinners( + const double minScore, const double maxDrawdownPercent, + const double minCalmarScore, const std::size_t topPerGroup, + std::span activeStrategies, + std::span symbols) { + GroupMap groups; + const std::string winnersAlias = + outcome_index::currentAlias(outcome_index::kWinnersBase); + for (const std::string_view strategyName : activeStrategies) { + for (const std::string_view symbol : symbols) { + const std::string pair = + std::string{strategyName} + "/" + std::string{symbol}; + std::string response; + long httpStatus = 0; + const int rc = elastic::searchIndex( + winnersAlias, + buildWinnersQueryBody(minScore, maxDrawdownPercent, + minCalmarScore, strategyName, symbol), + response, httpStatus); + if (rc != 0) { + // No fallback on a 4xx: retrying (say, without the sort + // clause) would silently void the invariant that truncation + // keeps the best-scoring docs, and mask a genuine query bug. + // Log the body — for a 400 it carries Elasticsearch's reason + // — and move on to the remaining pairs: searchIndex already + // retried transient failures internally, so trading the + // others beats trading none, and a persistent outage fails + // every query and lands on the caller's fatal zero-winners + // path anyway. + backtest_log::error( + "liveWinners: " + winnersAlias + " _search for " + pair + + " failed (code " + std::to_string(rc) + ", HTTP " + + std::to_string(httpStatus) + "): " + + response.substr(0, 500)); + continue; + } + + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(response); + } catch (const std::exception& e) { + backtest_log::error( + "liveWinners: unparseable _search response for " + pair + + ": " + e.what()); + continue; + } + + // Loud per-pair truncation: if more docs qualify than the page + // holds, say so instead of silently pretending full coverage. + // With the sort in place the kept page is still that pair's + // best. + try { + const auto total = parsed.at("hits").at("total").at("value") + .get(); + if (total > kMaxHits) { + backtest_log::error( + "liveWinners: " + std::to_string(total) + " " + pair + + " docs cleared minScore but only " + + std::to_string(kMaxHits) + " were fetched"); + } + } catch (const std::exception&) { + // hits.total is informational only; absence is not an error. + } + + collectCandidates(parsed, activeStrategies, groups); + } + } + return pickWinners(groups, topPerGroup); +} + +} // namespace live diff --git a/source/load/config/fvgStrategy/fvgStrategySweep.cppm b/source/load/config/fvgStrategy/fvgStrategySweep.cppm new file mode 100644 index 0000000..3ebb30b --- /dev/null +++ b/source/load/config/fvgStrategy/fvgStrategySweep.cppm @@ -0,0 +1,78 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module fvgStrategySweep; + +export import parameterGenerator; // buildFvgStrategySweep() returns sweep::ParameterGenerator + +import std; // replaces , + +export namespace sweep { + +// The symbol groups THIS sweep runs against (see kSymbolGroupsOverride in +// randomStrategySweep for the full semantics: empty = default set, entries +// are validated against symbol_scale::kTable at compile time). Named +// per-strategy because the sweep modules are imported side by side +// (loadCommand, tests) and two exported sweep::kSymbolGroupsOverride would +// collide. MIN_GAP_PIPS is pips (converted per symbol via symbol_scale), so +// any symbol set is coherent. The ranging crosses and copper (NZDUSD, +// USDCHF, EURGBP, EURCHF, EURNOK, AUDNZD, COPPERCMDUSD) produced zero +// winners across EVERY strategy in batch 2026-28 and are dropped from the +// sweep universe. +inline constexpr std::array kFvgSymbolGroupsOverride = + std::to_array({ + "AUDUSD", "EURUSD", "GBRIDXGBP", "GBPUSD", + "USDJPY", "GBPJPY", "EURJPY", "USDCAD", "FRAIDXEUR", + "USA500IDXUSD", "AUSIDXAUD", "XAUUSD", + "XAGUSD", "USATECHIDXUSD", "DEUIDXEUR", "USA30IDXUSD", + "LIGHTCMDUSD", "JPNIDXJPY", "BRENTCMDUSD", "EURAUD", + "HKGIDXHKD", "USDSEK"}); + +// Declares which parameters to sweep for the FvgStrategy. The OHLC COUNTS +// are NOT swept: makeFvgStrategy derives them at the ctor minimums +// (LOOKBACK_BARS + 3 and HTF_SMA_PERIOD + 2), so every combination is valid +// by construction — deeper windows add nothing because the scan depth is +// governed by LOOKBACK_BARS / MIN_GAP_AGE_BARS, not the window. Tune the +// values here; makeFvgStrategy reads back every name registered below. +ParameterGenerator buildFvgStrategySweep() { + ParameterGenerator generator; + generator.setSymbolGroups(); + // FVG timeframe: the closed 3-bar gap patterns. 5-minute was the weakest + // cell at every HTF in the 2026-28 winners — dropped. + generator.addList("FVG_OHLC_MINUTES", {15, 30}); + // HTF trend timeframe: SMA over its closed closes is the filter. Winners + // pinned at the old 240 top edge, so the grid extends to 480. Bars build + // cold from the window's first tick, so 480-minute bars x SMA 100 is + // already ~7 weeks of warm-up against the ladder's 3-month rungs — the + // reason HTF_SMA_PERIOD stops at 100 (200 would never warm on a rung). + generator.addList("HTF_OHLC_MINUTES", {120, 240, 480}); + // How far back the pattern scan may reach on the FVG timeframe. The + // 2026-28 winners were byte-identical across the old {10..30} range — + // non-binding — so it collapses to the two ends, kept as a pair to spot + // if the wider gaps below make depth bind again. + generator.addList("LOOKBACK_BARS", {10, 30}); + // Smallest tradeable gap in PIPS (converted to points per symbol by the + // strategy). Winners pinned at the old top edge (20) with the best + // scores in the family; 2 was dead weight. 35/50 are the new frontier. + generator.addList("MIN_GAP_PIPS", {5, 10, 20, 35, 50}); + // 50 beat 20 across the board in 2026-28; 100 probes slower (see the + // HTF_OHLC_MINUTES warm-up note for why 200 is excluded). + generator.addList("HTF_SMA_PERIOD", {50, 100}); + // Maximum pattern age despite the name (0 = newest pattern only). + generator.addList("MIN_GAP_AGE_BARS", {0, 2, 5}); + // Time cap on open trades, in minutes: during() closes a trade open + // strictly longer than this. No uncapped variant — every winner must + // carry an exit clock (trades were riding for 10+ hours live). + generator.addList("MAX_TRADE_DURATION_MINUTES", {30, 60, 120}); + // Exits are central (Operations enforces SL/TP). The values are ATR + // multipliers: conditions::check turns them into pip distances per entry + // (distance = ATR(10) x multiplier, clamped). + generator.addList("STOP_DISTANCE_IN_ATR", {1, 2}); + generator.addList("LIMIT_DISTANCE_IN_ATR", {3, 5, 7}); // 9 was flat vs 7 + return generator; +} + +} // namespace sweep diff --git a/source/load/config/fvgStrategy/makeFvgStrategy.cppm b/source/load/config/fvgStrategy/makeFvgStrategy.cppm new file mode 100644 index 0000000..17454e5 --- /dev/null +++ b/source/load/config/fvgStrategy/makeFvgStrategy.cppm @@ -0,0 +1,70 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module makeFvgStrategy; + +import std; +import sweepCombination; // sweep::Combination + +export namespace sweep { + +// Map one swept parameter combination onto an FvgStrategy config. Every +// parameter is read with getInt and no has() fallback: the sweep registers +// every name read here (buildFvgStrategySweep), so a missing one is a bug +// that should throw at load time, matching the strategy ctor's fail-fast +// validation on the run side. +tradingDefinitions::StrategyConfig makeFvgStrategy( + const sweep::Combination& combo) { + using namespace tradingDefinitions; + + const int lookbackBars = combo.getInt("LOOKBACK_BARS"); + const int htfSmaPeriod = combo.getInt("HTF_SMA_PERIOD"); + + return StrategyConfig{ + .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), + .TRADING_VARIABLES = TradingVariables{ + // Must match the dispatch string in strategyFactory. + .STRATEGY = "FvgStrategy", + .STOP_DISTANCE_IN_ATR = combo.getInt("STOP_DISTANCE_IN_ATR"), + .LIMIT_DISTANCE_IN_ATR = combo.getInt("LIMIT_DISTANCE_IN_ATR"), + .TRADING_SIZE = 1, + }, + // Positional contract with FvgStrategy: [0] is the FVG timeframe, + // [1] the HTF trend timeframe. The counts are DERIVED at the ctor + // minimums (LOOKBACK_BARS + 3 / HTF_SMA_PERIOD + 2) rather than + // swept, so every combination is valid by construction — the + // generator can't express cross-field constraints. + .OHLC_VARIABLES = { + OHLCVariables{ + .OHLC_COUNT = lookbackBars + 3, + .OHLC_MINUTES = combo.getInt("FVG_OHLC_MINUTES"), + }, + OHLCVariables{ + .OHLC_COUNT = htfSmaPeriod + 2, + .OHLC_MINUTES = combo.getInt("HTF_OHLC_MINUTES"), + }, + }, + .STRATEGY_VARIABLES = StrategyVariables{ + .FVG_STRATEGY_VARIABLES = FVGStrategyVariables{ + .LOOKBACK_BARS = lookbackBars, + .MIN_GAP_PIPS = combo.getInt("MIN_GAP_PIPS"), + .HTF_SMA_PERIOD = htfSmaPeriod, + .MIN_GAP_AGE_BARS = combo.getInt("MIN_GAP_AGE_BARS"), + .MAX_TRADE_DURATION_MINUTES = + combo.getInt("MAX_TRADE_DURATION_MINUTES"), + }, + }, + }; +} + +} // namespace sweep diff --git a/source/load/config/keltnerFadeStrategy/keltnerFadeStrategySweep.cppm b/source/load/config/keltnerFadeStrategy/keltnerFadeStrategySweep.cppm new file mode 100644 index 0000000..33c73a4 --- /dev/null +++ b/source/load/config/keltnerFadeStrategy/keltnerFadeStrategySweep.cppm @@ -0,0 +1,59 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module keltnerFadeStrategySweep; + +export import parameterGenerator; // buildKeltnerFadeStrategySweep() returns sweep::ParameterGenerator + +import std; // replaces , + +export namespace sweep { + +// The symbol groups THIS sweep runs against (see kSymbolGroupsOverride in +// randomStrategySweep for the full semantics: empty = default set, entries +// are validated against symbol_scale::kTable at compile time). Named +// per-strategy because the sweep modules are imported side by side +// (loadCommand, tests) and two exported sweep::kSymbolGroupsOverride would +// collide. Mean reversion targets the ranging crosses; EURUSD rides along as +// the trending-major control the hypothesis should do WORSE on. +inline constexpr std::array kKeltnerFadeSymbolGroupsOverride = + std::to_array({ + "EURGBP", "EURCHF", "AUDNZD", "USDCHF", "EURNOK", "USDSEK", "EURUSD"}); + +// Declares which parameters to sweep for the KeltnerFadeStrategy. The OHLC +// COUNT is NOT swept: makeKeltnerFadeStrategy derives it at the ctor minimum +// (BAND_SMA_PERIOD + 2), so every combination is valid by construction — a +// deeper window adds nothing because the band only reads BAND_SMA_PERIOD + 1 +// closed bars. Tune the values here; makeKeltnerFadeStrategy reads back every +// name registered below. +ParameterGenerator buildKeltnerFadeStrategySweep() { + ParameterGenerator generator; + generator.setSymbolGroups(); + // Signal timeframe: band centre + width both live on it. + generator.addList("OHLC_MINUTES", {5, 15, 30}); + // Lookback for the band centre (SMA) and width (ATR over the same bars). + generator.addList("BAND_SMA_PERIOD", {20, 50}); + // Band half-width in TENTHS of an ATR (15 = 1.5x): how stretched price + // must be before it is faded. 40 probes the extreme-stretch tail — the + // regime where the reversion thesis is strongest and the grid had no + // coverage. + generator.addList("BAND_ATR_MULT_TENTHS", {15, 20, 25, 30, 40}); + // Time cap on open trades, in minutes: during() closes a trade open + // strictly longer than this — the fade's thesis clock (a stretch that + // has not snapped back in time is a failed reversion). No uncapped + // variant — every winner must carry an exit clock. + generator.addList("MAX_TRADE_DURATION_MINUTES", {30, 60, 120}); + // Exits are central (Operations enforces SL/TP). The values are ATR + // multipliers: conditions::check turns them into pip distances per entry + // (distance = ATR(10) x multiplier, clamped). Reversion inverts the trend + // strategies' shape — the limit sits NEARER than the stop (take the + // snap-back, survive the excursion). + generator.addList("STOP_DISTANCE_IN_ATR", {2, 3}); + generator.addList("LIMIT_DISTANCE_IN_ATR", {1, 2, 3}); + return generator; +} + +} // namespace sweep diff --git a/source/load/config/keltnerFadeStrategy/makeKeltnerFadeStrategy.cppm b/source/load/config/keltnerFadeStrategy/makeKeltnerFadeStrategy.cppm new file mode 100644 index 0000000..de38600 --- /dev/null +++ b/source/load/config/keltnerFadeStrategy/makeKeltnerFadeStrategy.cppm @@ -0,0 +1,63 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module makeKeltnerFadeStrategy; + +import std; +import sweepCombination; // sweep::Combination + +export namespace sweep { + +// Map one swept parameter combination onto a KeltnerFadeStrategy config. +// Every parameter is read with getInt and no has() fallback: the sweep +// registers every name read here (buildKeltnerFadeStrategySweep), so a +// missing one is a bug that should throw at load time, matching the strategy +// ctor's fail-fast validation on the run side. +tradingDefinitions::StrategyConfig makeKeltnerFadeStrategy( + const sweep::Combination& combo) { + using namespace tradingDefinitions; + + const int bandSmaPeriod = combo.getInt("BAND_SMA_PERIOD"); + + return StrategyConfig{ + .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), + .TRADING_VARIABLES = TradingVariables{ + // Must match the dispatch string in strategyFactory. + .STRATEGY = "KeltnerFadeStrategy", + .STOP_DISTANCE_IN_ATR = combo.getInt("STOP_DISTANCE_IN_ATR"), + .LIMIT_DISTANCE_IN_ATR = combo.getInt("LIMIT_DISTANCE_IN_ATR"), + .TRADING_SIZE = 1, + }, + // Positional contract with KeltnerFadeStrategy: [0] is the signal + // timeframe (it also drives the ATR entry gate). The count is DERIVED + // at the ctor minimum (BAND_SMA_PERIOD + 2) rather than swept, so + // every combination is valid by construction — the generator can't + // express cross-field constraints. + .OHLC_VARIABLES = { + OHLCVariables{ + .OHLC_COUNT = bandSmaPeriod + 2, + .OHLC_MINUTES = combo.getInt("OHLC_MINUTES"), + }, + }, + .STRATEGY_VARIABLES = StrategyVariables{ + .KELTNER_FADE_VARIABLES = KeltnerFadeVariables{ + .BAND_SMA_PERIOD = bandSmaPeriod, + .BAND_ATR_MULT_TENTHS = combo.getInt("BAND_ATR_MULT_TENTHS"), + .MAX_TRADE_DURATION_MINUTES = + combo.getInt("MAX_TRADE_DURATION_MINUTES"), + }, + }, + }; +} + +} // namespace sweep diff --git a/source/load/config/liquiditySweepReversalStrategy/liquiditySweepReversalStrategySweep.cppm b/source/load/config/liquiditySweepReversalStrategy/liquiditySweepReversalStrategySweep.cppm new file mode 100644 index 0000000..136f8f1 --- /dev/null +++ b/source/load/config/liquiditySweepReversalStrategy/liquiditySweepReversalStrategySweep.cppm @@ -0,0 +1,81 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module liquiditySweepReversalStrategySweep; + +export import parameterGenerator; // buildLiquiditySweepReversalStrategySweep() returns sweep::ParameterGenerator + +import std; // replaces , + +export namespace sweep { + +// The symbol groups THIS sweep runs against (see kSymbolGroupsOverride in +// randomStrategySweep for the full semantics: empty = default set, entries +// are validated against symbol_scale::kTable at compile time). Named +// per-strategy because the sweep modules are imported side by side +// (loadCommand, tests) and two exported sweep::kSymbolGroupsOverride would +// collide. The 2026-28 full-universe run let the results decide (the +// KeltnerFade lesson): the highest average score of any family, but the +// ranging crosses and copper (NZDUSD, USDCHF, EURGBP, EURCHF, EURNOK, +// AUDNZD, COPPERCMDUSD) produced zero winners across EVERY strategy and +// are dropped from the sweep universe. +inline constexpr std::array kLiquiditySweepReversalSymbolGroupsOverride = + std::to_array({ + "AUDUSD", "EURUSD", "GBRIDXGBP", "GBPUSD", + "USDJPY", "GBPJPY", "EURJPY", "USDCAD", "FRAIDXEUR", + "USA500IDXUSD", "AUSIDXAUD", "XAUUSD", + "XAGUSD", "USATECHIDXUSD", "DEUIDXEUR", "USA30IDXUSD", + "LIGHTCMDUSD", "JPNIDXJPY", "BRENTCMDUSD", "EURAUD", + "HKGIDXHKD", "USDSEK"}); + +// Declares which parameters to sweep for the LiquiditySweepReversalStrategy. +// The OHLC COUNT is NOT swept: makeLiquiditySweepReversalStrategy derives it +// at the ctor minimum (max(LOOKBACK_BARS + PIVOT_BARS + 1, VALID_BARS + 11)), +// so every combination is valid by construction — a deeper window adds +// nothing because the scan depth is governed by LOOKBACK_BARS. Tune the +// values here; makeLiquiditySweepReversalStrategy reads back every name +// registered below. +ParameterGenerator buildLiquiditySweepReversalStrategySweep() { + ParameterGenerator generator; + generator.setSymbolGroups(); + // Signal timeframe the pivots, sweeps and rejections form on. + generator.addList("OHLC_MINUTES", {15, 30, 60}); + // Fractal wing: closed bars strictly beaten on each side of a pivot. + generator.addList("PIVOT_BARS", {2, 3, 5}); + // How far back pivots are scanned, in closed bars. 96 probes whether + // older levels still attract sweeps (widened from the pinned 48). + generator.addList("LOOKBACK_BARS", {48, 96}); + // Minimum wick excursion beyond the level for a sweep, in pips + // (0 = any strict poke). 2 tracked 0 in 2026-28; 5 scored best, so the + // grid slides up to probe 10. + generator.addList("MIN_SWEEP_PIPS", {0, 5, 10}); + // Rejection body demanded, in tenths of ATR(10) — 0 disables the gate. + // The 2026-28 A/B answered the deferred OB/breaker question: 15 (the + // high dose) produced ZERO winners from ~42k runs, while 5 held the + // family's best averages — displacement carries signal at a moderate + // dose and dies at high dose. 3/7 bracket the peak. + generator.addList("DISPLACEMENT_ATR_TENTHS", {0, 3, 5, 7}); + // How many closed bars a rejection stays tradeable for. Winners pinned + // at the old top edge (4) with the book's best cell average — 6/8 are + // the new frontier; 1 was weakest and is dropped. + generator.addList("VALID_BARS", {2, 4, 6, 8}); + // Time cap on open trades, in minutes: during() closes a trade open + // strictly longer than this. No uncapped variant — every winner must + // carry an exit clock (trades were riding for 10+ hours live). + generator.addList("MAX_TRADE_DURATION_MINUTES", {30, 60, 120}); + // Exits are central (Operations enforces SL/TP). The values are ATR + // multipliers: conditions::check turns them into pip distances per entry + // (distance = ATR(10) x multiplier, clamped). The original grid followed + // the KeltnerFade "limit NEARER than stop" reversal doctrine — the + // 2026-28 winners refuted it: scores rose monotonically toward the + // limit=3 edge and the best cell was limit == stop, so the limit grid + // now reaches PAST the stop (4/5 are the new frontier). + generator.addList("STOP_DISTANCE_IN_ATR", {2, 3}); + generator.addList("LIMIT_DISTANCE_IN_ATR", {2, 3, 4, 5}); + return generator; +} + +} // namespace sweep diff --git a/source/load/config/liquiditySweepReversalStrategy/makeLiquiditySweepReversalStrategy.cppm b/source/load/config/liquiditySweepReversalStrategy/makeLiquiditySweepReversalStrategy.cppm new file mode 100644 index 0000000..985da74 --- /dev/null +++ b/source/load/config/liquiditySweepReversalStrategy/makeLiquiditySweepReversalStrategy.cppm @@ -0,0 +1,74 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module makeLiquiditySweepReversalStrategy; + +import std; +import sweepCombination; // sweep::Combination + +export namespace sweep { + +// Map one swept parameter combination onto a LiquiditySweepReversalStrategy +// config. Every parameter is read with getInt and no has() fallback: the +// sweep registers every name read here +// (buildLiquiditySweepReversalStrategySweep), so a missing one is a bug that +// should throw at load time, matching the strategy ctor's fail-fast +// validation on the run side. +tradingDefinitions::StrategyConfig makeLiquiditySweepReversalStrategy( + const sweep::Combination& combo) { + using namespace tradingDefinitions; + + const int pivotBars = combo.getInt("PIVOT_BARS"); + const int lookbackBars = combo.getInt("LOOKBACK_BARS"); + const int validBars = combo.getInt("VALID_BARS"); + + // The window must cover the pivot scan (lookback + a left wing beyond its + // oldest candidate + the in-progress bar) and keep the displacement + // ATR(10) warm at the oldest valid rejection — the ctor minimum. Derived + // rather than swept, so every combination is valid by construction — the + // generator can't express cross-field constraints. + const int ohlcCount = + std::max(lookbackBars + pivotBars + 1, validBars + 11); + + return StrategyConfig{ + .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), + .TRADING_VARIABLES = TradingVariables{ + // Must match the dispatch string in strategyFactory. + .STRATEGY = "LiquiditySweepReversalStrategy", + .STOP_DISTANCE_IN_ATR = combo.getInt("STOP_DISTANCE_IN_ATR"), + .LIMIT_DISTANCE_IN_ATR = combo.getInt("LIMIT_DISTANCE_IN_ATR"), + .TRADING_SIZE = 1, + }, + // Positional contract with LiquiditySweepReversalStrategy: [0] is the + // signal timeframe (it also drives the ATR entry gate). + .OHLC_VARIABLES = { + OHLCVariables{ + .OHLC_COUNT = ohlcCount, + .OHLC_MINUTES = combo.getInt("OHLC_MINUTES"), + }, + }, + .STRATEGY_VARIABLES = StrategyVariables{ + .LIQUIDITY_SWEEP_REVERSAL_VARIABLES = LiquiditySweepReversalVariables{ + .PIVOT_BARS = pivotBars, + .LOOKBACK_BARS = lookbackBars, + .MIN_SWEEP_PIPS = combo.getInt("MIN_SWEEP_PIPS"), + .DISPLACEMENT_ATR_TENTHS = combo.getInt("DISPLACEMENT_ATR_TENTHS"), + .VALID_BARS = validBars, + .MAX_TRADE_DURATION_MINUTES = + combo.getInt("MAX_TRADE_DURATION_MINUTES"), + }, + }, + }; +} + +} // namespace sweep diff --git a/source/load/config/nyOpenRangeBreakoutStrategy/makeNyOpenRangeBreakoutStrategy.cppm b/source/load/config/nyOpenRangeBreakoutStrategy/makeNyOpenRangeBreakoutStrategy.cppm new file mode 100644 index 0000000..9d85bbc --- /dev/null +++ b/source/load/config/nyOpenRangeBreakoutStrategy/makeNyOpenRangeBreakoutStrategy.cppm @@ -0,0 +1,74 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module makeNyOpenRangeBreakoutStrategy; + +import std; +import sweepCombination; // sweep::Combination + +export namespace sweep { + +// Map one swept parameter combination onto a NyOpenRangeBreakoutStrategy +// config. Every parameter is read with getInt and no has() fallback: the +// sweep registers every name read here +// (buildNyOpenRangeBreakoutStrategySweep), so a missing one is a bug that +// should throw at load time, matching the strategy ctor's fail-fast +// validation on the run side. +tradingDefinitions::StrategyConfig makeNyOpenRangeBreakoutStrategy( + const sweep::Combination& combo) { + using namespace tradingDefinitions; + + const int ohlcMinutes = combo.getInt("OHLC_MINUTES"); + const int rangeHours = combo.getInt("RANGE_HOURS"); + const int entryWindowMinutes = combo.getInt("ENTRY_WINDOW_MINUTES"); + + // The window must span the range start through the entry cutoff, with + // the ctor's two-bar margin: ceil((RANGE_HOURS x 60 + window) / minutes) + // + 2 bars. Derived rather than swept, so every combination is valid by + // construction — the generator can't express cross-field constraints. + // Unlike the London strategy the requirement is anchored to the open, + // not midnight, so the DST regime never enters the formula. + const int ohlcCount = + (rangeHours * 60 + entryWindowMinutes + ohlcMinutes - 1) / ohlcMinutes + + 2; + + return StrategyConfig{ + .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), + .TRADING_VARIABLES = TradingVariables{ + // Must match the dispatch string in strategyFactory. + .STRATEGY = "NyOpenRangeBreakoutStrategy", + .STOP_DISTANCE_IN_ATR = combo.getInt("STOP_DISTANCE_IN_ATR"), + .LIMIT_DISTANCE_IN_ATR = combo.getInt("LIMIT_DISTANCE_IN_ATR"), + .TRADING_SIZE = 1, + }, + // Positional contract with NyOpenRangeBreakoutStrategy: [0] is the + // signal timeframe (it also drives the ATR entry gate). + .OHLC_VARIABLES = { + OHLCVariables{ + .OHLC_COUNT = ohlcCount, + .OHLC_MINUTES = ohlcMinutes, + }, + }, + .STRATEGY_VARIABLES = StrategyVariables{ + .NY_OPEN_RANGE_BREAKOUT_VARIABLES = NyOpenRangeBreakoutVariables{ + .RANGE_HOURS = rangeHours, + .BUFFER_PIPS = combo.getInt("BUFFER_PIPS"), + .ENTRY_WINDOW_MINUTES = entryWindowMinutes, + .MAX_TRADE_DURATION_MINUTES = + combo.getInt("MAX_TRADE_DURATION_MINUTES"), + }, + }, + }; +} + +} // namespace sweep diff --git a/source/load/config/nyOpenRangeBreakoutStrategy/nyOpenRangeBreakoutStrategySweep.cppm b/source/load/config/nyOpenRangeBreakoutStrategy/nyOpenRangeBreakoutStrategySweep.cppm new file mode 100644 index 0000000..e2e582f --- /dev/null +++ b/source/load/config/nyOpenRangeBreakoutStrategy/nyOpenRangeBreakoutStrategySweep.cppm @@ -0,0 +1,70 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module nyOpenRangeBreakoutStrategySweep; + +export import parameterGenerator; // buildNyOpenRangeBreakoutStrategySweep() returns sweep::ParameterGenerator + +import std; // replaces , + +export namespace sweep { + +// The symbol groups THIS sweep runs against (see kSymbolGroupsOverride in +// randomStrategySweep for the full semantics: empty = default set, entries +// are validated against symbol_scale::kTable at compile time). Named +// per-strategy because the sweep modules are imported side by side +// (loadCommand, tests) and two exported sweep::kSymbolGroupsOverride would +// collide. NewYork-session symbols only (market_hours' mapping): the +// strategy trades the NY open, and with PEAK_HOURS_ONLY on the run loop's +// entry gate only opens for these symbols during exactly that window. +// PROBE-OR-RETIRE grid since 2026-28: 56 winners from ~23k runs and a best +// score of 32 that barely clears live selection. This minimal grid keeps +// only the three symbols that produced any winner at all (USA30, LIGHT and +// USDCAD produced none) and the parameter cells the winners actually used — +// if the next batches stay this thin, retire it like KeltnerFade. +inline constexpr std::array kNyOpenRangeBreakoutSymbolGroupsOverride = + std::to_array({ + "USA500IDXUSD", "USATECHIDXUSD", "XAUUSD"}); + +// Declares which parameters to sweep for the NyOpenRangeBreakoutStrategy. +// The OHLC COUNT is NOT swept: makeNyOpenRangeBreakoutStrategy derives it +// from RANGE_HOURS, OHLC_MINUTES and ENTRY_WINDOW_MINUTES at the ctor +// minimum (span range start -> entry cutoff), so every combination is valid +// by construction — a deeper window adds nothing because bars are selected +// by date. Tune the values here; makeNyOpenRangeBreakoutStrategy reads back +// every name registered below. +ParameterGenerator buildNyOpenRangeBreakoutStrategySweep() { + ParameterGenerator generator; + generator.setSymbolGroups(); + // Signal timeframe the pre-open range is built from. 30 dropped in the + // probe grid. + generator.addList("OHLC_MINUTES", {5, 15}); + // Range depth in hours back from the open: 4 = tight pre-open coil, + // 13 = the whole overnight session (the ctor caps at 13 so the range + // never reaches past the previous UTC midnight). + generator.addList("RANGE_HOURS", {4, 8, 13}); + // Padding on the pre-open high/low, in pips (0 = raw range). + generator.addList("BUFFER_PIPS", {0, 2}); + // How long after the NY open entries may fire. The long windows carried + // no winners in 2026-28 — the probe keeps the first hour. + generator.addList("ENTRY_WINDOW_MINUTES", {30, 60}); + // Time cap on open trades, in minutes: during() closes a trade open + // strictly longer than this. No uncapped variant — every winner must + // carry an exit clock (an uncapped live winner rode a trade for hours). + // The trend family's winners pinned at the old 120 edge, so the probe + // keeps 120 and tries 240. + generator.addList("MAX_TRADE_DURATION_MINUTES", {120, 240}); + // Exits are central (Operations enforces SL/TP). The values are ATR + // multipliers: conditions::check turns them into pip distances per entry + // (distance = ATR(10) x multiplier, clamped). The wide-stop rationale + // failed its A/B (50 of the 56 winners sat at stop 1) — the probe pins + // stop 1, single-value so the mapper's getInt contract holds. + generator.addList("STOP_DISTANCE_IN_ATR", {1}); + generator.addList("LIMIT_DISTANCE_IN_ATR", {3, 5}); + return generator; +} + +} // namespace sweep diff --git a/source/load/config/ohlcBreakoutStrategy/makeOhlcBreakoutStrategy.cppm b/source/load/config/ohlcBreakoutStrategy/makeOhlcBreakoutStrategy.cppm index 3947f1d..dad0d61 100644 --- a/source/load/config/ohlcBreakoutStrategy/makeOhlcBreakoutStrategy.cppm +++ b/source/load/config/ohlcBreakoutStrategy/makeOhlcBreakoutStrategy.cppm @@ -20,9 +20,9 @@ export namespace sweep { // Map one swept parameter combination onto an OhlcBreakoutStrategy config. // Every parameter is read with getInt and no has() fallback: the sweep -// registers all seven names (buildOhlcBreakoutStrategySweep), so a missing one -// is a bug that should throw at load time, matching the strategy ctor's -// fail-fast validation on the run side. +// registers every name read here (buildOhlcBreakoutStrategySweep), so a +// missing one is a bug that should throw at load time, matching the strategy +// ctor's fail-fast validation on the run side. tradingDefinitions::StrategyConfig makeOhlcBreakoutStrategy( const sweep::Combination& combo) { using namespace tradingDefinitions; @@ -32,8 +32,8 @@ tradingDefinitions::StrategyConfig makeOhlcBreakoutStrategy( .TRADING_VARIABLES = TradingVariables{ // Must match the dispatch string in run/operations.cppm. .STRATEGY = "OhlcBreakoutStrategy", - .STOP_DISTANCE_IN_PIPS = combo.getInt("STOP_DISTANCE_IN_PIPS"), - .LIMIT_DISTANCE_IN_PIPS = combo.getInt("LIMIT_DISTANCE_IN_PIPS"), + .STOP_DISTANCE_IN_ATR = combo.getInt("STOP_DISTANCE_IN_ATR"), + .LIMIT_DISTANCE_IN_ATR = combo.getInt("LIMIT_DISTANCE_IN_ATR"), .TRADING_SIZE = 1, }, // Positional contract with OhlcBreakoutStrategy: [0] is the breakout @@ -51,6 +51,8 @@ tradingDefinitions::StrategyConfig makeOhlcBreakoutStrategy( .STRATEGY_VARIABLES = StrategyVariables{ .OHLC_BREAKOUT_VARIABLES = OHLCBreakoutVariables{ .BUFFER_PIPS = combo.getInt("BUFFER_PIPS"), + .MAX_TRADE_DURATION_MINUTES = + combo.getInt("MAX_TRADE_DURATION_MINUTES"), }, }, }; diff --git a/source/load/config/ohlcBreakoutStrategy/ohlcBreakoutStrategySweep.cppm b/source/load/config/ohlcBreakoutStrategy/ohlcBreakoutStrategySweep.cppm index 55751e3..4ecbc8d 100644 --- a/source/load/config/ohlcBreakoutStrategy/ohlcBreakoutStrategySweep.cppm +++ b/source/load/config/ohlcBreakoutStrategy/ohlcBreakoutStrategySweep.cppm @@ -17,30 +17,52 @@ export namespace sweep { // are validated against symbol_scale::kTable at compile time). Named // per-strategy because the sweep modules are imported side by side // (loadCommand, tests) and two exported sweep::kSymbolGroupsOverride would -// collide. +// collide. BUFFER_PIPS already converts per symbol via symbol_scale, so any +// symbol set is coherent. The ranging crosses and copper (NZDUSD, USDCHF, +// EURGBP, EURCHF, EURNOK, AUDNZD, COPPERCMDUSD) produced zero winners +// across EVERY strategy in batch 2026-28 and are dropped from the sweep +// universe. inline constexpr std::array kOhlcBreakoutSymbolGroupsOverride = - std::to_array({"EURUSD"}); + std::to_array({ + "AUDUSD", "EURUSD", "GBRIDXGBP", "GBPUSD", + "USDJPY", "GBPJPY", "EURJPY", "USDCAD", "FRAIDXEUR", + "USA500IDXUSD", "AUSIDXAUD", "XAUUSD", + "XAGUSD", "USATECHIDXUSD", "DEUIDXEUR", "USA30IDXUSD", + "LIGHTCMDUSD", "JPNIDXJPY", "BRENTCMDUSD", "EURAUD", + "HKGIDXHKD", "USDSEK"}); -// Declares which parameters to sweep for the OhlcBreakoutStrategy — all seven. -// Seven dimensions multiply fast: this set expands to 40^4 x 6 x 12 x 12 -// ≈ 2.2 BILLION combinations per run (loadCommand shows the exact count and -// asks for confirmation before queueing). Tune the values here; +// Declares which parameters to sweep for the OhlcBreakoutStrategy. The +// dimensions multiply fast (loadCommand shows the exact count and asks for +// confirmation before queueing); the grid is budgeted so combos x the +// symbol groups stays well under a million queued runs. Tune the values here; // makeOhlcBreakoutStrategy reads back every name registered below, and the // strategy ctor requires OHLC_COUNT >= 2 and OHLC_MINUTES >= 1. ParameterGenerator buildOhlcBreakoutStrategySweep() { ParameterGenerator generator; generator.setSymbolGroups(); - // Breakout timeframe: the closed-candle range price must clear. - generator.addRange("BREAKOUT_OHLC_MINUTES", 20, 20, 120); - generator.addRange("BREAKOUT_OHLC_COUNT", 20, 20, 120); + // Breakout timeframe: the closed-candle range price must clear. Log-ish + // spacing over the old 5..50 linear span — adjacent 5-step values were + // near-duplicate strategies. + generator.addList("BREAKOUT_OHLC_MINUTES", {5, 10, 15, 30, 50}); + generator.addList("BREAKOUT_OHLC_COUNT", {5, 10, 20, 35, 50}); // Trend timeframe: EMA over its closes (period = count / 2) is the filter. - generator.addRange("TREND_OHLC_MINUTES", 20, 20, 120); - generator.addRange("TREND_OHLC_COUNT", 20, 20, 120); - // Padding on the breakout levels, in pips (0 = raw range). - generator.addRange("BUFFER_PIPS", 0, 2, 10); - // Exits are central (Operations enforces SL/TP), so the distances sweep too. - generator.addRange("STOP_DISTANCE_IN_PIPS", 10, 20, 110); - generator.addRange("LIMIT_DISTANCE_IN_PIPS", 10, 20, 110); + // Starts at 40 — a 20-minute "trend" sits at/below the breakout + // timeframes; the counts give EMA periods 10/30/60, the old span. + generator.addList("TREND_OHLC_MINUTES", {40, 80, 120}); + generator.addList("TREND_OHLC_COUNT", {20, 60, 120}); + // Padding on the breakout levels, in pips (0 = raw range). 3 was flat + // against its neighbours in 2026-28 — the A/B keeps only the ends. + generator.addList("BUFFER_PIPS", {0, 6}); + // Time cap on open trades, in minutes: during() closes a trade open + // strictly longer than this. No uncapped variant — every winner must + // carry an exit clock. 120 was the pinned edge in 2026-28 (30 clearly + // the worst) — the cap slides up a notch to probe 240. + generator.addList("MAX_TRADE_DURATION_MINUTES", {60, 120, 240}); + // Exits are central (Operations enforces SL/TP). The values are ATR + // multipliers: conditions::check turns them into pip distances per entry + // (distance = ATR(10) x multiplier, clamped). + generator.addList("STOP_DISTANCE_IN_ATR", {2, 3}); + generator.addList("LIMIT_DISTANCE_IN_ATR", {2, 4, 6}); return generator; } diff --git a/source/load/config/randomStrategy/makeStrategy.cppm b/source/load/config/randomStrategy/makeStrategy.cppm index 73dc64e..88bb835 100644 --- a/source/load/config/randomStrategy/makeStrategy.cppm +++ b/source/load/config/randomStrategy/makeStrategy.cppm @@ -28,8 +28,8 @@ tradingDefinitions::StrategyConfig makeStrategy(const sweep::Combination& combo) .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), .TRADING_VARIABLES = TradingVariables{ .STRATEGY = "RandomStrategy", - .STOP_DISTANCE_IN_PIPS = combo.getInt("STOP_DISTANCE_IN_PIPS"), - .LIMIT_DISTANCE_IN_PIPS = combo.getInt("LIMIT_DISTANCE_IN_PIPS"), + .STOP_DISTANCE_IN_ATR = combo.getInt("STOP_DISTANCE_IN_ATR"), + .LIMIT_DISTANCE_IN_ATR = combo.getInt("LIMIT_DISTANCE_IN_ATR"), .TRADING_SIZE = 1, }, .OHLC_VARIABLES = { diff --git a/source/load/config/randomStrategy/randomStrategySweep.cppm b/source/load/config/randomStrategy/randomStrategySweep.cppm index 4c982c9..a60be37 100644 --- a/source/load/config/randomStrategy/randomStrategySweep.cppm +++ b/source/load/config/randomStrategy/randomStrategySweep.cppm @@ -21,7 +21,13 @@ export namespace sweep { // compile time, so a typo here fails the build instead of queueing a run the // worker rejects at runtime. inline constexpr std::array kSymbolGroupsOverride = - std::to_array({"EURUSD"}); + std::to_array({ + "AUDUSD", "EURUSD", "GBRIDXGBP", "GBPUSD", "NZDUSD", + "USDJPY", "GBPJPY", "EURJPY", "USDCAD", "FRAIDXEUR", + "EURGBP", "USA500IDXUSD", "AUSIDXAUD", "USDCHF", "XAUUSD", + "XAGUSD", "USATECHIDXUSD", "EURCHF", "DEUIDXEUR", "USA30IDXUSD", + "LIGHTCMDUSD", "JPNIDXJPY", "BRENTCMDUSD", "AUDNZD", "EURAUD", + "HKGIDXHKD", "COPPERCMDUSD", "USDSEK", "EURNOK"}); // Declares which parameters to sweep for the RandomStrategy. Keeping the ranges // in one place means a new strategy (or extra swept parameter) is a localised @@ -31,8 +37,10 @@ ParameterGenerator buildRandomStrategySweep() { generator.setSymbolGroups(); // generator.addRange("OHLC_COUNT", 80, 20, 140); // 80, 100, 120, 140 // generator.addList("OHLC_MINUTES", {1, 3, 5, 8}); - generator.addRange("LIMIT_DISTANCE_IN_PIPS", 1, 1, 100); - generator.addRange("STOP_DISTANCE_IN_PIPS", 1, 1, 100); + // ATR multipliers, not pips: conditions::check turns them into pip + // distances per entry (distance = ATR(10) x multiplier, clamped). + generator.addList("STOP_DISTANCE_IN_ATR", {1, 2}); + generator.addRange("LIMIT_DISTANCE_IN_ATR", 3, 2, 9); // 3, 5, 7, 9 return generator; } diff --git a/source/load/config/rangeVelocityStrategy/makeRangeVelocityStrategy.cppm b/source/load/config/rangeVelocityStrategy/makeRangeVelocityStrategy.cppm new file mode 100644 index 0000000..32d50c1 --- /dev/null +++ b/source/load/config/rangeVelocityStrategy/makeRangeVelocityStrategy.cppm @@ -0,0 +1,79 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module makeRangeVelocityStrategy; + +import std; +import sweepCombination; // sweep::Combination + +export namespace sweep { + +// Map one swept parameter combination onto a RangeVelocityStrategy config. +// Every parameter is read with getInt and no has() fallback: the sweep +// registers every name read here (buildRangeVelocityStrategySweep), so a +// missing one is a bug that should throw at load time, matching the strategy +// ctor's fail-fast validation on the run side. +tradingDefinitions::StrategyConfig makeRangeVelocityStrategy( + const sweep::Combination& combo) { + using namespace tradingDefinitions; + + const int runBars = combo.getInt("RUN_BARS"); + const int speedLookbackBars = combo.getInt("SPEED_LOOKBACK_BARS"); + const int exitRunBars = combo.getInt("EXIT_RUN_BARS"); + + // The bar window must cover the run + speed baseline scan and the + // during() opposite-run scan — the ctor minimum, +2 margin (the count + // also drives the live QuestDB warm-up depth, and reading from the end + // makes extra depth harmless). Derived rather than swept, so every + // combination is valid by construction — the generator can't express + // cross-field constraints (the OHLC_COUNT doctrine). + const int rangeCount = + std::max(runBars + speedLookbackBars, exitRunBars) + 2; + + return StrategyConfig{ + .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), + .TRADING_VARIABLES = TradingVariables{ + // Must match the dispatch string in strategyFactory. + .STRATEGY = "RangeVelocityStrategy", + .STOP_DISTANCE_IN_ATR = combo.getInt("STOP_DISTANCE_IN_ATR"), + .LIMIT_DISTANCE_IN_ATR = combo.getInt("LIMIT_DISTANCE_IN_ATR"), + .TRADING_SIZE = 1, + }, + // Deliberately no OHLC series: the strategy trades range bars only, + // and the ATR entry gate falls back to its default 15m series for + // stop/limit sizing (entryConditions::gateSeriesFor, the + // RandomStrategy path). + .OHLC_VARIABLES = {}, + // Positional contract with RangeVelocityStrategy: [0] is THE series + // it trades. + .RANGE_VARIABLES = { + RangeBarVariables{ + .RANGE_ATR_TICK_WINDOW = combo.getInt("RANGE_ATR_TICK_WINDOW"), + .RANGE_ATR_PERCENT = combo.getInt("RANGE_ATR_PERCENT"), + .RANGE_COUNT = rangeCount, + }, + }, + .STRATEGY_VARIABLES = StrategyVariables{ + .RANGE_VELOCITY_VARIABLES = RangeVelocityVariables{ + .RUN_BARS = runBars, + .SPEED_LOOKBACK_BARS = speedLookbackBars, + .SPEED_RATIO_PERCENT = combo.getInt("SPEED_RATIO_PERCENT"), + .EXIT_RUN_BARS = exitRunBars, + .MAX_TRADE_DURATION_MINUTES = + combo.getInt("MAX_TRADE_DURATION_MINUTES"), + }, + }, + }; +} + +} // namespace sweep diff --git a/source/load/config/rangeVelocityStrategy/rangeVelocityStrategySweep.cppm b/source/load/config/rangeVelocityStrategy/rangeVelocityStrategySweep.cppm new file mode 100644 index 0000000..e165d9d --- /dev/null +++ b/source/load/config/rangeVelocityStrategy/rangeVelocityStrategySweep.cppm @@ -0,0 +1,81 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module rangeVelocityStrategySweep; + +export import parameterGenerator; // buildRangeVelocityStrategySweep() returns sweep::ParameterGenerator + +import std; // replaces , + +export namespace sweep { + +// The symbol groups THIS sweep runs against (see kSymbolGroupsOverride in +// randomStrategySweep for the full semantics; entries are validated against +// symbol_scale::kTable at compile time). Named per-strategy because the +// sweep modules are imported side by side. Deliberately narrower than the +// full universe: formation-speed needs a DENSE tick stream to measure +// anything (a sparse feed makes every bar look slow). Indices and metals +// only: the 2026-28 batch answered the wider question — the FX-major +// control group returned near-zero (EURUSD literally zero), and both oils +// and copper returned zero winners from ~430k runs — so the sweep keeps +// the set that pays. +inline constexpr std::array kRangeVelocitySymbolGroupsOverride = + std::to_array({ + "USA500IDXUSD", "USATECHIDXUSD", "USA30IDXUSD", "DEUIDXEUR", + "FRAIDXEUR", "GBRIDXGBP", "JPNIDXJPY", "AUSIDXAUD", + "HKGIDXHKD", "XAUUSD", "XAGUSD"}); + +// Declares which parameters to sweep for the RangeVelocityStrategy. The +// RANGE_COUNT is NOT swept: makeRangeVelocityStrategy derives it at the ctor +// minimum + margin (max(RUN_BARS + SPEED_LOOKBACK_BARS, EXIT_RUN_BARS) + 2), +// so every combination is valid by construction. SPEED_LOOKBACK_BARS is +// registered single-value (the TREND_OHLC_COUNT precedent) so the mapper's +// no-fallback getInt contract holds and widening it later is a one-line +// edit. Tune the values here; makeRangeVelocityStrategy reads back every +// name registered below. +ParameterGenerator buildRangeVelocityStrategySweep() { + ParameterGenerator generator; + generator.setSymbolGroups(); + // The rolling tick window the bar threshold derives from — effectively + // a bars-per-hour knob on a dense feed. + generator.addList("RANGE_ATR_TICK_WINDOW", {500, 1000, 2500, 5000, 7000}); + // Bar size as a share of the rolling window range: smaller percent = + // smaller bars = more of them per move. 10 ran at triple the family's + // runs-per-winner in 2026-28 — dropped. + generator.addList("RANGE_ATR_PERCENT", {25, 40, 60}); + // K: consecutive same-direction closed bars demanded. 10 was dead in + // 2026-28 (62 winners from ~590k runs) — dropped. + generator.addList("RUN_BARS", {3, 5, 8}); + // M: baseline bars whose median formation time is the speed norm. + generator.addList("SPEED_LOOKBACK_BARS", {10, 20, 32, 40}); + // Bar passes when duration x 100 <= median x this — 100 is the + // "at the norm" control. The 2026-28 A/B answered against the filter: + // 100 beat 80 beat 60 on BOTH winner count and score, so the hard + // 60-demand is dropped; 80 stays as the last live dose. If 100 keeps + // winning, the speed gate itself is the next thing to question. + generator.addList("SPEED_RATIO_PERCENT", {80, 100}); + // E: closed bars against the position that close it from during(). + // Winners pinned at the old top edge (8) — 12 is the new frontier; 2/3 + // scored worst and are dropped. + generator.addList("EXIT_RUN_BARS", {5, 8, 12}); + // Time cap on open trades, in minutes: during() closes a trade open + // strictly longer than this. No uncapped variant — every winner must + // carry an exit clock. 120 was the pinned edge in 2026-28 (30 the worst + // cell) — the cap slides up a notch to probe 240. + generator.addList("MAX_TRADE_DURATION_MINUTES", {60, 120, 240}); + // Exits are central (Operations enforces SL/TP). The values are ATR + // multipliers: conditions::check turns them into pip distances per entry + // (distance = ATR(10) x multiplier, clamped). Momentum shape, the + // OhlcBreakout doctrine (NOT KeltnerFade's): limit FARTHER than stop — + // let the flow run, cut the failure quickly. Stop 4 was non-binding + // (scores identical to 2/3 — duplicate variants) and limit 7 was the + // pinned best edge, so the limit grid slides up to 9. + generator.addList("STOP_DISTANCE_IN_ATR", {2, 3}); + generator.addList("LIMIT_DISTANCE_IN_ATR", {3, 5, 7, 9}); + return generator; +} + +} // namespace sweep diff --git a/source/load/config/runConfigurationBuilder.cppm b/source/load/config/runConfigurationBuilder.cppm index e5ab825..2742a4d 100644 --- a/source/load/config/runConfigurationBuilder.cppm +++ b/source/load/config/runConfigurationBuilder.cppm @@ -9,7 +9,10 @@ module; #include #include +#include "run/reporting/elasticPublisher.hpp" // elastic::nowIsoUtc +#include "run/reporting/outcomeIndices.hpp" // outcome_index::currentBatchLabel #include "shared/tradingDefinitions/config/runConfiguration.hpp" +#include "shared/utilities/env.hpp" export module runConfigurationBuilder; @@ -60,17 +63,65 @@ std::vector resolveSymbolGroups(const std::vector& swe | std::ranges::to(); } +// The slippage stress toggle, read from the environment ONCE per load so the +// value is frozen into every queued run descriptor (results in Elasticsearch +// then record the stress they ran under — a worker-side env var would change +// the meaning of already-queued jobs). Tenths of a pip; unset/empty = 0 = off. +// Garbage or a negative value fails the load loudly — a stress sweep that +// silently ran unstressed is worse than no sweep. +int entrySlippageFromEnv() { + const std::string raw = env::getOr("ENTRY_SLIPPAGE_TENTH_PIPS", "0"); + int tenthPips = 0; + const auto [ptr, ec] = + std::from_chars(raw.data(), raw.data() + raw.size(), tenthPips); + if (ec != std::errc{} || ptr != raw.data() + raw.size() || tenthPips < 0) { + throw std::invalid_argument( + "ENTRY_SLIPPAGE_TENTH_PIPS must be a non-negative integer " + "(tenths of a pip), got: " + raw); + } + return tenthPips; +} + +// The batch identity a load stamps into every queued run descriptor: the +// weekly Elasticsearch index label plus the seed wall clock. Frozen ONCE per +// load invocation (same doctrine as entrySlippageFromEnv — the value must not +// change under already-queued jobs) and carried through Redis and every +// rolling-window rung, so the whole batch reports into one week's indices no +// matter when its runs drain. A default-constructed (empty) stamp is the +// legacy escape hatch: unsuffixed index names, no batch metadata. +struct BatchStamp { + std::string label; // "2026-28" — see outcome_index::isoWeekLabel + std::string executionTs; // ISO-8601 seed wall clock, shared by the batch +}; + +BatchStamp currentBatchStamp() { + return {outcome_index::currentBatchLabel(), elastic::nowIsoUtc()}; +} + // The run-level descriptor: what tick data to pull from QuestDB, plus the risk // limits every strategy in the run runs under. Shared by every strategy in this // run and linked to them by RUN_ID. SYMBOLS is one cleaned, comma-separated // group from kSymbolGroups (see cleanSymbols in load/utility/symbolGroups.cppm). +// `batch` has no default on purpose: a new call site that forgot it would +// silently write every document to the unsuffixed fallback indices. tradingDefinitions::RunConfiguration makeRunConfiguration(const std::string& runId, - const std::string& symbols) { + const std::string& symbols, + const BatchStamp& batch) { using namespace boost::decimal::literals; return tradingDefinitions::RunConfiguration{ .RUN_ID = runId, .SYMBOLS = symbols, - .LAST_MONTHS = 6, + .BATCH = batch.label, + .EXECUTION_TS = batch.executionTs, + // (3, 0) — the most recent 3 months — is the first rung of the + // rolling-window ladder (rolling::nextWindow): each strategy that + // completes this window is re-queued by the runner over the 3-month + // slice before it, and so on through 9 months of history, ending with + // one run over the full 9 months. OFFSET_MONTHS is how far back the + // window ends (0 = the present day); a window pair NOT on the ladder + // sweeps exactly once, with no follow-on runs. + .LAST_MONTHS = 3, + .OFFSET_MONTHS = 0, .STARTING_BALANCE = tradingDefinitions::DEFAULT_STARTING_BALANCE, // Cut a run off once it has lost 5% of the account (fail fast); // set <= 0 to run without any loss cutoff. @@ -79,9 +130,20 @@ tradingDefinitions::RunConfiguration makeRunConfiguration(const std::string& run // The per-symbol gate already limits to one trade per symbol, so this // only bites on multi-symbol runs. .MAX_OPEN_TRADES = 1, + // Cap on trade entries per sliding 60-second window of tick time + // (<= 0 = unlimited) — a runaway-strategy brake. + .MAX_TRADES_PER_MINUTE = 60, // Flip to false to silence liquidated runs from Elasticsearch once // sweeps scale up and loss-limit cutoffs are expected noise. - .REPORT_FAILURES = true, + .REPORT_FAILURES = false, + // Entries only inside each symbol's peak session window (see + // marketHours) — newly queued runs trade peak hours only; configs + // persisted before the field existed parse as false. + .PEAK_HOURS_ONLY = true, + // Slippage stress: export ENTRY_SLIPPAGE_TENTH_PIPS=3 before `load` + // to re-run the sweep with 0.3 pip of adverse entry slippage; the + // rolling-window ladder carries the value through every rung. + .ENTRY_SLIPPAGE_TENTH_PIPS = entrySlippageFromEnv(), }; } diff --git a/source/load/config/sessionRangeBreakoutStrategy/makeSessionRangeBreakoutStrategy.cppm b/source/load/config/sessionRangeBreakoutStrategy/makeSessionRangeBreakoutStrategy.cppm new file mode 100644 index 0000000..e818fe0 --- /dev/null +++ b/source/load/config/sessionRangeBreakoutStrategy/makeSessionRangeBreakoutStrategy.cppm @@ -0,0 +1,70 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module makeSessionRangeBreakoutStrategy; + +import std; +import sweepCombination; // sweep::Combination + +export namespace sweep { + +// Map one swept parameter combination onto a SessionRangeBreakoutStrategy +// config. Every parameter is read with getInt and no has() fallback: the +// sweep registers every name read here +// (buildSessionRangeBreakoutStrategySweep), so a missing one is a bug that +// should throw at load time, matching the strategy ctor's fail-fast +// validation on the run side. +tradingDefinitions::StrategyConfig makeSessionRangeBreakoutStrategy( + const sweep::Combination& combo) { + using namespace tradingDefinitions; + + const int ohlcMinutes = combo.getInt("OHLC_MINUTES"); + const int entryWindowMinutes = combo.getInt("ENTRY_WINDOW_MINUTES"); + + // The window must span midnight through the entry cutoff on a winter day + // (08:00 London open = 480 minutes), with the ctor's two-bar margin: + // ceil((480 + window) / minutes) + 2 bars. Derived rather than swept, so + // every combination is valid by construction — the generator can't + // express cross-field constraints. + const int ohlcCount = + (480 + entryWindowMinutes + ohlcMinutes - 1) / ohlcMinutes + 2; + + return StrategyConfig{ + .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), + .TRADING_VARIABLES = TradingVariables{ + // Must match the dispatch string in strategyFactory. + .STRATEGY = "SessionRangeBreakoutStrategy", + .STOP_DISTANCE_IN_ATR = combo.getInt("STOP_DISTANCE_IN_ATR"), + .LIMIT_DISTANCE_IN_ATR = combo.getInt("LIMIT_DISTANCE_IN_ATR"), + .TRADING_SIZE = 1, + }, + // Positional contract with SessionRangeBreakoutStrategy: [0] is the + // signal timeframe (it also drives the ATR entry gate). + .OHLC_VARIABLES = { + OHLCVariables{ + .OHLC_COUNT = ohlcCount, + .OHLC_MINUTES = ohlcMinutes, + }, + }, + .STRATEGY_VARIABLES = StrategyVariables{ + .SESSION_RANGE_BREAKOUT_VARIABLES = SessionRangeBreakoutVariables{ + .BUFFER_PIPS = combo.getInt("BUFFER_PIPS"), + .ENTRY_WINDOW_MINUTES = entryWindowMinutes, + .MAX_TRADE_DURATION_MINUTES = + combo.getInt("MAX_TRADE_DURATION_MINUTES"), + }, + }, + }; +} + +} // namespace sweep diff --git a/source/load/config/sessionRangeBreakoutStrategy/sessionRangeBreakoutStrategySweep.cppm b/source/load/config/sessionRangeBreakoutStrategy/sessionRangeBreakoutStrategySweep.cppm new file mode 100644 index 0000000..15361fe --- /dev/null +++ b/source/load/config/sessionRangeBreakoutStrategy/sessionRangeBreakoutStrategySweep.cppm @@ -0,0 +1,64 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module sessionRangeBreakoutStrategySweep; + +export import parameterGenerator; // buildSessionRangeBreakoutStrategySweep() returns sweep::ParameterGenerator + +import std; // replaces , + +export namespace sweep { + +// The symbol groups THIS sweep runs against (see kSymbolGroupsOverride in +// randomStrategySweep for the full semantics: empty = default set, entries +// are validated against symbol_scale::kTable at compile time). Named +// per-strategy because the sweep modules are imported side by side +// (loadCommand, tests) and two exported sweep::kSymbolGroupsOverride would +// collide. Europe-session symbols only (market_hours' mapping): the strategy +// trades the London open, and with PEAK_HOURS_ONLY on the run loop's entry +// gate only opens for these symbols during exactly that window. Index CFDs +// only since 2026-28: the seven Europe-mapped FX pairs contributed zero of +// the batch's 493 winners — the London-open edge lives on the indices. +inline constexpr std::array kSessionRangeBreakoutSymbolGroupsOverride = + std::to_array({ + "DEUIDXEUR", "FRAIDXEUR", "GBRIDXGBP"}); + +// Declares which parameters to sweep for the SessionRangeBreakoutStrategy. +// The OHLC COUNT is NOT swept: makeSessionRangeBreakoutStrategy derives it +// from OHLC_MINUTES and ENTRY_WINDOW_MINUTES at the ctor minimum (span +// midnight -> entry cutoff), so every combination is valid by construction — +// a deeper window adds nothing because bars are selected by date. Tune the +// values here; makeSessionRangeBreakoutStrategy reads back every name +// registered below. +ParameterGenerator buildSessionRangeBreakoutStrategySweep() { + ParameterGenerator generator; + generator.setSymbolGroups(); + // Signal timeframe the Asian range is built from. + generator.addList("OHLC_MINUTES", {5, 15, 30, 45}); + // Padding on the Asian high/low, in pips (0 = raw range). The dense + // {0..5} ladder was flat in 2026-28 — a coarse A/B is enough. + generator.addList("BUFFER_PIPS", {0, 2, 4}); + // How long after the London open entries may fire. The 2026-28 scores + // decayed monotonically with the window (180 had volume but the worst + // scores by far) — the edge is in the first hour, so the grid slides + // down and 20 probes tighter. + generator.addList("ENTRY_WINDOW_MINUTES", {20, 30, 40, 60}); + // Time cap on open trades, in minutes: during() closes a trade open + // strictly longer than this. No uncapped variant — every winner must + // carry an exit clock (trades were riding for 10+ hours live). 120 was + // the pinned edge in 2026-28 — the cap slides up a notch to probe 240. + generator.addList("MAX_TRADE_DURATION_MINUTES", {60, 120, 240}); + // Exits are central (Operations enforces SL/TP). The values are ATR + // multipliers: conditions::check turns them into pip distances per entry + // (distance = ATR(10) x multiplier, clamped). The "vol-expansion needs a + // wide stop" hypothesis failed its A/B: 1-ATR stops took 59% of the + // 2026-28 winners with the best scores, and 3 was the worst — dropped. + generator.addList("STOP_DISTANCE_IN_ATR", {1, 2}); + generator.addList("LIMIT_DISTANCE_IN_ATR", {3, 5, 7}); // 9 was flat vs 7 + return generator; +} + +} // namespace sweep diff --git a/source/load/config/squeezeBreakoutStrategy/makeSqueezeBreakoutStrategy.cppm b/source/load/config/squeezeBreakoutStrategy/makeSqueezeBreakoutStrategy.cppm new file mode 100644 index 0000000..897a7cb --- /dev/null +++ b/source/load/config/squeezeBreakoutStrategy/makeSqueezeBreakoutStrategy.cppm @@ -0,0 +1,70 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module makeSqueezeBreakoutStrategy; + +import std; +import sweepCombination; // sweep::Combination + +export namespace sweep { + +// Map one swept parameter combination onto a SqueezeBreakoutStrategy config. +// Every parameter is read with getInt and no has() fallback: the sweep +// registers every name read here (buildSqueezeBreakoutStrategySweep), so a +// missing one is a bug that should throw at load time, matching the strategy +// ctor's fail-fast validation on the run side. +tradingDefinitions::StrategyConfig makeSqueezeBreakoutStrategy( + const sweep::Combination& combo) { + using namespace tradingDefinitions; + + const int nrLookback = combo.getInt("NR_LOOKBACK"); + const int validBars = combo.getInt("VALID_BARS"); + + return StrategyConfig{ + .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), + .TRADING_VARIABLES = TradingVariables{ + // Must match the dispatch string in strategyFactory. + .STRATEGY = "SqueezeBreakoutStrategy", + .STOP_DISTANCE_IN_ATR = combo.getInt("STOP_DISTANCE_IN_ATR"), + .LIMIT_DISTANCE_IN_ATR = combo.getInt("LIMIT_DISTANCE_IN_ATR"), + .TRADING_SIZE = 1, + }, + // Positional contract with SqueezeBreakoutStrategy: [0] is the signal + // timeframe (it also drives the ATR entry gate), [1] the trend + // timeframe. The signal count is DERIVED at the ctor minimum + // (VALID_BARS + max(1, NR_LOOKBACK - 1) + 1) rather than swept, so + // every combination is valid by construction — the generator can't + // express cross-field constraints. + .OHLC_VARIABLES = { + OHLCVariables{ + .OHLC_COUNT = validBars + std::max(1, nrLookback - 1) + 1, + .OHLC_MINUTES = combo.getInt("OHLC_MINUTES"), + }, + OHLCVariables{ + .OHLC_COUNT = combo.getInt("TREND_OHLC_COUNT"), + .OHLC_MINUTES = combo.getInt("TREND_OHLC_MINUTES"), + }, + }, + .STRATEGY_VARIABLES = StrategyVariables{ + .SQUEEZE_BREAKOUT_VARIABLES = SqueezeBreakoutVariables{ + .NR_LOOKBACK = nrLookback, + .VALID_BARS = validBars, + .BUFFER_PIPS = combo.getInt("BUFFER_PIPS"), + .MAX_TRADE_DURATION_MINUTES = + combo.getInt("MAX_TRADE_DURATION_MINUTES"), + }, + }, + }; +} + +} // namespace sweep diff --git a/source/load/config/squeezeBreakoutStrategy/squeezeBreakoutStrategySweep.cppm b/source/load/config/squeezeBreakoutStrategy/squeezeBreakoutStrategySweep.cppm new file mode 100644 index 0000000..77ecf64 --- /dev/null +++ b/source/load/config/squeezeBreakoutStrategy/squeezeBreakoutStrategySweep.cppm @@ -0,0 +1,73 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module squeezeBreakoutStrategySweep; + +export import parameterGenerator; // buildSqueezeBreakoutStrategySweep() returns sweep::ParameterGenerator + +import std; // replaces , + +export namespace sweep { + +// The symbol groups THIS sweep runs against (see kSymbolGroupsOverride in +// randomStrategySweep for the full semantics: empty = default set, entries +// are validated against symbol_scale::kTable at compile time). Named +// per-strategy because the sweep modules are imported side by side +// (loadCommand, tests) and two exported sweep::kSymbolGroupsOverride would +// collide. Squeeze breakouts historically favour indices and metals — the +// 2026-28 batch agreed — and the ranging crosses and copper (NZDUSD, +// USDCHF, EURGBP, EURCHF, EURNOK, AUDNZD, COPPERCMDUSD) produced zero +// winners across EVERY strategy, so they are dropped from the sweep +// universe. +inline constexpr std::array kSqueezeBreakoutSymbolGroupsOverride = + std::to_array({ + "AUDUSD", "EURUSD", "GBRIDXGBP", "GBPUSD", + "USDJPY", "GBPJPY", "EURJPY", "USDCAD", "FRAIDXEUR", + "USA500IDXUSD", "AUSIDXAUD", "XAUUSD", + "XAGUSD", "USATECHIDXUSD", "DEUIDXEUR", "USA30IDXUSD", + "LIGHTCMDUSD", "JPNIDXJPY", "BRENTCMDUSD", "EURAUD", + "HKGIDXHKD", "USDSEK"}); + +// Declares which parameters to sweep for the SqueezeBreakoutStrategy. The +// signal OHLC COUNT is NOT swept: makeSqueezeBreakoutStrategy derives it at +// the ctor minimum (VALID_BARS + max(1, NR_LOOKBACK - 1) + 1), so every +// combination is valid by construction — a deeper window adds nothing +// because the scan depth is governed by VALID_BARS / NR_LOOKBACK. Tune the +// values here; makeSqueezeBreakoutStrategy reads back every name registered +// below. +ParameterGenerator buildSqueezeBreakoutStrategySweep() { + ParameterGenerator generator; + generator.setSymbolGroups(); + // Signal timeframe the contraction patterns form on. + generator.addList("OHLC_MINUTES", {15, 30, 60}); + // Pattern mode: 0 = inside bar, N = narrowest range of the last N. + // 5 was decisively the worst mode in 2026-28 while 0 and 7 both paid — + // dropped. + generator.addList("NR_LOOKBACK", {0, 7}); + // How many closed bars a matched pattern stays tradeable for. 3 beat 1 + // at the old top edge — 5 is the new frontier. + generator.addList("VALID_BARS", {1, 3, 5}); + // Padding on the pattern bar's high/low, in pips (0 = raw levels). + generator.addList("BUFFER_PIPS", {0, 2}); + // Time cap on open trades, in minutes: during() closes a trade open + // strictly longer than this. No uncapped variant — every winner must + // carry an exit clock (trades were riding for 10+ hours live). + generator.addList("MAX_TRADE_DURATION_MINUTES", {30, 60, 120}); + // Trend timeframe: EMA over its closes (period = count / 2) is the + // filter — the OhlcBreakoutStrategy idiom. Counts give EMA periods + // 20/40: fast vs slow filter on the same timeframes. + generator.addList("TREND_OHLC_MINUTES", {60, 120}); + generator.addList("TREND_OHLC_COUNT", {40, 80}); + // Exits are central (Operations enforces SL/TP). The values are ATR + // multipliers: conditions::check turns them into pip distances per entry + // (distance = ATR(10) x multiplier, clamped). + generator.addList("STOP_DISTANCE_IN_ATR", {1, 2}); + generator.addList("LIMIT_DISTANCE_IN_ATR", {3, 5, 7}); // 9 was flat vs 7 + return generator; +} + + +} // namespace sweep diff --git a/source/load/loadCommand.cppm b/source/load/loadCommand.cppm index e71bd6b..e74e05f 100644 --- a/source/load/loadCommand.cppm +++ b/source/load/loadCommand.cppm @@ -11,6 +11,8 @@ module; #include #include +#include "run/reporting/elasticPublisher.hpp" // ensureIndexExists, repointAlias +#include "run/reporting/outcomeIndices.hpp" // weekly index names + aliases #include "shared/utilities/env.hpp" #include "shared/utilities/queueKeys.hpp" #include "load/redisLoader.hpp" @@ -20,12 +22,27 @@ module; export module loadCommand; import std; // replaces , , , +import backtestLog; // backtest_log::logLine — timestamped, flushed stdout import parameterGenerator; // sweep::ParameterGenerator, sweep::Combination import randomStrategySweep; // buildRandomStrategySweep import ohlcBreakoutStrategySweep; // buildOhlcBreakoutStrategySweep +import fvgStrategySweep; // buildFvgStrategySweep +import keltnerFadeStrategySweep; // buildKeltnerFadeStrategySweep +import sessionRangeBreakoutStrategySweep; // buildSessionRangeBreakoutStrategySweep +import squeezeBreakoutStrategySweep; // buildSqueezeBreakoutStrategySweep +import nyOpenRangeBreakoutStrategySweep; // buildNyOpenRangeBreakoutStrategySweep +import liquiditySweepReversalStrategySweep; // buildLiquiditySweepReversalStrategySweep +import rangeVelocityStrategySweep; // buildRangeVelocityStrategySweep import runConfigurationBuilder; // makeRunConfiguration, resolveSymbolGroups import makeStrategy; // sweep::makeStrategy import makeOhlcBreakoutStrategy; // sweep::makeOhlcBreakoutStrategy +import makeFvgStrategy; // sweep::makeFvgStrategy +import makeKeltnerFadeStrategy; // sweep::makeKeltnerFadeStrategy +import makeSessionRangeBreakoutStrategy; // sweep::makeSessionRangeBreakoutStrategy +import makeSqueezeBreakoutStrategy; // sweep::makeSqueezeBreakoutStrategy +import makeNyOpenRangeBreakoutStrategy; // sweep::makeNyOpenRangeBreakoutStrategy +import makeLiquiditySweepReversalStrategy; // sweep::makeLiquiditySweepReversalStrategy +import makeRangeVelocityStrategy; // sweep::makeRangeVelocityStrategy import symbolGroups; // sweep::cleanSymbols export namespace sweep { @@ -76,10 +93,6 @@ constexpr std::size_t kChunkSize = 1000; // so the boundary always lands on a chunk edge. constexpr std::size_t kProgressEvery = 100'000; -// Safety-net expiry on payload keys so a crashed or abandoned run cannot leak -// them forever. The normal cleanup is the worker's GETDEL. -constexpr long kPayloadTtlSeconds = 7L * 24 * 60 * 60; - // Feeds RedisLoader::loadKeyedPayloadStream one lazily-built chunk at a time, // walking the grid indices [0, total) exactly once. class SweepChunkSource final : public RedisLoader::ChunkSource { @@ -98,8 +111,8 @@ public: const std::size_t end = std::min(begin + kChunkSize, total_); next_ = end; if (begin != 0 && begin % kProgressEvery == 0) { - std::println("LoadCommand: queued {}/{} strategies...", - withThousands(begin), withThousands(total_)); + backtest_log::logLine("LoadCommand: queued {}/{} strategies...", + withThousands(begin), withThousands(total_)); } return sweep::buildStrategyChunk(generator_, strategyFactory_, runId_, begin, end); @@ -113,6 +126,30 @@ private: std::size_t next_ = 0; }; +// Create this batch's weekly outcome indices and atomically repoint their +// -current aliases, once per load invocation — alias admin belongs to the +// seed, not the workers, where a late-flushing old-week run could flip an +// alias backwards. Best-effort: on the first failure log and stop (each admin +// call already retried transient failures internally; six more bounded-retry +// stalls against a dead host help nobody). Runs still carry the batch label +// and Elasticsearch auto-creates a weekly index on its first _bulk write, so +// only the alias lags — the next successful load self-heals it. With +// $ELASTIC_ENABLED=0 every call is a successful no-op, keeping a Redis-only +// local load Elastic-free. +void prepareWeeklyOutcomeIndices(const std::string& batchLabel) { + for (const std::string_view base : outcome_index::kWeeklyBases) { + const std::string index = outcome_index::weeklyIndex(base, batchLabel); + if (elastic::ensureIndexExists(index) != 0 || + elastic::repointAlias(outcome_index::currentAlias(base), index) != 0) { + backtest_log::logLine( + "LoadCommand: weekly index/alias admin failed at {} — skipping " + "the rest; the next successful load repoints the aliases", + index); + return; + } + } +} + } // namespace std::vector sweep::buildStrategyChunk( @@ -137,13 +174,13 @@ std::vector sweep::buildStrategyChunk( int LoadCommand::run(const int argc, const char* argv[]) { - // Select the sweep from the command line, e.g. `load random`. Defaults to - // "random" when omitted. An unknown name is a usage error, not a crash, so - // report it and bail. Each sweep pairs its parameter grid with the factory - // that maps a combination onto that strategy's config, so both are chosen - // together — a new sweep is one extra branch here plus a mention in the - // error. - const std::string_view sweepName = argc > 2 ? argv[2] : "ohlcBreakout"; + // Select the sweep from the command line, e.g. `load random`. The name is + // required — omitting it or passing an unknown name is a usage error, not + // a crash, so report the valid choices and bail. Each sweep pairs its + // parameter grid with the factory that maps a combination onto that + // strategy's config, so both are chosen together — a new sweep is one + // extra branch here plus a mention in the error. + const std::string_view sweepName = argc > 2 ? argv[2] : ""; sweep::ParameterGenerator generator; sweep::StrategyFactory strategyFactory = nullptr; if (sweepName == "random") { @@ -152,9 +189,33 @@ int LoadCommand::run(const int argc, const char* argv[]) { } else if (sweepName == "ohlcBreakout") { generator = sweep::buildOhlcBreakoutStrategySweep(); strategyFactory = sweep::makeOhlcBreakoutStrategy; + } else if (sweepName == "fvg") { + generator = sweep::buildFvgStrategySweep(); + strategyFactory = sweep::makeFvgStrategy; + } else if (sweepName == "keltnerFade") { + generator = sweep::buildKeltnerFadeStrategySweep(); + strategyFactory = sweep::makeKeltnerFadeStrategy; + } else if (sweepName == "sessionRangeBreakout") { + generator = sweep::buildSessionRangeBreakoutStrategySweep(); + strategyFactory = sweep::makeSessionRangeBreakoutStrategy; + } else if (sweepName == "squeezeBreakout") { + generator = sweep::buildSqueezeBreakoutStrategySweep(); + strategyFactory = sweep::makeSqueezeBreakoutStrategy; + } else if (sweepName == "nyOpenRangeBreakout") { + generator = sweep::buildNyOpenRangeBreakoutStrategySweep(); + strategyFactory = sweep::makeNyOpenRangeBreakoutStrategy; + } else if (sweepName == "liquiditySweepReversal") { + generator = sweep::buildLiquiditySweepReversalStrategySweep(); + strategyFactory = sweep::makeLiquiditySweepReversalStrategy; + } else if (sweepName == "rangeVelocity") { + generator = sweep::buildRangeVelocityStrategySweep(); + strategyFactory = sweep::makeRangeVelocityStrategy; } else { std::println(stderr, - "LoadCommand: unknown sweep generator '{}' (valid: random, ohlcBreakout)", + "LoadCommand: unknown sweep generator '{}' (valid: random, " + "ohlcBreakout, fvg, keltnerFade, sessionRangeBreakout, " + "squeezeBreakout, nyOpenRangeBreakout, " + "liquiditySweepReversal, rangeVelocity)", sweepName); return 1; } @@ -168,9 +229,10 @@ int LoadCommand::run(const int argc, const char* argv[]) { // prompt. const auto symbolGroups = sweep::resolveSymbolGroups(generator.symbolGroups()); const auto combinationCount = generator.combinationCount(); - std::println("LoadCommand: '{}' sweep: {} combination(s) x {} symbol group(s) = {} backtest(s)", - sweepName, withThousands(combinationCount), symbolGroups.size(), - withThousands(combinationCount * symbolGroups.size())); + backtest_log::logLine("LoadCommand: '{}' sweep: {} combination(s) x {} symbol group(s) = {} backtest(s)", + sweepName, withThousands(combinationCount), symbolGroups.size(), + withThousands(combinationCount * symbolGroups.size())); + std::print("Press Enter to queue them (Ctrl+C to abort)... "); std::fflush(stdout); if (std::string ack; !std::getline(std::cin, ack)) { @@ -178,8 +240,23 @@ int LoadCommand::run(const int argc, const char* argv[]) { return 1; } + // Freeze this load's batch identity (weekly index label + seed wall + // clock) before anything is queued, then prepare the weekly indices and + // aliases. scripts/load.sh pins $BACKTEST_BATCH across its per-strategy + // invocations so a load straddling the ISO-week boundary cannot split one + // batch over two labels. + const sweep::BatchStamp batch = sweep::currentBatchStamp(); + backtest_log::logLine("LoadCommand: batch {} (execution {})", batch.label, + batch.executionTs); + prepareWeeklyOutcomeIndices(batch.label); + const auto redisHost = env::getOr("REDIS_HOST", "127.0.0.1"); + // One loader = one persistent Redis connection shared by every run below + // (strategy streams and run descriptors alike), instead of a fresh + // resolve/connect per push. + RedisLoader loader(redisHost, 6379); + // Fan out: every resolved symbol group becomes its own run — the sweep's // own list when it set one, the full kSymbolGroups default otherwise. A // comma-separated entry like "EURUSD,AUDUSD" is one run over multiple @@ -192,8 +269,8 @@ int LoadCommand::run(const int argc, const char* argv[]) { // distinguished by its parameter values and freshly-minted UUID. const auto runId = boost::uuids::to_string(boost::uuids::random_generator()()); - std::println("LoadCommand: sweeping {} parameter combination(s) for RUN_ID={} symbols={}", - withThousands(combinationCount), runId, symbols); + backtest_log::logLine("LoadCommand: sweeping {} parameter combination(s) for RUN_ID={} symbols={}", + withThousands(combinationCount), runId, symbols); // Stream every strategy into Redis BEFORE the run descriptor: payload // keys first, then their names onto the run's list (RedisLoader keeps @@ -201,9 +278,8 @@ int LoadCommand::run(const int argc, const char* argv[]) { // the name list and retires the run when empty, so the full set must // already be present the moment the run becomes visible. SweepChunkSource source(generator, strategyFactory, runId, combinationCount); - if (const auto strategyStatus = RedisLoader::loadKeyedPayloadStream( - redisHost, 6379, queue_keys::strategyKey(runId), source, - kPayloadTtlSeconds); + if (const auto strategyStatus = loader.loadKeyedPayloadStream( + queue_keys::strategyKey(runId), source, queue_keys::PAYLOAD_TTL_SECONDS); strategyStatus != 0) { return strategyStatus; @@ -212,8 +288,8 @@ int LoadCommand::run(const int argc, const char* argv[]) { // Now advertise the run so workers can pick it up. runJson is pinned to // nlohmann::JSON (not auto) because makeRunConfiguration returns a // RunConfiguration and relies on the implicit conversion for .dump(). - const nlohmann::json runJson = sweep::makeRunConfiguration(runId, symbols); - if (const auto runStatus = RedisLoader::loadPayload(redisHost, 6379, queue_keys::RUN, runJson.dump()); + const nlohmann::json runJson = sweep::makeRunConfiguration(runId, symbols, batch); + if (const auto runStatus = loader.loadPayload(queue_keys::RUN, runJson.dump()); runStatus != 0) { return runStatus; diff --git a/source/load/redisLoader.cpp b/source/load/redisLoader.cpp index f228875..edd4b25 100644 --- a/source/load/redisLoader.cpp +++ b/source/load/redisLoader.cpp @@ -45,13 +45,10 @@ asio::awaitable pushOnce( std::string encoded) { RedisOperations ops(conn); co_await ops.listPushFront(std::move(queueKey), std::move(encoded)); - - // Ephemeral connection: cancel so io_context::run() returns after the push. - conn->cancel(); co_return; } -// Drains `source` chunk by chunk over the one borrowed connection. Per chunk: +// Drains `source` chunk by chunk over the loader's connection. Per chunk: // pipelined SET (PX ttl) of every payload under its own key, THEN one // pipelined LPUSH of the key names — awaited in that order so a consumer can // never pop a name whose payload is not yet stored. Chunk production happens @@ -64,48 +61,71 @@ asio::awaitable pushKeyedStream( std::chrono::milliseconds ttl) { RedisOperations ops(conn); std::size_t total = 0; - try { - for (;;) { - std::vector chunk = source.next(); - if (chunk.empty()) { - break; - } + for (;;) { + std::vector chunk = source.next(); + if (chunk.empty()) { + break; + } - std::vector> keyedValues; - std::vector keyNames; - keyedValues.reserve(chunk.size()); - keyNames.reserve(chunk.size()); - for (RedisLoader::KeyedPayload& payload : chunk) { - if (payload.key.empty() || isBlank(payload.rawJson)) { - throw std::invalid_argument( - "RedisLoader: empty payload or key rejected"); - } - keyNames.push_back(payload.key); - keyedValues.emplace_back(std::move(payload.key), - Base64::b64encode(payload.rawJson)); + std::vector> keyedValues; + std::vector keyNames; + keyedValues.reserve(chunk.size()); + keyNames.reserve(chunk.size()); + for (RedisLoader::KeyedPayload& payload : chunk) { + if (payload.key.empty() || isBlank(payload.rawJson)) { + throw std::invalid_argument( + "RedisLoader: empty payload or key rejected"); } - - co_await ops.setMultipleWithTTL(std::move(keyedValues), ttl); - co_await ops.listPushFront(listKey, std::move(keyNames)); - total += chunk.size(); + keyNames.push_back(payload.key); + keyedValues.emplace_back(std::move(payload.key), + Base64::b64encode(payload.rawJson)); } - } catch (...) { - // Cancel on the failure path too: the connection's detached async_run - // otherwise keeps io_context::run() from ever returning. - conn->cancel(); - throw; - } - // Ephemeral connection: cancel so io_context::run() returns after the push. - conn->cancel(); + co_await ops.setMultipleWithTTL(std::move(keyedValues), ttl); + co_await ops.listPushFront(listKey, std::move(keyNames)); + total += chunk.size(); + } co_return total; } } // namespace -int RedisLoader::loadPayload(const std::string& redisHost, - int redisPort, - const std::string& queueKey, +struct RedisLoader::Impl { + asio::io_context ioc; + std::shared_ptr conn; + + Impl(const std::string& host, const int port) + : conn(redis_util::makeRedisConnection(ioc, host, port)) {} + + // Pumps the io_context until `done` flips. The connection's detached + // async_run keeps the context supplied with work between operations, so + // handlers are dispatched one at a time until the spawned coroutine's + // completion sets the flag — the connection itself stays live for the + // next call. Returns false only if the context runs dry first (async_run + // died), which callers treat as a failed operation. + bool runUntil(const bool& done) { + ioc.restart(); + while (!done) { + if (ioc.run_one() == 0) { + return false; + } + } + return true; + } +}; + +RedisLoader::RedisLoader(std::string redisHost, const int redisPort) + : impl_(std::make_unique(redisHost, redisPort)) {} + +RedisLoader::~RedisLoader() { + // Cancel the connection's detached async_run, then drain the context so + // it winds down cleanly before the io_context is destroyed. + impl_->conn->cancel(); + impl_->ioc.restart(); + impl_->ioc.run(); +} + +int RedisLoader::loadPayload(const std::string& queueKey, const std::string& rawJson) { if (isBlank(rawJson)) { std::cerr << "RedisLoader: empty payload rejected" << std::endl; @@ -114,21 +134,22 @@ int RedisLoader::loadPayload(const std::string& redisHost, const std::string encoded = Base64::b64encode(rawJson); - asio::io_context ioc; - auto conn = redis_util::makeRedisConnection(ioc, redisHost, redisPort); - std::exception_ptr pushError; + bool done = false; asio::co_spawn( - ioc, - pushOnce(conn, queueKey, encoded), - [&pushError](std::exception_ptr e) { - if (e) { - pushError = e; - } + impl_->ioc, + pushOnce(impl_->conn, queueKey, encoded), + [&pushError, &done](std::exception_ptr e) { + pushError = e; + done = true; }); - ioc.run(); + if (!impl_->runUntil(done)) { + std::cerr << "RedisLoader: connection stopped before LPUSH completed" + << std::endl; + return 3; + } if (pushError) { try { @@ -142,29 +163,28 @@ int RedisLoader::loadPayload(const std::string& redisHost, return 0; } -int RedisLoader::loadKeyedPayloadStream(const std::string& redisHost, - const int redisPort, - const std::string& listKey, +int RedisLoader::loadKeyedPayloadStream(const std::string& listKey, ChunkSource& source, const long ttlSeconds) { - asio::io_context ioc; - auto conn = redis_util::makeRedisConnection(ioc, redisHost, redisPort); - std::exception_ptr pushError; - std::size_t stored = 0; + bool done = false; asio::co_spawn( - ioc, - pushKeyedStream(conn, listKey, source, std::chrono::seconds(ttlSeconds)), - [&pushError, &stored](const std::exception_ptr& e, std::size_t total) { + impl_->ioc, + pushKeyedStream(impl_->conn, listKey, source, + std::chrono::seconds(ttlSeconds)), + [&pushError, &done](const std::exception_ptr& e, std::size_t) { if (e) { pushError = e; - } else { - stored = total; } + done = true; }); - ioc.run(); + if (!impl_->runUntil(done)) { + std::cerr << "RedisLoader: connection stopped before payload stream completed" + << std::endl; + return 3; + } if (pushError) { try { @@ -178,8 +198,5 @@ int RedisLoader::loadKeyedPayloadStream(const std::string& redisHost, } } - std::cout << "RedisLoader: stored " << stored - << " keyed payload(s), key names on " << listKey << std::endl; - return 0; } diff --git a/source/load/redisLoader.hpp b/source/load/redisLoader.hpp index fc643c7..cb4d671 100644 --- a/source/load/redisLoader.hpp +++ b/source/load/redisLoader.hpp @@ -7,10 +7,16 @@ #pragma once #include +#include #include #include // LPUSH pairs with RedisRunner's RPOP so consumers observe FIFO ordering. +// +// One loader owns ONE persistent Redis connection: construction is cheap (the +// TCP connect happens lazily inside the first load call), every subsequent +// load call reuses the same connection, and teardown lives in the destructor. +// A multi-run sweep therefore resolves/connects once, not once per run. class RedisLoader { public: // One strategy destined for its own Redis string key: `key` is the full @@ -31,21 +37,29 @@ class RedisLoader { virtual std::vector next() = 0; }; + RedisLoader(std::string redisHost, int redisPort); + ~RedisLoader(); + + RedisLoader(const RedisLoader&) = delete; + RedisLoader& operator=(const RedisLoader&) = delete; + // LPUSHes a single Base64-encoded payload onto queueKey without assuming a // payload type (run descriptor or strategy). - static int loadPayload(const std::string& redisHost, - int redisPort, - const std::string& queueKey, - const std::string& rawJson); - - // Streams every chunk from `source` over ONE connection. Per chunk, one - // pipelined SET (with ttlSeconds expiry) per payload key, then the key - // NAMES are LPUSHed onto listKey — in that order, so a consumer can never - // pop a name whose payload is not yet stored. The TTL is a safety net for - // crashed/abandoned runs; the consumer's GETDEL is the normal cleanup. - static int loadKeyedPayloadStream(const std::string& redisHost, - int redisPort, - const std::string& listKey, - ChunkSource& source, - long ttlSeconds); + int loadPayload(const std::string& queueKey, const std::string& rawJson); + + // Streams every chunk from `source` over the loader's connection. Per + // chunk, one pipelined SET (with ttlSeconds expiry) per payload key, then + // the key NAMES are LPUSHed onto listKey — in that order, so a consumer + // can never pop a name whose payload is not yet stored. The TTL is a + // safety net for crashed/abandoned runs; the consumer's GETDEL is the + // normal cleanup. + int loadKeyedPayloadStream(const std::string& listKey, + ChunkSource& source, + long ttlSeconds); + +private: + // Hides the Boost.Asio/Boost.Redis machinery (io_context + connection) so + // includers of this header don't inherit those dependencies. + struct Impl; + std::unique_ptr impl_; }; diff --git a/source/main.cpp b/source/main.cpp index 664a797..c84d950 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -9,6 +9,11 @@ import std; import loadCommand; import runCommand; import ingestCommand; +import liveCommand; +import trackingCommand; +import positionsCommand; +import experimentsCommand; +import analysisCommand; int main(const int argc, const char* argv[]) { @@ -18,9 +23,15 @@ int main(const int argc, const char* argv[]) { } const std::string_view subcommand = argv[1]; + if (subcommand == "load") return LoadCommand::run(argc, argv); if (subcommand == "run") return RunCommand::run(argc, argv); if (subcommand == "ingest") return IngestCommand::run(argc, argv); + if (subcommand == "live") return LiveCommand::run(argc, argv); + if (subcommand == "tracking") return TrackingCommand::run(argc, argv); + if (subcommand == "positions") return PositionsCommand::run(argc, argv); + if (subcommand == "experiments") return ExperimentsCommand::run(argc, argv); + if (subcommand == "analysis") return AnalysisCommand::run(argc, argv); std::println(std::cerr, "Error: unknown subcommand '{}'.", subcommand); return 1; diff --git a/source/positions/positionSync.cppm b/source/positions/positionSync.cppm new file mode 100644 index 0000000..55d0034 --- /dev/null +++ b/source/positions/positionSync.cppm @@ -0,0 +1,516 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// positionSync — one cycle of the IG position producer, the C++ port of the +// C# igmarkets_positions cron (vortex igmarkets_positions/IGMarketCalls.cs + +// Program.cs). Each cycle mirrors the broker's own /positions book into the +// Redis position store the rest of the ecosystem reads: +// +// GET /positions -> per matched deal, save/refresh PO# (10-min +// update TTL, 30-min fresh, weekend hold on Friday nights) and ensure it +// is in PL# -> rebuild the CG# cluster sets -> refresh +// every PL# list (prune deals whose PO# expired) -> report each stage +// to the live-function-logs Elastic index, like the C# LiveFunctionReport. +// +// The producer doctrine: PO# records carry a deliberately SHORT TTL, so this +// sync re-stamping them every minute is what keeps a live deal alive in +// Redis — a deal the broker no longer reports simply expires and falls out +// of the PL# lists on the refresh pass. The pure helpers (TTL rule, date +// parse, JSON decode, record build/update) are exported for the unit tests; +// everything Redis/HTTP flows through the shared PositionManager / +// PositionClustering / ig_rest seams. +// +// The GMF only #includes Asio-free headers (the Redis connection machinery +// stays behind positionManager.cpp / positionClustering.cpp), so it is safe +// to `import std` here; nlohmann in a module GMF follows liveWinners. + +module; + +#include + +#include "run/reporting/elasticPublisher.hpp" +#include "shared/ig/igRestClient.hpp" +#include "shared/redis/positionClustering.hpp" +#include "shared/redis/positionManager.hpp" + +export module positionSync; + +import std; +import backtestLog; // backtest_log::logLine +import marketDefinitions; // live::findMarketByEpicMini — the tradable-epic gate +import symbolScale; // symbol_scale::getPriceScale — decimal -> INT32 points + +export namespace positions { + +// Same std::function instantiation as ig::AuthProvider (igRequests), spelled +// locally so this module doesn't have to import the whole order machinery — +// the command passes ig::dynamoAuthProvider(...) straight in. +using AuthProvider = std::function()>; + +// One entry of the /positions response (the C# AccountPositions model): the +// nested market/position objects flattened to just the fields the producer +// consumes. IG's nullable doubles (stopLevel/limitLevel) decode as 0, +// matching the C# `?? 0`. +struct IgPosition { + std::string epic; // market.epic — matched against epicMini + std::string dealReference; // the PO#/PL# book key; skipped when empty + std::string dealId; + std::string direction; // "BUY" | "SELL" + std::string createdDate; // IG's "yyyy/MM/dd HH:mm:ss:fff" + double dealSize{}; + double openLevel{}; + double stopLevel{}; // null/absent -> 0 + double limitLevel{}; // null/absent -> 0 +}; + +// Decode a /positions body. nullopt = undecodable (not an object, or a +// positions value that is neither array nor null) — the C# JsonException +// path, reported FAILED-DESERIALIZE. A decodable body with positions +// null/absent is an EMPTY book, not an error (the C# SavePositions early +// return). Missing market/position sub-objects are tolerated per entry +// (fields stay empty/zero), like the C# null-conditional chains. +[[nodiscard]] std::optional> decodeAccountPositions( + const std::string& json); + +// IG's createdDate ("yyyy/MM/dd HH:mm:ss:fff", UTC — the C# AssumeUniversal) +// -> epoch microseconds. Hand-parsed with from_chars + sys_days: libc++ has +// no std::chrono::parse and gmtime_r/timegm are not in `import std`. Any +// malformed input returns `fallbackMicros` (the caller passes "now") — the +// C# ParseExact threw and killed the cron; in-process the record is simply +// stamped with the sync time instead. +[[nodiscard]] std::int64_t parseCreatedDateMicros(std::string_view createdDate, + std::int64_t fallbackMicros); + +// The update-path TTL rule, pure over the passed instant: 10 minutes, EXCEPT +// Friday 21:55:00–21:59:59 UTC where it becomes 2 days + 2 hours — IG closes +// the week at 21:00 Friday less the CFD after-hours, and a position still +// open then must survive Redis until Sunday-night trading resumes (the C# +// weekend hold, ported verbatim). +[[nodiscard]] std::chrono::milliseconds updateTtl(std::chrono::sys_seconds nowUtc); + +// Fresh saves get a longer leash than updates: a brand-new deal's receipt / +// book entries may lag a cycle or two (the C# Save's 30 minutes). +inline constexpr std::chrono::minutes kFreshPositionTtl{30}; + +// The C# Save(): build a PO# record from the broker's own numbers. Prices +// arrive as decimals and are stored as scaled INT32 points via `priceScale` +// (llround = the C# MidpointRounding.AwayFromZero); a priceScale <= 0 +// (symbol_scale::kUnknown) writes level 0, and the caller logs the warning. +// The scale is injected so tests exercise the arithmetic without depending +// on the current symbolScale table. openedAt comes from createdDate with +// `nowMicros` as the malformed-date fallback. +[[nodiscard]] redis_positions::PositionRecord makeFreshRecord( + const IgPosition& position, std::string_view symbol, + std::string_view strategyId, std::string_view strategyName, int priceScale, + std::int64_t nowMicros); + +// The C# Update() mutation on an existing PO# record: a missing strategy +// attribution ("" or "Unknown") is filled from the deal receipt, and the +// dealId is refreshed when IG reports one (the C# null-coalesce, hardened +// against an empty string clobbering a known id). Everything else — levels, +// size, openedAt — keeps the stored engine-side values. +void applyBrokerUpdate(redis_positions::PositionRecord& record, + const IgPosition& position, std::string_view strategyId); + +// One producer instance: owns its Redis connections (one PositionManager, +// one PositionClustering) and the IG session provider. Single-threaded by +// design — the command's minute loop is the only caller, so the members' +// internal mutexes are uncontended. +class Sync { +public: + struct Config { + std::string redisHost; + int redisPort; + std::string tradingEnv; // names the Auth# session, log-only here + }; + + Sync(Config config, AuthProvider auth) + : config_(std::move(config)), auth_(std::move(auth)), + store_(config_.redisHost, config_.redisPort), + clusters_(config_.redisHost, config_.redisPort) {} + + Sync(const Sync&) = delete; + Sync& operator=(const Sync&) = delete; + + // One full cron cycle, in the C# order: fetch -> save/update each + // matched position -> cluster rebuild -> Success report -> PL# refresh. + // Any fetch-stage failure (no session, HTTP failure, undecodable body) + // logs, files a FAILED-* report and returns without touching Redis — + // the C# threw there, and the refresh pass never ran on a failed fetch. + // Never throws; a failed cycle is retried by the next minute's. + void syncOnce(); + +private: + void applyPosition(const IgPosition& position, + std::vector& members); + + Config config_; + AuthProvider auth_; + redis_positions::PositionManager store_; + redis_clusters::PositionClustering clusters_; + + // Per-cycle counters for the summary line, reset each syncOnce(). + std::size_t matched_{}; + std::size_t fresh_{}; + std::size_t updated_{}; + std::size_t skippedUnknown_{}; +}; + +} // namespace positions + +namespace positions { + +namespace { + +// The C# shared.Elastic.LiveFunctionReport: {function, action, status, +// details, date} into live-function-logs, queued for the publisher's +// background flusher so an Elastic outage never stalls the sync loop. +void reportFunction(const std::string& function, const std::string& action, + const std::string& status, std::string details) { + nlohmann::json doc{ + {"function", function}, {"action", action}, + {"status", status}, {"details", std::move(details)}, + {"date", elastic::nowIsoUtc()}, + }; + elastic::enqueueDocument("live-function-logs", doc.dump()); +} + +void readString(const nlohmann::json& object, const char* key, + std::string& out) { + if (const auto it = object.find(key); + it != object.end() && it->is_string()) { + out = it->get(); + } +} + +void readNumber(const nlohmann::json& object, const char* key, double& out) { + // null (IG's absent stop/limit) and missing both leave the 0 default. + if (const auto it = object.find(key); + it != object.end() && it->is_number()) { + out = it->get(); + } +} + +std::int64_t nowEpochMicros() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +} // namespace + +std::optional> decodeAccountPositions( + const std::string& json) { + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(json); + } catch (const std::exception&) { + return std::nullopt; + } + if (!parsed.is_object()) { + return std::nullopt; + } + const auto positionsIt = parsed.find("positions"); + if (positionsIt == parsed.end() || positionsIt->is_null()) { + return std::vector{}; // an empty book, not an error + } + if (!positionsIt->is_array()) { + return std::nullopt; + } + std::vector book; + book.reserve(positionsIt->size()); + for (const auto& entry : *positionsIt) { + if (!entry.is_object()) { + return std::nullopt; // the C# deserializer would throw here + } + IgPosition position; + if (const auto market = entry.find("market"); + market != entry.end() && market->is_object()) { + readString(*market, "epic", position.epic); + } + if (const auto detail = entry.find("position"); + detail != entry.end() && detail->is_object()) { + readString(*detail, "dealReference", position.dealReference); + readString(*detail, "dealId", position.dealId); + readString(*detail, "direction", position.direction); + readString(*detail, "createdDate", position.createdDate); + readNumber(*detail, "dealSize", position.dealSize); + readNumber(*detail, "openLevel", position.openLevel); + readNumber(*detail, "stopLevel", position.stopLevel); + readNumber(*detail, "limitLevel", position.limitLevel); + } + book.push_back(std::move(position)); + } + return book; +} + +std::int64_t parseCreatedDateMicros(const std::string_view createdDate, + const std::int64_t fallbackMicros) { + // "yyyy/MM/dd HH:mm:ss:fff" — fixed width, exact separators (the C# + // ParseExact contract; anything else falls back). + if (createdDate.size() != 23 || createdDate[4] != '/' + || createdDate[7] != '/' || createdDate[10] != ' ' + || createdDate[13] != ':' || createdDate[16] != ':' + || createdDate[19] != ':') { + return fallbackMicros; + } + const auto readInt = [createdDate](const std::size_t pos, + const std::size_t len, int& out) { + const char* first = createdDate.data() + pos; + const auto [ptr, ec] = std::from_chars(first, first + len, out); + return ec == std::errc{} && ptr == first + len; + }; + int year = 0; + int month = 0; + int day = 0; + int hour = 0; + int minute = 0; + int second = 0; + int millis = 0; + if (!readInt(0, 4, year) || !readInt(5, 2, month) || !readInt(8, 2, day) + || !readInt(11, 2, hour) || !readInt(14, 2, minute) + || !readInt(17, 2, second) || !readInt(20, 3, millis)) { + return fallbackMicros; + } + const std::chrono::year_month_day date{ + std::chrono::year{year}, std::chrono::month{static_cast(month)}, + std::chrono::day{static_cast(day)}}; + if (!date.ok() || hour > 23 || minute > 59 || second > 59) { + return fallbackMicros; + } + const auto instant = std::chrono::sys_days{date} + std::chrono::hours{hour} + + std::chrono::minutes{minute} + + std::chrono::seconds{second} + + std::chrono::milliseconds{millis}; + return std::chrono::duration_cast( + instant.time_since_epoch()) + .count(); +} + +std::chrono::milliseconds updateTtl(const std::chrono::sys_seconds nowUtc) { + const auto day = std::chrono::floor(nowUtc); + const std::chrono::weekday weekday{day}; + const std::chrono::hh_mm_ss timeOfDay{nowUtc - day}; + if (weekday == std::chrono::Friday + && timeOfDay.hours() == std::chrono::hours{21} + && timeOfDay.minutes() >= std::chrono::minutes{55}) { + return std::chrono::days{2} + std::chrono::hours{2}; + } + return std::chrono::minutes{10}; +} + +redis_positions::PositionRecord makeFreshRecord( + const IgPosition& position, const std::string_view symbol, + const std::string_view strategyId, const std::string_view strategyName, + const int priceScale, const std::int64_t nowMicros) { + const auto scalePrice = [priceScale](const double level) -> std::int32_t { + if (priceScale <= 0) { + return 0; // no scale — level 0, as the C# wrote (caller warns) + } + // llround = the C# Math.Round(..., MidpointRounding.AwayFromZero). + return static_cast(std::llround(level * priceScale)); + }; + redis_positions::PositionRecord record; + record.dealId = position.dealId; + record.dealReference = position.dealReference; + record.symbol = std::string{symbol}; + record.epic = position.epic; + record.direction = position.direction; + record.size = position.dealSize; + record.level = scalePrice(position.openLevel); + record.stopLevel = scalePrice(position.stopLevel); + record.limitLevel = scalePrice(position.limitLevel); + record.strategyId = std::string{strategyId}; + record.strategyName = std::string{strategyName}; + record.openedAtMicros = + parseCreatedDateMicros(position.createdDate, nowMicros); + return record; +} + +void applyBrokerUpdate(redis_positions::PositionRecord& record, + const IgPosition& position, + const std::string_view strategyId) { + if (record.strategyId.empty() || record.strategyId == "Unknown") { + record.strategyId = std::string{strategyId}; + } + if (!position.dealId.empty()) { + record.dealId = position.dealId; + } +} + +void Sync::applyPosition(const IgPosition& position, + std::vector& members) { + using backtest_log::logLine; + + const live::MarketDefinition* market = + live::findMarketByEpicMini(position.epic); + if (market == nullptr || position.dealReference.empty()) { + return; // not an epic this engine trades — the C# match skip + } + ++matched_; + const std::string symbol{market->symbol}; + + // Strategy attribution from the deal receipt the order channel wrote + // when it opened the deal; a deal opened outside the engine (or whose + // receipt expired) attributes to "Unknown", like the C# coalesce. + std::string strategyId = "Unknown"; + std::string strategyName = "Unknown"; + if (const auto receiptJson = + store_.getDealReceipt(position.dealReference, symbol)) { + if (const auto receipt = + redis_positions::decodeDealReceipt(*receiptJson)) { + if (!receipt->strategyId.empty()) { + strategyId = receipt->strategyId; + } + if (!receipt->strategyName.empty()) { + strategyName = receipt->strategyName; + } + } + } + + // Collected for the cluster rebuild BEFORE the store round trips: the + // deal is open at the broker, so it occupies its clusters whatever + // happens to its PO# record below — omitting it would let the entry + // gate under-count a cluster for the sets' 5-minute TTL. (The C# never + // faced this: its Redis failures killed the whole cron before + // SyncAllClusters ran.) + members.push_back(redis_clusters::ClusterMember{ + .symbol = symbol, + .strategyName = strategyName, + .dealReference = position.dealReference}); + + const auto payload = store_.getPositionPayload(position.dealReference); + if (!payload) { + // Redis state UNKNOWN: rebuilding from scratch could stamp a live, + // fully-attributed record with an Unknown one — sit this deal out + // and let the next cycle retry. + ++skippedUnknown_; + return; + } + + std::optional record; + if (payload->has_value()) { + // Undecodable/incomplete payloads fall through to the fresh-save + // rebuild, the same rule as the C# DoesThisPositionAlreadyExist. + record = redis_positions::decodePositionRecord(**payload); + } + + if (record) { + applyBrokerUpdate(*record, position, strategyId); + const auto now = std::chrono::floor( + std::chrono::system_clock::now()); + store_.savePosition(position.dealReference, + redis_positions::encodePositionRecord(*record), + updateTtl(now)); + store_.addPosition(record->strategyId, record->dealReference); + ++updated_; + } else { + const int priceScale = symbol_scale::getPriceScale(symbol); + if (priceScale <= 0) { + logLine("PositionSync: WARNING — no price scale for {}, writing " + "level 0", + symbol); + } + const redis_positions::PositionRecord freshRecord = + makeFreshRecord(position, symbol, strategyId, strategyName, + priceScale, nowEpochMicros()); + const std::string payloadJson = + redis_positions::encodePositionRecord(freshRecord); + store_.savePosition(position.dealReference, payloadJson, + kFreshPositionTtl); + store_.addPosition(freshRecord.strategyId, freshRecord.dealReference); + logLine("PositionSync: new position found! {}", payloadJson); + reportFunction("PositionRequest", "Position Update", "New", + payloadJson); + ++fresh_; + } +} + +void Sync::syncOnce() { + using backtest_log::logLine; + matched_ = fresh_ = updated_ = skippedUnknown_ = 0; + + // 1. Session + fetch. Every failure here files the same FAILED-* report + // the C# did and abandons the cycle — refresh must never run against a + // book we could not read (an unreadable book is not an empty one). + const std::optional auth = auth_(); + if (!auth) { + logLine("PositionSync: no IG session (Auth#{}) — skipping this cycle", + config_.tradingEnv); + reportFunction("IG-Account", "PositionRequest", "FAILED-REQUEST", + nlohmann::json{{"responseWasNull", true}, + {"reason", "no IG session credentials"}} + .dump()); + return; + } + const std::optional response = + ig_rest::execute(*auth, "/positions", "GET", ""); + if (!response || response->status != 200) { + logLine("PositionSync: /positions request failed (status={}) — " + "skipping this cycle", + response ? response->status : 0); + reportFunction( + "IG-Account", "PositionRequest", "FAILED-REQUEST", + nlohmann::json{{"statusCode", response ? response->status : 500}, + {"responseWasNull", !response.has_value()}, + {"content", + response ? response->body : "No response content"}} + .dump()); + return; + } + + // 2. Decode. + const auto book = decodeAccountPositions(response->body); + if (!book) { + logLine("PositionSync: /positions body would not decode — skipping " + "this cycle"); + reportFunction("IG-Account", "PositionRequest", "FAILED-DESERIALIZE", + nlohmann::json{{"statusCode", response->status}, + {"content", response->body}} + .dump()); + return; + } + + // 3. Save/update each matched deal, collecting the cluster members. + std::vector members; + members.reserve(book->size()); + for (const IgPosition& position : *book) { + applyPosition(position, members); + } + + // 4. Rebuild the CG# sets (non-empty clusters only — see syncAllClusters). + const bool clustersSynced = clusters_.syncAllClusters(members); + + // 5. The Success report carries the raw broker book, the ground-truth + // snapshot the C# stored (re-serialised there, verbatim here). + reportFunction("IG-Account", "PositionRequest", "Success", response->body); + + // 6. Refresh every PL# list so deals whose PO# expired fall out. The C# + // walked its deployed-strategies config; the keyspace is this engine's + // equivalent (and also prunes lists for strategies no longer deployed). + std::size_t refreshed = 0; + bool scanOk = false; + if (const auto strategyIds = store_.listStrategyIds()) { + scanOk = true; + for (const std::string& strategyId : *strategyIds) { + if (store_.refreshPositionList(strategyId)) { + ++refreshed; + } + } + } else { + logLine("PositionSync: PL#* scan failed — skipping the refresh pass " + "this cycle"); + } + + logLine("PositionSync: cycle complete (positions={} matched={} fresh={} " + "updated={} skippedUnknown={} clusterMembers={} clustersSynced={} " + "listsRefreshed={}{})", + book->size(), matched_, fresh_, updated_, skippedUnknown_, + members.size(), clustersSynced, refreshed, + scanOk ? "" : " REFRESH-SKIPPED"); +} + +} // namespace positions diff --git a/source/positions/positionsCommand.cppm b/source/positions/positionsCommand.cppm new file mode 100644 index 0000000..bdb916c --- /dev/null +++ b/source/positions/positionsCommand.cppm @@ -0,0 +1,125 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// positionsCommand — the `positions` subcommand: the IG position producer, +// replacing the external C# igmarkets_positions cron. Where the cron ran +// once a minute under an external scheduler, this process loops forever and +// paces itself: the first sync fires immediately (cron parity), then one +// cycle per minute, each cycle mirroring the broker's /positions book into +// Redis (PO#/PL#/CG#) via positionSync. +// +// Loop shape: the deadline is re-armed AFTER each cycle (next = now + 1min, +// cron semantics) so a slow IG exchange — worst case ~90s with retries — +// never causes a catch-up burst; the wait sleeps in 1-second ticks so +// SIGINT/SIGTERM (a plain signal flag; no UDP receiver owns the signals +// here) is honoured within ~1s. Deliberately NOT std::jthread + stop_token: +// its condition_variable_any wait fails to link under `import std` (see the +// module-migration notes / liveReporter). +// +// `positions --once` runs a single cycle and exits — the literal C# cron +// behaviour, and the smoke-test mode. + +module; + +#include // SIGINT/SIGTERM are macros — `import std` can't supply them + +#include "run/reporting/elasticPublisher.hpp" +#include "shared/utilities/env.hpp" + +export module positionsCommand; + +import std; +import backtestLog; // backtest_log::logLine +import igRequests; // ig::dynamoAuthProvider — Auth# from DynamoDB +import positionSync; // positions::Sync — one producer cycle + +export class PositionsCommand { +public: + static int run(int argc, const char* argv[]); +}; + +namespace { + +constexpr std::chrono::minutes kInterval{1}; + +// Written from the signal handler: atomic is lock-free on the deploy +// targets, which makes the relaxed store async-signal-safe. +std::atomic stopRequested{false}; + +void onSignal(int) { stopRequested.store(true, std::memory_order_relaxed); } + +} // namespace + +int PositionsCommand::run(const int argc, const char* argv[]) { + using backtest_log::logLine; + + const std::string redisHost = env::getOr("REDIS_HOST", "127.0.0.1"); + constexpr int redisPort = 6379; // by convention, as liveSettings + std::string tradingEnv = env::getOr("TRADING_ENVIRONMENT", "demo"); + std::ranges::transform(tradingEnv, tradingEnv.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + + const bool runOnce = + argc >= 3 && std::string_view{argv[2]} == "--once"; + + // One probe pull at startup, loud like liveCommand: without a session + // every cycle files a FAILED-REQUEST report and touches nothing — safe, + // but the operator should hear it before the first minute, not from + // Kibana. Auth is re-pulled per cycle, so a later login-service refresh + // is picked up without a restart. + const ig::AuthProvider auth = ig::dynamoAuthProvider(tradingEnv); + const bool haveAuth = auth().has_value(); + if (!haveAuth) { + logLine("PositionsCommand: WARNING — could not pull IG session " + "Auth#{} from DynamoDB ({} table); every cycle will skip " + "until a session appears (check AWS credentials in the " + "environment and the login service)", + tradingEnv, "MarketDataLive"); + } + + positions::Sync sync( + positions::Sync::Config{.redisHost = redisHost, + .redisPort = redisPort, + .tradingEnv = tradingEnv}, + auth); + + logLine("PositionsCommand: starting; IG /positions -> Redis PO#/PL#/CG# " + "{} ([{}]{}, redis {}:{})", + runOnce ? "once (--once)" + : std::format("every {}s", std::chrono::seconds{kInterval} + .count()), + tradingEnv, haveAuth ? "" : " WITHOUT a session", redisHost, + redisPort); + + std::signal(SIGINT, onSignal); + std::signal(SIGTERM, onSignal); + + std::uint64_t cycles = 0; + for (;;) { + sync.syncOnce(); // never throws; a failed cycle waits for the next + ++cycles; + if (runOnce) { + break; + } + // A signal during a cycle (the IG exchange can run tens of seconds) + // is honoured here, before any wait. + const auto next = std::chrono::steady_clock::now() + kInterval; + while (!stopRequested.load(std::memory_order_relaxed) + && std::chrono::steady_clock::now() < next) { + std::this_thread::sleep_for(std::chrono::seconds{1}); + } + if (stopRequested.load(std::memory_order_relaxed)) { + break; + } + } + + logLine("PositionsCommand: shutting down (cycles={})", cycles); + // Deliver the reporting tail now, deterministically, rather than leaving + // it to the publisher's exit-time flush. + elastic::flushQueuedDocuments(); + return 0; +} diff --git a/source/run/execution/backtestRunner.cppm b/source/run/execution/backtestRunner.cppm index 1e68fd8..dee83d0 100644 --- a/source/run/execution/backtestRunner.cppm +++ b/source/run/execution/backtestRunner.cppm @@ -6,55 +6,93 @@ module; -#include "shared/utilities/env.hpp" #include "shared/tradingDefinitions/config/configuration.hpp" export module backtestRunner; -import std; // replaces , , , +import std; // replaces , , , , import priceData; // PriceData import operations; // Operations import sqlManager; // SqlManager +import connectionFactory; // questdb::connectionFromEnv import databaseConnection; // DatabaseConnection +import tickCache; // tick_cache::Superset -// Pulls all tick data for the run's symbols/window out of QuestDB. Expensive — -// call once per run and reuse the result across that run's strategies. -export std::vector loadTicks(const std::string& questdbHost, - const std::string& symbolsCsv, - int lastMonths) { +namespace { - // QuestDB pg-wire port: defaults to 8812 but is overridable via QUESTDB_PORT - // so a non-default instance (e.g. a converted dataset) can be targeted - // without a rebuild. - const int questdbPort = std::stoi(env::getOr("QUESTDB_PORT", "8812")); - DatabaseConnection db(questdbHost, questdbPort, "qdb", "admin", "quest"); - - // Get a list of symbols +// Split a comma-separated symbols string into the list SqlManager expects. +std::vector splitSymbols(const std::string& symbolsCsv) { std::vector symbols; std::istringstream ss(symbolsCsv); for (std::string token; std::getline(ss, token, ',');) { symbols.push_back(token); } + return symbols; +} + +} // namespace + +// Pulls all tick data for the run's symbols/window out of QuestDB. The window +// is lastMonths long and ends offsetMonths before now (0 = the present day). +// Expensive — call once per run and reuse the result across that run's +// strategies. +export std::vector loadTicks(const std::string& questdbHost, + const std::string& symbolsCsv, + int lastMonths, + int offsetMonths) { + + // Host comes from argv; the port (default 8812, overridable via + // QUESTDB_PORT) and credentials come from the shared factory, so a + // non-default instance (e.g. a converted dataset) can be targeted without a + // rebuild. + const DatabaseConnection db = questdb::connectionFromEnv(questdbHost); // Get all the tick data out of QuestDB for these symbols - std::vector ticks = SqlManager::loadPriceData(db, symbols, lastMonths); + std::vector ticks = + SqlManager::loadPriceData(db, splitSymbols(symbolsCsv), lastMonths, offsetMonths); std::printf("Total ticks streamed: %zu\n", ticks.size()); return ticks; } +// Pulls `months` of history ending at one QuestDB-computed snapshot instant T, +// with the month boundaries that produced it: the boundary row is fetched +// first (QuestDB's own dateadd semantics, one now() per statement), then the +// ticks are loaded with literal bounds from that row, so the data and the +// boundaries used to slice it can never disagree. One superset serves every +// rolling-ladder window as a contiguous slice — see tick_cache. +export tick_cache::Superset loadTickSuperset(const std::string& questdbHost, + const std::string& symbolsCsv, + int months) { + const DatabaseConnection db = questdb::connectionFromEnv(questdbHost); + const std::vector symbols = splitSymbols(symbolsCsv); + + auto boundaries = SqlManager::loadMonthBoundaries(db, months); + std::vector ticks = SqlManager::loadPriceDataBetween( + db, symbols, boundaries[static_cast(months)], boundaries[0]); + + std::printf("Superset loaded: %zu ticks, %d months ending %s\n", + ticks.size(), months, + SqlManager::formatTimestamp(boundaries[0]).c_str()); + + return tick_cache::Superset{std::move(ticks), std::move(boundaries)}; +} + // Runs one backtest against already-loaded ticks (no QuestDB access). -export void runBacktestOnTicks(const std::vector& ticks, - const tradingDefinitions::Configuration& config) { - Operations::run(ticks, config); +// chainWindows opts the run into the rolling-window ladder (queue path only — +// see Operations::run). +export void runBacktestOnTicks(std::span ticks, + const tradingDefinitions::Configuration& config, + bool chainWindows = false) { + Operations::run(ticks, config, chainWindows); } // Convenience for the direct path: loads ticks then runs a single backtest. export int runBacktest(const std::string& questdbHost, const tradingDefinitions::Configuration& config) { const std::vector ticks = - loadTicks(questdbHost, config.SYMBOLS, config.LAST_MONTHS); + loadTicks(questdbHost, config.SYMBOLS, config.LAST_MONTHS, config.OFFSET_MONTHS); runBacktestOnTicks(ticks, config); return 0; } diff --git a/source/run/execution/runnerBridge.cpp b/source/run/execution/runnerBridge.cpp index 8fa0f5e..8ac3a3a 100644 --- a/source/run/execution/runnerBridge.cpp +++ b/source/run/execution/runnerBridge.cpp @@ -6,26 +6,84 @@ #include "run/execution/runnerBridge.hpp" +#include "shared/utilities/env.hpp" + import std; import priceData; // PriceData -import backtestRunner; // loadTicks, runBacktestOnTicks +import tickCache; // tick_cache::TickCache, SliceView, sliceForWindow +import backtestRunner; // loadTickSuperset, loadTicks, runBacktestOnTicks +import rollingWindow; // rolling::kFullHistory -// The tick buffer the opaque handle wraps. Kept out of redisRunner.cpp so that -// TU never has to name PriceData (a module type) or import a module — see -// runnerBridge.hpp for why that matters. -struct LoadedTicks { - std::vector ticks; +// The tick window the opaque handle wraps. Kept out of redisRunner.cpp so that +// TU never has to name a module type or import a module — see runnerBridge.hpp +// for why that matters. The SliceView's shared_ptr keeps the whole superset +// alive while any task still holds a copy of this slice. +struct TickSliceImpl { + tick_cache::SliceView view; }; -std::shared_ptr bridgeLoadTicks(const std::string& questdbHost, - const std::string& symbolsCsv, - int lastMonths) { - auto loaded = std::make_shared(); - loaded->ticks = loadTicks(questdbHost, symbolsCsv, lastMonths); - return loaded; +namespace { + +// Positive integer env override, engine-fatal on junk: a mistyped cache knob +// should fail the worker at startup, not silently run with a default. +int envInt(const char* name, const std::string& fallback) { + const std::string raw = env::getOr(name, fallback); + int value = 0; + const auto [ptr, ec] = std::from_chars(raw.data(), raw.data() + raw.size(), value); + if (ec != std::errc{} || ptr != raw.data() + raw.size() || value < 0) { + throw std::runtime_error(std::string("Invalid ") + name + ": " + raw); + } + return value; +} + +// The worker's one cache instance. Function-local static so construction (env +// reads) happens on first use, after main() has the environment set up. +// Single-threaded by construction: bridgeGetTicks is only ever called from the +// one drain coroutine on the io_context thread; pool threads only read the +// immutable Supersets through their own shared_ptr copies, never the cache. +tick_cache::TickCache& cacheFor(const std::string& questdbHost) { + static tick_cache::TickCache cache( + // Superset loads capture the host by value — it is fixed per worker + // process (argv), so one cache serves every run the worker claims. + [host = questdbHost](const std::string& symbolsCsv, int months) { + return loadTickSuperset(host, symbolsCsv, months); + }, + // Ad-hoc loads (windows too deep for the superset) keep the original + // now()-relative single-window behavior. + [host = questdbHost](const std::string& symbolsCsv, int lastMonths, + int offsetMonths) { + return loadTicks(host, symbolsCsv, lastMonths, offsetMonths); + }, + std::chrono::minutes(envInt("TICK_CACHE_TTL_MINUTES", "60")), + static_cast(envInt("TICK_CACHE_MAX_SUPERSETS", "1")), + // The ladder's deepest window is the terminal full-history run, so a + // default superset serves every rung. Single source of truth for the + // depth: rollingWindow's kFullHistory. + rolling::kFullHistory.lastMonths); + return cache; +} + +} // namespace + +TickSlice bridgeGetTicks(const std::string& questdbHost, + const std::string& symbolsCsv, + const int lastMonths, + const int offsetMonths, + const std::function& beforeLoad) { + auto impl = std::make_shared(TickSliceImpl{ + cacheFor(questdbHost).get(symbolsCsv, lastMonths, offsetMonths, beforeLoad)}); + const std::size_t count = impl->view.count; + return TickSlice{std::move(impl), count}; } -void bridgeRunOnTicks(const LoadedTicks& ticks, +void bridgeRunOnTicks(const TickSlice& slice, const tradingDefinitions::Configuration& config) { - runBacktestOnTicks(ticks.ticks, config); + const tick_cache::SliceView& view = slice.impl->view; + const std::span ticks = + std::span(view.superset->ticks).subspan(view.begin, view.count); + // The bridge is the Redis-queue path by construction (redisRunner is its + // only consumer), so queue runs always participate in the rolling-window + // ladder; direct `run ` invocations bypass the bridge and + // never chain. + runBacktestOnTicks(ticks, config, /*chainWindows=*/true); } diff --git a/source/run/execution/runnerBridge.hpp b/source/run/execution/runnerBridge.hpp index 71b0e7e..2cd8a33 100644 --- a/source/run/execution/runnerBridge.hpp +++ b/source/run/execution/runnerBridge.hpp @@ -4,27 +4,43 @@ // This code is licensed under MIT license (see LICENSE.txt for details) // --------------------------------------- #pragma once +#include +#include #include #include #include "shared/tradingDefinitions/config/configuration.hpp" -// Opaque handle to a run's loaded tick data, defined in runnerBridge.cpp. +// Opaque handle to a run's tick window, defined in runnerBridge.cpp. // -// runnerBridge.cpp is the import boundary to the backtestRunner module. -// redisRunner.cpp must stay a *purely textual* TU: its ThreadPool instantiates -// std::condition_variable_any::wait(lock, stop_token, pred), whose libc++ -// internal helper (__atomic_unique_lock::__set_locked_bit) the toolchain fails -// to emit when the same TU also imports a module. So redisRunner.cpp reaches -// the module only through these plain (global-module) functions, never naming -// PriceData or importing anything itself. -struct LoadedTicks; +// runnerBridge.cpp is the import boundary to the backtestRunner/tickCache +// modules. redisRunner.cpp must stay a *purely textual* TU: its ThreadPool +// instantiates std::condition_variable_any::wait(lock, stop_token, pred), +// whose libc++ internal helper (__atomic_unique_lock::__set_locked_bit) the +// toolchain fails to emit when the same TU also imports a module. So +// redisRunner.cpp reaches the modules only through these plain (global-module) +// functions, never naming PriceData or importing anything itself. +struct TickSliceImpl; -// Pulls a run's ticks out of QuestDB once; the handle is shared so tasks can -// hold it for the lifetime of their backtest. -std::shared_ptr bridgeLoadTicks(const std::string& questdbHost, - const std::string& symbolsCsv, - int lastMonths); +// A window of cached tick data. Value-copyable without naming any module type: +// pool tasks capture it BY VALUE, so the shared_ptr keeps the underlying tick +// buffer alive for the lifetime of each backtest even after the cache evicts +// or replaces it. +struct TickSlice { + std::shared_ptr impl; + std::size_t tickCount = 0; // logging only +}; -// Runs a single backtest against already-loaded ticks (no QuestDB access). -void bridgeRunOnTicks(const LoadedTicks& ticks, +// The ticks for one run's window, served from a per-symbols cached superset +// (loaded once, then sliced for every window that fits — the rolling ladder's +// windows all do). beforeLoad is invoked exactly once, immediately before any +// actual QuestDB load, and never on a cache hit: the drain loop quiesces its +// pool there so no in-flight backtest still reads a buffer being evicted. +TickSlice bridgeGetTicks(const std::string& questdbHost, + const std::string& symbolsCsv, + int lastMonths, + int offsetMonths, + const std::function& beforeLoad); + +// Runs a single backtest against an already-loaded slice (no QuestDB access). +void bridgeRunOnTicks(const TickSlice& slice, const tradingDefinitions::Configuration& config); diff --git a/source/run/execution/tickCache.cppm b/source/run/execution/tickCache.cppm new file mode 100644 index 0000000..69cab80 --- /dev/null +++ b/source/run/execution/tickCache.cppm @@ -0,0 +1,226 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module tickCache; + +import std; // replaces , , , , + // , , +import priceData; // PriceData + +// Cross-run tick reuse for the queue worker. The rolling-window ladder turns +// every surviving strategy into its own single-strategy run, so without a +// cache each offset window pays a full QuestDB load per strategy. But the +// ladder's windows all nest inside one months-deep history ending at a common +// snapshot, and ticks are time-ordered — so one superset load serves every +// window as a contiguous slice found by binary search. +export namespace tick_cache { + +// One loaded block of tick history for a symbols set: the ticks span +// [boundaries[months], boundaries[0]) where boundaries[m] is QuestDB's own +// dateadd('M', -m, now()) at load time (index 0 = the snapshot instant T). +// Boundaries travel with the ticks so slicing can never disagree with the +// query that produced them. Immutable once published — pool tasks read it +// concurrently through shared_ptr copies. +struct Superset { + std::vector ticks; + std::vector boundaries; +}; + +// A window of a superset: ticks[begin, begin+count). Holds the superset alive, +// so a task can outlive the cache entry it was sliced from. +struct SliceView { + std::shared_ptr superset; + std::size_t begin = 0; + std::size_t count = 0; +}; + +// The window (lastMonths, offsetMonths) as a slice of `superset`: +// [boundaries[last+offset], boundaries[offset]), matching the SQL semantics +// `timestamp >= lower AND timestamp < upper`. lower_bound gives the first tick +// >= a boundary, which is simultaneously the inclusive start at the lower +// bound and the exclusive end at the upper. offset == 0 keeps the upper end +// open (the superset's own end), mirroring loadPriceData leaving the upper +// bound off entirely for unshifted windows. +// Throws std::logic_error when the window reaches deeper than the superset — +// callers are expected to have checked fit and fallen back to an ad-hoc load. +SliceView sliceForWindow(std::shared_ptr superset, + int lastMonths, + int offsetMonths) { + if (lastMonths < 1 || offsetMonths < 0) { + throw std::logic_error("sliceForWindow: degenerate window"); + } + const auto depth = static_cast(lastMonths) + + static_cast(offsetMonths); + if (depth >= superset->boundaries.size()) { + throw std::logic_error("sliceForWindow: window reaches beyond the superset"); + } + + const auto byTimestamp = [](const PriceData& tick, + const std::chrono::system_clock::time_point boundary) { + return tick.timestamp < boundary; + }; + const auto& ticks = superset->ticks; + const auto lower = std::lower_bound(ticks.begin(), ticks.end(), + superset->boundaries[depth], byTimestamp); + const auto upper = offsetMonths == 0 + ? ticks.end() + : std::lower_bound(lower, ticks.end(), + superset->boundaries[static_cast(offsetMonths)], + byTimestamp); + + return SliceView{ + .superset = std::move(superset), + .begin = static_cast(lower - ticks.begin()), + .count = static_cast(upper - lower), + }; +} + +// Keyed superset cache with TTL and size-capped eviction. Loading is injected +// (SupersetLoader for cacheable superset pulls, WindowLoader for one-off +// windows too deep to fit) so the machinery tests without QuestDB; NowFn is +// injected so TTL expiry tests without sleeping. +// +// NOT thread-safe by design: the one drain coroutine is the only caller. +// Pool threads never touch the cache — they only hold shared_ptr copies of +// the immutable Supersets it hands out. +class TickCache { +public: + using SupersetLoader = std::function; + using WindowLoader = std::function(const std::string& symbolsCsv, + int lastMonths, + int offsetMonths)>; + using NowFn = std::function; + + TickCache(SupersetLoader supersetLoader, + WindowLoader windowLoader, + std::chrono::seconds ttl, + std::size_t maxSupersets, + int defaultMonths, + NowFn now = [] { return std::chrono::steady_clock::now(); }) + : supersetLoader_(std::move(supersetLoader)), + windowLoader_(std::move(windowLoader)), + ttl_(ttl), + maxSupersets_(std::max(1, maxSupersets)), + defaultMonths_(defaultMonths), + now_(std::move(now)) {} + + // The ticks for one run's window. beforeLoad() is invoked exactly once, + // immediately before ANY QuestDB load (superset miss, TTL reload, or + // ad-hoc window) and never on a cache hit — the drain loop quiesces its + // pool there so no in-flight backtest still reads a buffer being evicted. + SliceView get(const std::string& symbolsCsv, + const int lastMonths, + const int offsetMonths, + const std::function& beforeLoad = {}) { + const auto callBeforeLoad = [&] { + if (beforeLoad) { + beforeLoad(); + } + }; + + // Degenerate windows go straight to the ad-hoc loader (current + // now()-relative behavior), untracked. A window deeper than a FRESH + // resident superset is also served ad hoc — a hand-queued one-off must + // not evict the ladder's resident superset — but on a plain miss the + // new superset simply grows to fit (max below). + const bool cacheable = lastMonths >= 1 && offsetMonths >= 0; + const int monthsNeeded = cacheable ? lastMonths + offsetMonths : 0; + + if (cacheable) { + if (const auto it = supersets_.find(symbolsCsv); it != supersets_.end()) { + const Entry& entry = it->second; + const bool fresh = now_() - entry.loadedAt < ttl_; + const bool fits = + static_cast(monthsNeeded) < entry.superset->boundaries.size(); + if (fresh && fits) { + ++hits_; + auto slice = sliceForWindow(entry.superset, lastMonths, offsetMonths); + std::println("TickCache: HIT symbols={} window=({},{}) ticks={} [{}..{}) hits={} misses={}", + symbolsCsv, lastMonths, offsetMonths, slice.count, + slice.begin, slice.begin + slice.count, hits_, misses_); + return slice; + } + if (fresh && !fits) { + // The resident superset is good, just not deep enough for + // this window — load the window ad hoc and keep the entry. + return loadAdHoc(symbolsCsv, lastMonths, offsetMonths, callBeforeLoad); + } + // Stale: fall through to reload below (evicting this entry). + } + + ++misses_; + callBeforeLoad(); + // Evict BEFORE loading: beforeLoad's quiesce guarantees nothing + // still reads the old buffers, and dropping them first keeps the + // peak at ~one superset plus the load's own parse spike. + evictFor(symbolsCsv); + const int months = std::max(defaultMonths_, monthsNeeded); + const auto loadStart = std::chrono::steady_clock::now(); + auto superset = std::make_shared( + supersetLoader_(symbolsCsv, months)); + const std::chrono::duration loadSeconds = + std::chrono::steady_clock::now() - loadStart; + std::println("TickCache: superset loaded symbols={} months={} ticks={} in {:.1f}s hits={} misses={}", + symbolsCsv, months, superset->ticks.size(), + loadSeconds.count(), hits_, misses_); + supersets_.insert_or_assign(symbolsCsv, Entry{superset, now_()}); + return sliceForWindow(std::move(superset), lastMonths, offsetMonths); + } + + return loadAdHoc(symbolsCsv, lastMonths, offsetMonths, callBeforeLoad); + } + +private: + struct Entry { + std::shared_ptr superset; + std::chrono::steady_clock::time_point loadedAt; + }; + + // A one-off load outside the superset scheme, wrapped as a whole-buffer + // slice with no boundaries (it is never re-sliced or cached). + SliceView loadAdHoc(const std::string& symbolsCsv, + const int lastMonths, + const int offsetMonths, + const std::function& callBeforeLoad) { + callBeforeLoad(); + std::println("TickCache: ad-hoc load symbols={} window=({},{})", + symbolsCsv, lastMonths, offsetMonths); + auto superset = std::make_shared( + Superset{windowLoader_(symbolsCsv, lastMonths, offsetMonths), {}}); + const std::size_t count = superset->ticks.size(); + return SliceView{std::move(superset), 0, count}; + } + + // Drop stale entries and, if the incoming key still pushes the cache over + // its cap, the oldest-loaded entries — the ladder drains rung by rung, so + // the least recently loaded key is also the least likely to recur next. + void evictFor(const std::string& incomingKey) { + std::erase_if(supersets_, [&](const auto& kv) { + return now_() - kv.second.loadedAt >= ttl_; + }); + while (supersets_.size() >= maxSupersets_ && + !supersets_.contains(incomingKey)) { + const auto oldest = std::min_element( + supersets_.begin(), supersets_.end(), [](const auto& a, const auto& b) { + return a.second.loadedAt < b.second.loadedAt; + }); + std::println("TickCache: evicting symbols={}", oldest->first); + supersets_.erase(oldest); + } + } + + SupersetLoader supersetLoader_; + WindowLoader windowLoader_; + std::chrono::seconds ttl_; + std::size_t maxSupersets_; + int defaultMonths_; + NowFn now_; + std::unordered_map supersets_; + std::size_t hits_ = 0; + std::size_t misses_ = 0; +}; + +} // namespace tick_cache diff --git a/source/run/operations.cppm b/source/run/operations.cppm index c3c5ce1..68cf663 100644 --- a/source/run/operations.cppm +++ b/source/run/operations.cppm @@ -6,76 +6,216 @@ module; +#include +#include + #include "shared/utilities/backtestLog.hpp" +#include "shared/utilities/env.hpp" +#include "shared/utilities/queueKeys.hpp" +#include "load/redisLoader.hpp" #include "run/reporting/elasticPublisher.hpp" #include "shared/tradingDefinitions/config/configuration.hpp" #include "run/reporting/tradingResults.hpp" export module operations; -import std; // replaces , , , , , - // , +import std; // replaces , , , , , + // , , +import barStore; // bars::BarStore — the run's shared bar pipeline +import entryConditions; // conditions::gateSeriesFor — the ATR gate's series +import rangeBarBuilder; // rangebar::RangeBarSpec — range-bar registrations import priceData; // PriceData import tradeManager; // TradeManager import runLoop; // trading::runTicks, RiskLimits, RunStatus import resultsSummary; // ResultsSummary import symbolScale; // symbol_scale::get import strategy; // IStrategy -import randomStrategy; // RandomStrategy -import ohlcBreakoutStrategy; // OhlcBreakoutStrategy -import strategyErrors; // UnknownStrategyError +import strategyFactory; // strategies::makeStrategy import elasticClient; // ElasticClient +import rollingWindow; // rolling::nextWindow, rolling::nextRunConfiguration export class Operations { public: - static void run(const std::vector& ticks, - const tradingDefinitions::Configuration& config); + // chainWindows opts a run into the rolling-window ladder (see + // rolling::nextWindow): true only on the Redis-queue path, so a direct + // `run ` invocation can never LPUSH new work into the + // cluster's queue as a side effect. Ticks arrive as a span so a cached + // superset can hand each window a sub-range without copying. + static void run(std::span ticks, + const tradingDefinitions::Configuration& config, + bool chainWindows = false); }; namespace { -// Adding a new strategy means adding one branch here; nothing else in -// Operations needs to know about the concrete type. -std::unique_ptr selectStrategy(const tradingDefinitions::Configuration& config) { - const auto& name = config.STRATEGY.TRADING_VARIABLES.STRATEGY; - if (name == "RandomStrategy") { - return std::make_unique(config.STRATEGY); +// Adapts one strategy to RedisLoader's chunked pull interface (built for +// whole sweeps): a single one-payload chunk, then exhausted. +class SinglePayloadSource final : public RedisLoader::ChunkSource { +public: + explicit SinglePayloadSource(RedisLoader::KeyedPayload payload) + : payload_(std::move(payload)) {} + + std::vector next() override { + if (delivered_) { + return {}; + } + delivered_ = true; + std::vector chunk; + chunk.push_back(std::move(payload_)); + return chunk; } - if (name == "OhlcBreakoutStrategy") { - return std::make_unique(config.STRATEGY); + +private: + RedisLoader::KeyedPayload payload_; + bool delivered_ = false; +}; + +// Re-queues a finished strategy as a fresh single-strategy run over `next`. +// The StrategyConfig travels byte-identical (same UUID), so a strategy can be +// followed across its windows in Elasticsearch; only the run descriptor — +// fresh RUN_ID, next window — changes. Best-effort like the outcome puts: a +// Redis failure is logged and recorded, never allowed to abort the run's +// reporting (the chain link is simply lost). +void queueNextWindow(const tradingDefinitions::Configuration& config, + const rolling::Window next) { + try { + // One persistent Redis connection for the whole process, shared by + // every pool thread. RedisLoader is single-threaded by design, so the + // mutex serialises the pushes; a re-queue is two small pipelined + // round-trips at most once per completed run, so contention is + // negligible — not worth a connection per worker thread. Both statics + // initialise lazily on first chain push, and a throwing constructor + // is retried on the next call (magic statics), landing in this same + // catch either way. + static std::mutex loaderMutex; + static RedisLoader loader(env::getOr("REDIS_HOST", "127.0.0.1"), 6379); + const std::scoped_lock loaderLock(loaderMutex); + + const auto newRunId = + boost::uuids::to_string(boost::uuids::random_generator()()); + + // Same ordering contract as loadCommand: payload key first, then its + // name on the run's strategy list, then the run advert — so a worker + // can never see the run before its strategy is claimable. + const nlohmann::json strategyJson = config.STRATEGY; + SinglePayloadSource source( + {queue_keys::strategyPayloadKey(newRunId, config.STRATEGY.UUID), + strategyJson.dump()}); + if (loader.loadKeyedPayloadStream(queue_keys::strategyKey(newRunId), + source, + queue_keys::PAYLOAD_TTL_SECONDS) != 0) { + throw std::runtime_error("strategy payload push failed"); + } + + // The advert goes on the rung's own chain queue (not RUN): workers + // drain queue_keys::RUN_QUEUES in priority order, so grid sweeps and + // earlier rungs always outrank this run. + const nlohmann::json runJson = + rolling::nextRunConfiguration(config, next, newRunId); + if (loader.loadPayload(rolling::queueKeyFor(next), runJson.dump()) != 0) { + throw std::runtime_error("run descriptor push failed"); + } + + if (!backtest_log::is_quiet()) { + std::println("Operations: window ({},{}) complete — queued window ({},{}) as RUN_ID={}", + config.LAST_MONTHS, config.OFFSET_MONTHS, + next.lastMonths, next.offsetMonths, newRunId); + } + } catch (const std::exception& e) { + backtest_log::error( + std::string("Operations: rolling-window re-queue failed: ") + + e.what()); + elastic::putEngineException( + {elastic::nowIsoUtc(), "Operations", + std::string("rolling-window re-queue failed: ") + e.what(), + config.RUN_ID}); } - throw UnknownStrategyError(name); } -} // namespace +} // namespace -void Operations::run(const std::vector& ticks, - const tradingDefinitions::Configuration& config) { +void Operations::run(const std::span ticks, + const tradingDefinitions::Configuration& config, + const bool chainWindows) { // Function-local (stack) start time: each worker thread times only its own // run. steady_clock is monotonic, the correct clock for elapsed durations. const auto runStart = std::chrono::steady_clock::now(); - TradeManager tradeManager; - auto strategy = selectStrategy(config); - // The per-tick loop (exit review -> re-entry gate -> entry -> manage) lives // in trading::runTicks so it can be driven with a deterministic strategy and // an inspectable TradeManager under test. The run-level risk limits make a // breaching run stop early (fail fast) instead of burning ticks. // The loss floor is pip-denominated; scale it to points using the run's // primary (first) symbol. Single-symbol/single-asset-class runs are exact. - const std::string primarySymbol = - config.SYMBOLS.substr(0, config.SYMBOLS.find(',')); + const std::string primarySymbol = config.SYMBOLS.substr(0, config.SYMBOLS.find(',')); + + // Multi-symbol caveat (documented, not fixed): PnL is summed in raw integer + // points across ALL of the run's symbols, but the loss floor is scaled with + // the PRIMARY symbol's points-per-pip only. Symbols sharing a scale are + // fine — EURUSD and USDJPY are both 10 points/pip (their priceScale differs, + // but that's already baked into the stored prices), as is every FX pair. + // The skew appears when a run mixes asset classes (FX 10, indices/ + // commodities 100, metals 1000): a 1-pip move on XAUUSD (1000 points) would + // count as 100 EURUSD pips in the equity check. Revisit before running + // cross-asset-class sweeps. const trading::RiskLimits riskLimits{ - .startingBalance = config.STARTING_BALANCE, - .maxLossPercent = config.MAX_LOSS_PERCENT, - .maxOpenTrades = config.MAX_OPEN_TRADES, - .pointsPerPip = symbol_scale::get(primarySymbol), + .startingBalance = config.STARTING_BALANCE, + .maxLossPercent = config.MAX_LOSS_PERCENT, + .maxOpenTrades = config.MAX_OPEN_TRADES, + .maxTradesPerMinute = config.MAX_TRADES_PER_MINUTE, + .peakHoursOnly = config.PEAK_HOURS_ONLY, + .pointsPerPip = symbol_scale::get(primarySymbol), + // Performance gate (evaluated at the bottom of runTicks): a finished + // run only counts as Completed — reporting to the results/winners index and, on + // the queue path, chaining to its next window — when its performance + // score clears 5 on more than 5 decisive trades. One lucky winner on + // a quiet window is a sample, not a strategy; it lands as + // Underperformed, recorded in the final-record index only. + .minPerformanceScore = boost::decimal::decimal64_t{5}, + .minDecisiveTrades = 5, + .lastMonths = config.LAST_MONTHS, }; + + TradeManager tradeManager; + // Slippage stress toggle: every entry this run opens pays this many + // tenths of a pip against the trade (see tradeManager.openTrade). Rides + // in the run config so the results doc records the stress it ran under. + tradeManager.entrySlippageTenthPips = config.ENTRY_SLIPPAGE_TENTH_PIPS; + auto strategy = strategies::makeStrategy(config.STRATEGY); + + // The run's shared bar pipeline: every strategy OHLC timeframe plus the + // ATR entry gate's series, registered before the first tick. {0,0} + // OHLC_VARIABLES entries are the documented "builds no bars" sentinel + // (RandomStrategy) and register nothing. One store per run — per-symbol + // state inside covers multi-symbol runs. + bars::BarStore barStore; + for (const auto& ohlcVars : config.STRATEGY.OHLC_VARIABLES) { + if (ohlcVars.OHLC_MINUTES >= 1 && ohlcVars.OHLC_COUNT >= 1) { + barStore.registerSeries(std::chrono::minutes{ohlcVars.OHLC_MINUTES}, + ohlcVars.OHLC_COUNT); + } + } + // Range-bar series, same sentinel convention: an all-zeros (or partially + // zero) RANGE_VARIABLES entry registers nothing, anything negative + // already failed the config parse. + for (const auto& rangeVars : config.STRATEGY.RANGE_VARIABLES) { + if (rangeVars.RANGE_ATR_TICK_WINDOW >= 1 && + rangeVars.RANGE_ATR_PERCENT >= 1 && rangeVars.RANGE_COUNT >= 1) { + barStore.registerRangeSeries(rangebar::RangeBarSpec{ + .atrTickWindow = rangeVars.RANGE_ATR_TICK_WINDOW, + .atrPercent = rangeVars.RANGE_ATR_PERCENT, + .count = rangeVars.RANGE_COUNT}); + } + } + const bars::SeriesSpec gateSeries = + conditions::gateSeriesFor(config.STRATEGY); + barStore.registerSeries(gateSeries.minutes, gateSeries.count); + const trading::RunStatus status = trading::runTicks(tradeManager, *strategy, ticks, - config.STRATEGY.TRADING_VARIABLES, riskLimits); + config.STRATEGY.TRADING_VARIABLES, riskLimits, + &barStore, gateSeries); ResultsSummary::summarise(tradeManager, config); @@ -92,39 +232,85 @@ void Operations::run(const std::vector& ticks, if (status == trading::RunStatus::LossLimitBreached) { std::println("Operations: run RUN_ID={} stopped after {:.3f}s — account loss limit reached", config.RUN_ID, durationSeconds); + } else if (status == trading::RunStatus::Underperformed) { + std::println("Operations: run RUN_ID={} completed in {:.3f}s — below performance gate", + config.RUN_ID, durationSeconds); } else { std::println("Operations: run RUN_ID={} completed in {:.3f}s", config.RUN_ID, durationSeconds); } } - // Best-effort: persist this run's outcome to Elasticsearch — results for a - // completed run, a failure doc for one cut off by the loss limit. The + // Rolling-window chain (queue path only): a strategy that COMPLETES its + // window advances to the next one on rolling::nextWindow's ladder — each + // 3-month slice back through 9 months of history, then one final run over + // the full 9 months, after which the ladder ends. Completed already embeds + // the performance gate (score and decisive-trade thresholds on riskLimits + // above): a liquidated run does not advance, and neither does one that + // merely survived — the ladder exists to find live candidates, not to + // spend three more windows on a config the winner selection would discard. + // Off-ladder windows never chain, so a hand-queued one-off sweep stays + // one-off. + if (chainWindows && status == trading::RunStatus::Completed) { + if (const auto next = + rolling::nextWindow(config.LAST_MONTHS, config.OFFSET_MONTHS)) { + queueNextWindow(config, *next); + } + } + + // Best-effort: queue this run's outcome for Elasticsearch — results for a + // gate-cleared run, a failure doc for one cut off by the loss limit, and + // only the terminal record for one that underperformed the gate. The // backtest has already produced its summary, so nothing here may abort the - // run. The put functions log every transport/HTTP outcome themselves (so - // we do not re-log on a non-zero return, that would double the warning), - // but a throw from JSON serialisation or the network layer bypasses their - // logging, so the catch handlers emit the single warning for that path. + // run. The put functions serialise here, hand the documents to the + // publisher's background flusher and return at once, so this worker is + // free for its next backtest immediately; delivery, retries and + // dead-lettering all happen on the flusher thread (which logs its own + // failures — nothing to check here). A throw from JSON serialisation is + // still this thread's, so the catch handlers report that path. try { // Every outcome doc carries the host that produced it, so a bad node in // a distributed sweep can be traced from any of the three indices. const std::string hostname = TradeFinal::localHostname(); // Compact terminal record for this run, emitted for every run regardless - // of outcome or the REPORT_FAILURES silencer below: the outcome flag (a - // completed run is success=1; a loss-limit cutoff is success=0), how long - // it took, and the host that produced it, alongside the run config. Sent - // first so the silenced-failure early return below cannot skip it. + // of outcome or the REPORT_FAILURES silencer below: the outcome flag + // (success=1 means the performance gate was cleared, mirroring exactly + // the runs the results/winners indices receive; underperformed and liquidated runs + // are both success=0, told apart by the status keyword), how long it + // took, and the host that produced it, alongside the run config. Sent + // first so the silenced-failure early return below cannot skip it. For + // an Underperformed run this is the ONLY record: the results/winners + // indices only take gate-cleared runs (see below). + const std::string statusLabel = + status == trading::RunStatus::LossLimitBreached ? "loss_limit_breached" + : status == trading::RunStatus::Underperformed ? "underperformed" + : "completed"; const TradeFinal tradeFinal{ config.RUN_ID, TradingResults::nowIsoUtc(), durationSeconds, - status == trading::RunStatus::LossLimitBreached ? 0 : 1, + status == trading::RunStatus::Completed ? 1 : 0, + statusLabel, hostname, config, }; + ElasticClient::putTradeFinal(tradeFinal); + // Per-trade documents, opt-in via $ELASTIC_TRADES_ENABLED (default + // OFF). TradeManager already accumulated every closed trade for the + // stats summary, so the flag gates only this end-of-run serialisation + // and enqueue — nothing on the per-tick path changes either way. + // Queued before the REPORT_FAILURES silencer below so a breached run's + // trades still land when its failure doc is suppressed; like the other + // puts it sits outside the timed section and inside this best-effort + // try/catch, so it cannot affect the backtest or its duration metric. + if (env::getOr("ELASTIC_TRADES_ENABLED", "0") == "1") { + ElasticClient::bulkPutTrades(tradeManager.getClosedTrades(), + config, hostname); + } + if (status == trading::RunStatus::LossLimitBreached) { // Silencer for large sweeps: liquidated runs are expected noise once // the system is trusted, so the run config can opt out of the @@ -135,25 +321,42 @@ void Operations::run(const std::vector& ticks, } // Open trades were liquidated at their last marked prices on the // breach, so calculatePnl() is the true account PnL at cutoff. - // PnL is int64 points-per-lot; report it in pips. - const double pnlPips = riskLimits.pointsPerPip != 0 - ? static_cast(tradeManager.calculatePnl()) / riskLimits.pointsPerPip + // PnL is int64 points × trade size; normalise by both points-per-pip + // and size so the reported figure is pips of price movement. + const int tradingSize = + std::max(1, config.STRATEGY.TRADING_VARIABLES.TRADING_SIZE); + const double pnlDivisor = + static_cast(riskLimits.pointsPerPip) * tradingSize; + const double breachPnlPips = pnlDivisor != 0.0 + ? static_cast(tradeManager.calculatePnl()) / pnlDivisor : 0.0; - std::ostringstream reason; - reason << "account loss limit reached: PnL " << pnlPips << " pips breached " - << config.MAX_LOSS_PERCENT << "% of starting balance " - << config.STARTING_BALANCE << " (open trades liquidated)"; + // The budget that was breached, in pips: balance × percent/100 read + // directly as a pip count (the engine has no pip-value model — see + // the floor derivation in runLoop). + const double lossFloorPips = static_cast( + config.STARTING_BALANCE * config.MAX_LOSS_PERCENT / 100); + // `reason` stays static so reason.keyword aggregates to one value + // per failure class; the run-specific numbers go in the dedicated + // numeric fields. const TradingFailure failure{ - config.RUN_ID, - TradingResults::nowIsoUtc(), - durationSeconds, - reason.str(), - hostname, - config, - ResultsSummary::collect(tradeManager, config), + .RUN_ID = config.RUN_ID, + .timestamp = TradingResults::nowIsoUtc(), + .durationSeconds = durationSeconds, + .reason = "account loss limit reached (open trades liquidated)", + .breachPnlPips = breachPnlPips, + .lossFloorPips = lossFloorPips, + .hostname = hostname, + .config = config, + .results = ResultsSummary::collect(tradeManager, config), }; ElasticClient::putTradingFailure(failure); - } else { + } else if (status == trading::RunStatus::Completed) { + // The results/winners indices take only gate-cleared runs: they + // land there, so a document's presence means "reported AND chained + // to its next window" — exactly the population liveWinners selects + // from. An Underperformed run writes nothing beyond the terminal + // record above (and the opt-in per-trade docs): its stats are the + // noise the gate exists to keep out of the results. const TradingResults results{ config.RUN_ID, TradingResults::nowIsoUtc(), diff --git a/source/run/queue/drainRuns.cpp b/source/run/queue/drainRuns.cpp index 0547bb8..963939b 100644 --- a/source/run/queue/drainRuns.cpp +++ b/source/run/queue/drainRuns.cpp @@ -49,10 +49,22 @@ asio::awaitable drainRuns(std::shared_ptr conn, std::atomic* gauge = nullptr; try { control.emplace(); + // state() exposes the control block living inside the mapped + // segment; `&...active_jobs` takes the address of its atomic + // in-flight counter so the ThreadPool can tick it up/down where a + // monitor process can see it. A pointer (not a reference) so it + // can stay nullptr — pool skips the bookkeeping — when the + // segment failed to map. gauge = &control->state().active_jobs; } catch (const std::exception& ex) { std::println(stderr, "RedisRunner: shm control unavailable: {}", ex.what()); } + // Reusable stop-check predicate: a lambda capturing `control` by + // reference. True once a monitor has written a nonzero stop_signal + // into the shared segment (the acquire load pairs with the writer's + // release store); short-circuits to false when the segment never + // mapped. Polled at loop boundaries below — a stop never interrupts a + // backtest mid-flight, it just stops new work being claimed. const auto stopRequested = [&] { return control && control->state().stop_signal.load(std::memory_order_acquire) != 0; @@ -69,7 +81,9 @@ asio::awaitable drainRuns(std::shared_ptr conn, // Loop forever, claiming one run per iteration. An empty queue makes us // wait and re-peek (below); only an exception or a stop request blows the - // loop. + // loop. Backtests from consecutive runs PIPELINE on the pool (tasks hold + // their tick buffer by value), so single-strategy chained runs no longer + // drain the pool between claims. bool waitingLogged = false; for (;;) { // Stop requested between runs (or while idle on the empty-queue timer, @@ -79,15 +93,26 @@ asio::awaitable drainRuns(std::shared_ptr conn, stopping = true; break; } - const std::optional descriptorB64 = co_await run_queue::peekRunTail(conn); - if (!descriptorB64.has_value()) { - // Queue empty: stay alive and poll until work reappears. The timer - // is co_awaited, so this suspends (not a busy wait) while keeping - // the io_context and the Redis connection alive. + + // Surface the first infrastructure-shaped task failure from any + // still-pipelining backtest. Strategy-scoped errors are contained + // inside the tasks themselves, so anything here aborts the drain + // exactly as the old per-run check did — at worst one run later. + if (std::exception_ptr err = pool.takeError()) { + std::rethrow_exception(err); + } + const std::optional peeked = + co_await run_queue::peekRunTail(conn); + if (!peeked.has_value()) { + // Every run queue empty: stay alive and poll until work + // reappears. The timer is co_awaited, so this suspends (not a + // busy wait) while keeping the io_context and the Redis + // connection alive. if (!waitingLogged) { - std::println("RedisRunner: queue empty, waiting for work..."); + std::println("RedisRunner: run queues empty, waiting for work..."); waitingLogged = true; } + asio::steady_timer timer(co_await asio::this_coro::executor); timer.expires_after(std::chrono::seconds(1)); co_await timer.async_wait(asio::use_awaitable); @@ -95,24 +120,69 @@ asio::awaitable drainRuns(std::shared_ptr conn, } waitingLogged = false; // got a run; re-arm the idle log for next time - const tradingDefinitions::RunConfiguration runCfg = JsonParser::parseRunConfigurationFromBase64(*descriptorB64); + // A run descriptor that cannot be parsed is a poison pill: without + // this guard it would abort the worker AND stay at the tail of the + // queue to kill the next worker too. removeRun LREMs by the raw + // base64 value, so the descriptor can be retired without ever + // parsing it. + tradingDefinitions::RunConfiguration runCfg; + bool descriptorOk = true; + try { + runCfg = JsonParser::parseRunConfigurationFromBase64(peeked->descriptorB64); + } catch (const std::exception& ex) { + // co_await is illegal inside a catch handler, so the removal + // happens just below, outside the try/catch. + std::println(stderr, "RedisRunner: unparseable run descriptor, retiring: {}", + ex.what()); + elastic::putEngineException( + {elastic::nowIsoUtc(), "drainRuns", + std::string("unparseable run descriptor retired: ") + ex.what(), ""}); + descriptorOk = false; + } + if (!descriptorOk) { + co_await run_queue::removeRun(conn, peeked->queueKey, + peeked->descriptorB64); + continue; + } const std::string strategyKey = queue_keys::strategyKey(runCfg.RUN_ID); - std::println("RedisRunner: picked up RUN_ID={} SYMBOLS={} LAST_MONTHS={}", runCfg.RUN_ID, runCfg.SYMBOLS, runCfg.LAST_MONTHS); - - // Pull this run's tick data once, then reuse across every strategy. - // `ticks` is an opaque shared handle (see runnerBridge.hpp) so this - // TU never names PriceData or imports a module. - const std::shared_ptr ticks = - bridgeLoadTicks(questdbHost, runCfg.SYMBOLS, runCfg.LAST_MONTHS); - - // Backtests run on the pool but every task reads this run's `ticks` - // by reference (no copies). This guard joins all in-flight backtests - // before `ticks` is destroyed, even if a co_await below throws. - // wait() never throws, so it is safe during stack unwinding. - struct Quiesce { - ThreadPool& pool; - ~Quiesce() { pool.wait(); } - } quiesce{pool}; + + std::println("drainRuns, RUN_ID={} SYMBOLS={} LAST_MONTHS={} OFFSET_MONTHS={} QUEUE={} STRATEGY_KEY={}", + runCfg.RUN_ID, runCfg.SYMBOLS, runCfg.LAST_MONTHS, + runCfg.OFFSET_MONTHS, peeked->queueKey, strategyKey); + + // Claim the first strategy BEFORE the expensive tick load: when + // several workers converge on a nearly-drained run, the losers + // would otherwise each pay a full QuestDB fetch only to find the + // list already empty. + std::optional payloadKey = + co_await run_queue::popStrategyKey(conn, strategyKey); + if (!payloadKey.has_value()) { + // Another worker drained this run; retire it and re-peek. + // removeRun's LREM is idempotent across competing workers. + co_await run_queue::removeRun(conn, peeked->queueKey, + peeked->descriptorB64); + std::println("RedisRunner: RUN_ID={} already drained, retiring", + runCfg.RUN_ID); + continue; + } + + // This run's tick window, usually a slice of a cached superset — + // the rolling ladder's single-strategy runs all share one load. + // `slice` is an opaque value handle (see runnerBridge.hpp) so this + // TU never names PriceData or imports a module. The quiesce runs + // ONLY when a real QuestDB load is coming (cache miss/expiry): it + // joins every pipelined backtest still reading a buffer the cache + // is about to evict, and surfaces any failure they raised. Cache + // hits claim and submit without ever waiting. + const TickSlice slice = bridgeGetTicks( + questdbHost, runCfg.SYMBOLS, runCfg.LAST_MONTHS, + runCfg.OFFSET_MONTHS, + /*beforeLoad=*/[&pool] { + pool.wait(); + if (std::exception_ptr err = pool.takeError()) { + std::rethrow_exception(err); + } + }); // Drain the run's strategy list, competing with any other workers. // The list carries payload KEY NAMES: RPOP hands each name to @@ -120,21 +190,11 @@ asio::awaitable drainRuns(std::shared_ptr conn, // under it (atomically, leaving nothing behind). Redis stays on // this coroutine thread; only the CPU-bound backtest is handed to // the pool. submit() applies backpressure, so we keep popping at - // the rate the workers can absorb. + // the rate the workers can absorb. The first key was claimed above + // (before the tick load), so the loop consumes `payloadKey` first + // and pops the next one at the bottom. int strategiesRun = 0; for (;;) { - // Stop requested mid-run: stop submitting new strategies. The - // Quiesce guard / pool.wait() below still drains everything - // already in flight, and the gauge falls to 0 as it does. - if (stopRequested()) { - stopping = true; - break; - } - const std::optional payloadKey = - co_await run_queue::popStrategyKey(conn, strategyKey); - if (!payloadKey.has_value()) { - break; // strategy list drained - } const std::optional strategyB64 = co_await run_queue::takeStrategyPayload(conn, *payloadKey); if (!strategyB64.has_value()) { @@ -145,51 +205,112 @@ asio::awaitable drainRuns(std::shared_ptr conn, std::println(stderr, "RedisRunner: payload {} missing (expired?), skipping", *payloadKey); - continue; + } else { + // Reassemble the Configuration the rest of the pipeline + // expects. Parsing stays on this thread; the worker only + // runs the backtest. A payload that fails to parse or + // validate is a poison pill: report it and move on — one + // bad strategy must not abort the worker (its payload is + // already consumed, so it cannot recur). + try { + tradingDefinitions::Configuration config{ + .RUN_ID = runCfg.RUN_ID, + .SYMBOLS = runCfg.SYMBOLS, + .BATCH = runCfg.BATCH, + .EXECUTION_TS = runCfg.EXECUTION_TS, + .LAST_MONTHS = runCfg.LAST_MONTHS, + .OFFSET_MONTHS = runCfg.OFFSET_MONTHS, + .STARTING_BALANCE = runCfg.STARTING_BALANCE, + .MAX_LOSS_PERCENT = runCfg.MAX_LOSS_PERCENT, + .MAX_OPEN_TRADES = runCfg.MAX_OPEN_TRADES, + .MAX_TRADES_PER_MINUTE = runCfg.MAX_TRADES_PER_MINUTE, + .REPORT_FAILURES = runCfg.REPORT_FAILURES, + .PEAK_HOURS_ONLY = runCfg.PEAK_HOURS_ONLY, + .ENTRY_SLIPPAGE_TENTH_PIPS = + runCfg.ENTRY_SLIPPAGE_TENTH_PIPS, + .STRATEGY = JsonParser::parseStrategyFromBase64(*strategyB64), + }; + // Strategy-scoped failures inside the backtest (unknown + // strategy name, config the strategy rejects) are + // likewise contained per task: reported, and the drain + // continues. Non-std exceptions still reach the pool's + // error slot and abort at the next loop-top check — + // those are not strategy-shaped. `slice` is captured BY + // VALUE: its shared handle keeps the tick buffer alive + // for this task even after the cache moves on, which is + // what lets runs pipeline without a per-run barrier. + pool.submit([slice, cfg = std::move(config)]() { + try { + bridgeRunOnTicks(slice, cfg); + } catch (const std::exception& ex) { + std::println(stderr, + "RedisRunner: backtest failed (RUN_ID={} strategy={}): {}", + cfg.RUN_ID, cfg.STRATEGY.UUID, ex.what()); + elastic::putEngineException( + {elastic::nowIsoUtc(), "backtest", + std::string("strategy ") + cfg.STRATEGY.UUID + + " failed: " + ex.what(), + cfg.RUN_ID}); + } + }); + ++strategiesRun; + } catch (const std::exception& ex) { + std::println(stderr, + "RedisRunner: unparseable strategy payload {} (RUN_ID={}), skipping: {}", + *payloadKey, runCfg.RUN_ID, ex.what()); + elastic::putEngineException( + {elastic::nowIsoUtc(), "drainRuns", + std::string("unparseable strategy payload skipped: ") + + ex.what(), + runCfg.RUN_ID}); + } } - // Reassemble the Configuration the rest of the pipeline expects. - // Parsing stays on this thread; the worker only runs the backtest. - tradingDefinitions::Configuration config{ - .RUN_ID = runCfg.RUN_ID, - .SYMBOLS = runCfg.SYMBOLS, - .LAST_MONTHS = runCfg.LAST_MONTHS, - .STARTING_BALANCE = runCfg.STARTING_BALANCE, - .MAX_LOSS_PERCENT = runCfg.MAX_LOSS_PERCENT, - .MAX_OPEN_TRADES = runCfg.MAX_OPEN_TRADES, - .REPORT_FAILURES = runCfg.REPORT_FAILURES, - .STRATEGY = JsonParser::parseStrategyFromBase64(*strategyB64), - }; - pool.submit([&ticks, cfg = std::move(config)]() { - bridgeRunOnTicks(*ticks, cfg); - }); - ++strategiesRun; - } - - // Wait for this run's backtests to finish, then surface the first - // failure (if any) so it aborts the drain exactly as the old - // synchronous call did. - pool.wait(); - if (std::exception_ptr err = pool.takeError()) { - std::rethrow_exception(err); + // Stop requested mid-run: stop claiming new strategies. The + // Quiesce guard / pool.wait() below still drains everything + // already in flight, and the gauge falls to 0 as it does. + if (stopRequested()) { + stopping = true; + break; + } + payloadKey = co_await run_queue::popStrategyKey(conn, strategyKey); + if (!payloadKey.has_value()) { + break; // strategy list drained + } } - // Stop requested: the in-flight work has now drained. Leave this run - // in Redis (we stopped mid-list) and break out to pause — don't load - // the next run. + // Stop requested: leave this run in Redis (we stopped mid-list) and + // break out to pause — don't claim the next run. The drain-and-wait + // happens once, below the loop. if (stopping) { break; } - // Retire the run. A failure here propagates and aborts the loop, we - // never re-peek the same run and reload its ticks in a tight loop. - co_await run_queue::removeRun(conn, *descriptorB64); + // Retire the run as soon as its strategy list is drained — its + // backtests may still be pipelining on the pool, but every payload + // was already RPOP+GETDEL'd (destructively consumed), so the advert + // signals nothing claimable either way; a crash loses exactly the + // claimed-but-unfinished strategies, same as before. A removal + // failure propagates and aborts the loop, we never re-peek the same + // run and reload its ticks in a tight loop. + co_await run_queue::removeRun(conn, peeked->queueKey, + peeked->descriptorB64); - std::println("RedisRunner: completed RUN_ID={} ({} strateg{})", + std::println("RedisRunner: drained RUN_ID={} ({} strateg{} queued, backtests pipelined)", runCfg.RUN_ID, strategiesRun, strategiesRun == 1 ? "y" : "ies"); } + // The loop only breaks for a stop request (exceptions unwind past this + // point into the catch blocks below). Join every pipelined backtest + // before parking, and surface the first infrastructure failure they + // raised — matching the abort semantics the per-run barrier used to + // provide. + pool.wait(); + if (std::exception_ptr err = pool.takeError()) { + std::rethrow_exception(err); + } + // A stop signal drains the pool (above) and then parks the engine here: // alive but idle, never accepting more work. This is a one-way pause, not // a shutdown — the process stays up and the control segment stays mapped diff --git a/source/run/queue/drainRuns.hpp b/source/run/queue/drainRuns.hpp index c1c6e8c..05a10e1 100644 --- a/source/run/queue/drainRuns.hpp +++ b/source/run/queue/drainRuns.hpp @@ -20,9 +20,13 @@ // imports nothing itself. namespace redis_runner { -// Drains BACKTESTING_QUEUE_RUN on a single long-lived connection. For each run it -// loads the QuestDB ticks once, drains that run's strategy list, then retires the -// run. When the queue is empty it waits and re-peeks rather than exiting, so the +// Drains the run queues (queue_keys::RUN_QUEUES, in strict priority order) on a +// single long-lived connection. Each run's tick window comes from the bridge's +// per-symbols superset cache (one QuestDB load serves every rolling-ladder +// window that fits); a run's strategy list is drained onto the pool with the +// tick buffer held by value per task, so backtests from consecutive runs +// pipeline — the pool only quiesces right before a real (cache-miss) load. +// When every queue is empty it waits and re-peeks rather than exiting, so the // worker stays up as a daemon; only a Redis/DB/decode error leaves the loop // (return 3). boost::asio::awaitable drainRuns( diff --git a/source/run/queue/redisRunner.hpp b/source/run/queue/redisRunner.hpp index 55ef5b6..974edd8 100644 --- a/source/run/queue/redisRunner.hpp +++ b/source/run/queue/redisRunner.hpp @@ -7,11 +7,12 @@ #pragma once #include -// Worker entry point. Drains BACKTESTING_QUEUE_RUN: for each run it loads the -// QuestDB tick data once, then drains that run's per-RUN_ID strategy list, -// running every strategy against the cached ticks. Safe to launch many workers -// concurrently — they peek the same run, load ticks once each, and compete on -// RPOP of the shared strategy list. +// Worker entry point. Drains the run queues (queue_keys::RUN_QUEUES, priority +// ordered — grid sweeps before chained rolling-window runs): for each run it +// loads the QuestDB tick data once, then drains that run's per-RUN_ID strategy +// list, running every strategy against the cached ticks. Safe to launch many +// workers concurrently — they peek the same run, load ticks once each, and +// compete on RPOP of the shared strategy list. class RedisRunner { public: static int run(const std::string& questdbHost, diff --git a/source/run/queue/rollingWindow.cppm b/source/run/queue/rollingWindow.cppm new file mode 100644 index 0000000..831082a --- /dev/null +++ b/source/run/queue/rollingWindow.cppm @@ -0,0 +1,132 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include "shared/tradingDefinitions/config/configuration.hpp" +#include "shared/utilities/queueKeys.hpp" + +export module rollingWindow; + +import std; // replaces , , + +export namespace rolling { + +// One backtest window: LAST_MONTHS long, ending offsetMonths before now — +// the same semantics as RunConfiguration's LAST_MONTHS/OFFSET_MONTHS. +struct Window { + int lastMonths; + int offsetMonths; + + bool operator==(const Window&) const = default; +}; + +// The ladder's terminal rung: one run over the full 9 months of history, +// ending at the present day. Exported because "proven over the full window" +// is also live's eligibility bar — the terminal run's results land in the +// dedicated winners index (elasticClient's resultsBaseFor) that liveWinners +// reads, so routing, ladder and live stay in lockstep if the ladder ever +// grows. +inline constexpr Window kFullHistory{9, 0}; + +// The rolling-window ladder a queue-sourced strategy walks after each +// completed backtest: the most recent 3 months, then each 3-month slice +// before it until 9 months of history are covered, then one final run over +// the full 9 months. Returns the window to queue next, or nullopt when the +// finished window is the terminal full-history run — or is not on the ladder +// at all, so a hand-queued one-off window can never start a chain. +std::optional nextWindow(int lastMonths, int offsetMonths); + +// The run descriptor for a strategy advancing to `next`: a fresh RUN_ID and +// the next window, with SYMBOLS and every risk cap carried over from the +// finished run so the strategy is judged under identical rules in every +// window. +tradingDefinitions::RunConfiguration nextRunConfiguration( + const tradingDefinitions::Configuration& finished, + Window next, + const std::string& newRunId); + +// The Redis run queue a window's descriptor belongs on: rung N of the ladder +// lands on the N-th chain queue in queue_keys::RUN_QUEUES, so workers drain +// the whole grid (and every earlier rung) before touching it. Anything not +// produced by the ladder — the seed window, a hand-queued one-off — belongs +// on the shared RUN queue. +std::string queueKeyFor(Window window); + +} // namespace rolling + +namespace { + +struct Step { + rolling::Window from; + rolling::Window to; +}; + +// The ladder is an explicit table, not a formula, so only windows that are +// exactly on it chain — (3,6) is the last slice (it reaches the full 9 months +// back), so it advances to the terminal (9,0) full-history run, which appears +// on no left-hand side and therefore ends the chain. +constexpr std::array kLadder = std::to_array({ + {{3, 0}, {3, 3}}, + {{3, 3}, {3, 6}}, + {{3, 6}, rolling::kFullHistory}, +}); + +// The queues are depth-indexed by rung, so the ladder and the queue list must +// grow together: RUN at index 0, then one chain queue per rung. +static_assert(kLadder.size() + 1 == queue_keys::RUN_QUEUES.size(), + "one chain queue per ladder rung: extend queue_keys::RUN_QUEUES " + "alongside kLadder"); + +} // namespace + +namespace rolling { + +std::optional nextWindow(const int lastMonths, const int offsetMonths) { + const Window finished{lastMonths, offsetMonths}; + for (const Step& step : kLadder) { + if (step.from == finished) { + return step.to; + } + } + return std::nullopt; +} + +std::string queueKeyFor(const Window window) { + for (std::size_t rung = 0; rung < kLadder.size(); ++rung) { + if (kLadder[rung].to == window) { + return queue_keys::RUN_QUEUES[rung + 1]; + } + } + return queue_keys::RUN; +} + +tradingDefinitions::RunConfiguration nextRunConfiguration( + const tradingDefinitions::Configuration& finished, + const Window next, + const std::string& newRunId) { + return tradingDefinitions::RunConfiguration{ + .RUN_ID = newRunId, + .SYMBOLS = finished.SYMBOLS, + // The batch identity must survive every rung: the terminal {9,0} + // run's documents are what the weekly winners index exists for, and + // dropping BATCH here would silently send them to the unsuffixed + // fallback index instead. + .BATCH = finished.BATCH, + .EXECUTION_TS = finished.EXECUTION_TS, + .LAST_MONTHS = next.lastMonths, + .OFFSET_MONTHS = next.offsetMonths, + .STARTING_BALANCE = finished.STARTING_BALANCE, + .MAX_LOSS_PERCENT = finished.MAX_LOSS_PERCENT, + .MAX_OPEN_TRADES = finished.MAX_OPEN_TRADES, + .MAX_TRADES_PER_MINUTE = finished.MAX_TRADES_PER_MINUTE, + .REPORT_FAILURES = finished.REPORT_FAILURES, + .PEAK_HOURS_ONLY = finished.PEAK_HOURS_ONLY, + .ENTRY_SLIPPAGE_TENTH_PIPS = finished.ENTRY_SLIPPAGE_TENTH_PIPS, + }; +} + +} // namespace rolling diff --git a/source/run/queue/runQueue.cpp b/source/run/queue/runQueue.cpp index 1966ed2..409dee3 100644 --- a/source/run/queue/runQueue.cpp +++ b/source/run/queue/runQueue.cpp @@ -26,11 +26,27 @@ namespace run_queue { // rest of the worker loop. RedisOperations is a named local so it outlives the // inner async awaits. -asio::awaitable> peekRunTail( - std::shared_ptr conn) { +asio::awaitable> peekQueueTail( + std::shared_ptr conn, + std::string queueKey) { RedisOperations ops(std::move(conn)); // LINDEX -1 reads the tail (oldest) run without removing it. - co_return co_await ops.listIndex(queue_keys::RUN, -1); + if (auto descriptor = co_await ops.listIndex(queueKey, -1)) { + co_return PeekedRun{std::move(queueKey), std::move(*descriptor)}; + } + co_return std::nullopt; +} + +asio::awaitable> peekRunTail( + std::shared_ptr conn) { + // First non-empty queue in priority order wins, so the chained backlog is + // untouched while any grid (or earlier-rung) work remains. + for (const char* queueKey : queue_keys::RUN_QUEUES) { + if (auto peeked = co_await peekQueueTail(conn, queueKey)) { + co_return peeked; + } + } + co_return std::nullopt; } asio::awaitable> popStrategyKey( @@ -48,10 +64,11 @@ asio::awaitable> takeStrategyPayload( } asio::awaitable removeRun(std::shared_ptr conn, + std::string queueKey, std::string descriptorB64) { RedisOperations ops(std::move(conn)); // count 0 removes every occurrence, so retiring the run is idempotent. - co_await ops.listRemove(queue_keys::RUN, 0, std::move(descriptorB64)); + co_await ops.listRemove(std::move(queueKey), 0, std::move(descriptorB64)); } } // namespace run_queue diff --git a/source/run/queue/runQueue.hpp b/source/run/queue/runQueue.hpp index 7fad14a..f9fab5a 100644 --- a/source/run/queue/runQueue.hpp +++ b/source/run/queue/runQueue.hpp @@ -19,10 +19,28 @@ // the drain loop (see drainRuns.cpp) keeps each TU small and single-purpose. namespace run_queue { -// Non-destructively reads the run that RPOP would take (the queue tail, i.e. the -// oldest run since LoadCommand LPUSHes onto the head). Multiple workers all -// observe the same run and pile onto it. nullopt when the run queue is empty. -boost::asio::awaitable> peekRunTail( +// A peeked run descriptor plus the queue it was found on, so retiring the run +// LREMs the same list it was claimed from. +struct PeekedRun { + std::string queueKey; + std::string descriptorB64; +}; + +// Non-destructively reads the run that RPOP would take from ONE queue (the +// tail, i.e. the oldest run since producers LPUSH onto the head). Multiple +// workers all observe the same run and pile onto it. nullopt when empty. +// Shared by peekRunTail's priority scan and the single-queue experiment +// drain (drainExperiments). +boost::asio::awaitable> peekQueueTail( + std::shared_ptr conn, + std::string queueKey); + +// Non-destructively reads the run that RPOP would take (the queue tail, i.e. +// the oldest run since producers LPUSH onto the head), scanning the run queues +// in strict priority order (queue_keys::RUN_QUEUES): a chained run is only +// visible once every queue before its own is empty. Multiple workers all +// observe the same run and pile onto it. nullopt when every queue is empty. +boost::asio::awaitable> peekRunTail( std::shared_ptr conn); // Claims one strategy payload KEY NAME off the run's per-RUN_ID list (the list @@ -39,10 +57,11 @@ boost::asio::awaitable> takeStrategyPayload( std::shared_ptr conn, std::string payloadKey); -// Retires a run by removing its descriptor. Idempotent: LREM removes 0 if a peer -// worker already retired it. +// Retires a run by removing its descriptor from the queue it was peeked on. +// Idempotent: LREM removes 0 if a peer worker already retired it. boost::asio::awaitable removeRun( std::shared_ptr conn, + std::string queueKey, std::string descriptorB64); } // namespace run_queue diff --git a/source/run/reporting/elasticClient.cppm b/source/run/reporting/elasticClient.cppm index e29c4ab..6d2a3a4 100644 --- a/source/run/reporting/elasticClient.cppm +++ b/source/run/reporting/elasticClient.cppm @@ -9,31 +9,173 @@ module; #include #include "run/reporting/elasticPublisher.hpp" +#include "run/reporting/outcomeIndices.hpp" +#include "run/reporting/tradeDocument.hpp" #include "run/reporting/tradingResults.hpp" export module elasticClient; -// Typed front-end over the shared Elasticsearch publisher (elastic::putDocument -// in shared/reporting). Completed runs land in index "trading_results"; runs -// cut off early (loss limit) land in "trading_failures". The transport, env -// config and auth all live in the shared publisher so the run path and the -// shared redis-consumer path report through one implementation. +import std; // replaces , , +import rollingWindow; // rolling::kFullHistory — routes {9,0} to the winners index +import trade; // Trade, Direction +import symbolScale; // symbol_scale::getPriceScale + +// Typed front-end over the shared Elasticsearch publisher. Index names come +// from outcomeIndices.hpp: each outcome index is the batch's weekly one +// ("backtesting-results-2026-28" — config.BATCH rides in from the load, so a +// run's documents target the same index no matter when the flusher delivers +// them), and gate-cleared full-history runs are split off into the winners +// index, the small population live's winner selection boots against. The +// transport, env config and auth all live in the shared publisher so the run +// path and the shared redis-consumer path report through one implementation. +// +// Every method serialises on the calling thread, hands the document(s) to the +// publisher's background flusher (elastic::enqueueDocument) and returns +// immediately — a pool worker finishing a run never blocks on Elastic, and the +// flusher's periodic _bulk batches replace the per-run PUT storm that fast +// sweeps were answering with 429s. Delivery failures are logged and +// dead-lettered from the flusher thread; there is nothing to report back here. export class ElasticClient { public: - static int putTradingResults(const TradingResults& results); - static int putTradingFailure(const TradingFailure& failure); - // Compact per-run terminal record; lands in index "trading_final". - static int putTradeFinal(const TradeFinal& result); + static void putTradingResults(const TradingResults& results); + static void putTradingFailure(const TradingFailure& failure); + // Compact per-run terminal record; lands in the weekly kFinalBase index. + static void putTradeFinal(const TradeFinal& result); + // One document per closed trade, queued for outcome_index::kTradesIndex. + // Doc _id = RUN_ID:STRATEGY.UUID:tradeId (RUN_ID:tradeId when the config + // carries no UUID) so bulk retries overwrite rather than duplicate — the + // single-doc paths' random-UUID fallback would break that idempotency. + // Strategy metadata is taken from `config` (Trade.strategyId/strategyName + // are never populated by the engine). + static void bulkPutTrades(const std::vector& closedTrades, + const tradingDefinitions::Configuration& config, + const std::string& hostname); }; -int ElasticClient::putTradingResults(const TradingResults& results) { - return elastic::putDocument("trading_results", nlohmann::json(results).dump()); +// Which base index putTradingResults routes a completed run to: the terminal +// full-history window — exactly live's eligibility bar, so the two stay in +// lockstep through rolling::kFullHistory if the ladder ever grows — lands in +// the winners index; every other window is a screening pass and stays in the +// general results index. Exported so a test can pin the split. +export std::string_view resultsBaseFor(const int lastMonths, + const int offsetMonths) { + const bool fullHistory = lastMonths == rolling::kFullHistory.lastMonths && + offsetMonths == rolling::kFullHistory.offsetMonths; + return fullHistory ? outcome_index::kWinnersBase + : outcome_index::kResultsBase; } -int ElasticClient::putTradingFailure(const TradingFailure& failure) { - return elastic::putDocument("trading_failures", nlohmann::json(failure).dump()); +namespace { + +// Deterministic Elasticsearch _id: one strategy execution produces at most one +// document per index, so RUN_ID + strategy UUID identifies it stably — the +// publisher's retries become idempotent overwrites instead of duplicates, and +// the same execution can be correlated across the three indices. Falls back to +// a publisher-generated UUID for configs the sweep did not stamp with one. +std::string outcomeDocId(const std::string& runId, + const tradingDefinitions::Configuration& config) { + const std::string& uuid = config.STRATEGY.UUID; + return uuid.empty() ? std::string{} : runId + ":" + uuid; } -int ElasticClient::putTradeFinal(const TradeFinal& result) { - return elastic::putDocument("trading_final", nlohmann::json(result).dump()); +} // namespace + +void ElasticClient::putTradingResults(const TradingResults& results) { + elastic::enqueueDocument( + outcome_index::weeklyIndex(resultsBaseFor(results.config.LAST_MONTHS, + results.config.OFFSET_MONTHS), + results.config.BATCH), + nlohmann::json(results).dump(), + outcomeDocId(results.RUN_ID, results.config)); +} + +void ElasticClient::putTradingFailure(const TradingFailure& failure) { + elastic::enqueueDocument( + outcome_index::weeklyIndex(outcome_index::kFailuresBase, + failure.config.BATCH), + nlohmann::json(failure).dump(), + outcomeDocId(failure.RUN_ID, failure.config)); +} + +void ElasticClient::putTradeFinal(const TradeFinal& result) { + elastic::enqueueDocument( + outcome_index::weeklyIndex(outcome_index::kFinalBase, + result.config.BATCH), + nlohmann::json(result).dump(), + outcomeDocId(result.RUN_ID, result.config)); +} + +void ElasticClient::bulkPutTrades(const std::vector& closedTrades, + const tradingDefinitions::Configuration& config, + const std::string& hostname) { + if (closedTrades.empty()) { + return; + } + + // Per-trade _id extends the per-run outcome id with the trade's own id + // ("T{n}", unique within one strategy execution). + const std::string outcomeId = outcomeDocId(config.RUN_ID, config); + const std::string idPrefix = + (outcomeId.empty() ? config.RUN_ID : outcomeId) + ":"; + + // One wall-clock stamp for the whole run's trades: they are queued + // together, and a shared value groups them in ops queries. + const std::string ingestedAt = elastic::nowIsoUtc(); + const std::string& strategyName = config.STRATEGY.TRADING_VARIABLES.STRATEGY; + + std::vector docs; + docs.reserve(closedTrades.size()); + for (const Trade& trade : closedTrades) { + // Convert stored INT32 points back to real decimal prices for Kibana. + // Unknown symbols (scale 0) keep the raw integers; priceScale = 0 in + // the doc tells the consumer which form it is looking at. + const int priceScale = symbol_scale::getPriceScale(trade.symbol); + const auto toPrice = [priceScale](std::int32_t points) { + return priceScale > 0 ? static_cast(points) / priceScale + : static_cast(points); + }; + // Same normalisation as the "Trade Closed" log line: PnL is stored in + // points × size, so divide by both points-per-pip and size to get pips + // of price movement. pnlPoints carries the lossless integer alongside. + const double pnlPips = + (trade.scalingFactor != 0 && trade.size != 0) + ? static_cast(trade.pnl) / + (static_cast(trade.scalingFactor) * trade.size) + : 0.0; + const std::string closeIso = TradeDocument::isoUtcMillis(trade.closeTime); + const TradeDocument doc{ + .RUN_ID = config.RUN_ID, + .timestamp = closeIso, // @timestamp = sim close time + .ingestedAt = ingestedAt, + .hostname = hostname, + .strategyUuid = config.STRATEGY.UUID, + .strategyName = strategyName, + .tradeId = trade.id, + .symbol = trade.symbol, + .direction = + trade.direction == Direction::LONG ? "LONG" : "SHORT", + .size = trade.size, + .entryPrice = toPrice(trade.entryPrice), + .entryBid = toPrice(trade.entryBid), + .entryAsk = toPrice(trade.entryAsk), + .closePrice = toPrice(trade.closePrice), + .stopPrice = toPrice(trade.stopPrice), + .limitPrice = toPrice(trade.limitPrice), + .stopDistancePips = trade.stopDistancePips, + .limitDistancePips = trade.limitDistancePips, + .openTime = TradeDocument::isoUtcMillis(trade.openTime), + .closeTime = closeIso, + .holdSeconds = std::chrono::duration(trade.closeTime - + trade.openTime) + .count(), + .pnlPoints = trade.pnl, + .pnlPips = pnlPips, + .liquidated = trade.liquidated, + .scalingFactor = trade.scalingFactor, + .priceScale = priceScale, + }; + docs.push_back({idPrefix + trade.id, nlohmann::json(doc).dump()}); + } + elastic::enqueueDocuments(std::string{outcome_index::kTradesIndex}, + std::move(docs)); } diff --git a/source/run/reporting/elasticPublisher.cpp b/source/run/reporting/elasticPublisher.cpp index 8d969c7..3250463 100644 --- a/source/run/reporting/elasticPublisher.cpp +++ b/source/run/reporting/elasticPublisher.cpp @@ -6,11 +6,20 @@ #include "run/reporting/elasticPublisher.hpp" +#include #include +#include #include #include +#include +#include #include +#include +#include +#include #include +#include +#include #include #include @@ -42,38 +51,20 @@ std::size_t discardResponse(char* /*ptr*/, std::size_t size, std::size_t nmemb, return size * nmemb; } -} // namespace - -namespace elastic { - -std::string nowIsoUtc() { - const auto now = std::chrono::system_clock::to_time_t( - std::chrono::system_clock::now()); - std::tm tm_buf{}; - gmtime_r(&now, &tm_buf); - char buf[32]; - std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", &tm_buf); - return std::string{buf}; -} - -int putDocument(const std::string& index, const std::string& body) { - // Allow runs to opt out of result reporting entirely (e.g. local backtests - // with no Elastic instance). On by default to preserve existing behaviour. - if (env::getOr("ELASTIC_ENABLED", "1") == "0") { - return 0; - } - - ensureCurlInit(); - - const std::string host = env::getOr("ELASTIC_HOST", "http://localhost:9200"); - const std::string user = env::getOr("ELASTIC_USER", ""); - const std::string password = env::getOr("ELASTIC_USER_PASSWORD", ""); - const std::string url = host + "/" + index + "/_doc/" + generateUuid(); +// One PUT attempt. `code` follows putDocument's contract (0 ok, 1 curl init +// failure, 2 transport error, 3 non-2xx HTTP); `httpStatus` lets the caller +// decide whether a code-3 outcome is retryable (429/5xx) or permanent (4xx). +struct PutAttempt { + int code; + long httpStatus; +}; +PutAttempt attemptPut(const std::string& url, const std::string& user, + const std::string& password, const std::string& body) { CURL* curl = curl_easy_init(); if (!curl) { backtest_log::error("ElasticPublisher: curl_easy_init failed"); - return 1; + return {1, 0}; } struct curl_slist* headers = nullptr; @@ -92,6 +83,15 @@ int putDocument(const std::string& index, const std::string& body) { curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(body.size())); + // Bounded I/O so a hung Elastic endpoint cannot block a pool worker + // forever (every publish runs on a ThreadPool task; an unbounded + // curl_easy_perform would eventually fill every slot and stall the whole + // drain loop). NOSIGNAL is required for libcurl on threads: without it, + // resolver timeouts use SIGALRM/siglongjmp, which is not thread-safe. + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 5L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L); + // Discard the response body instead of letting curl print it to stdout. curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, discardResponse); @@ -112,34 +112,671 @@ int putDocument(const std::string& index, const std::string& body) { if (rc != CURLE_OK) { backtest_log::error(std::string("ElasticPublisher: PUT failed: ") + curl_easy_strerror(rc)); - return 2; + return {2, httpStatus}; } if (httpStatus < 200 || httpStatus >= 300) { backtest_log::error("ElasticPublisher: HTTP " + std::to_string(httpStatus) + " from " + url); - return 3; + return {3, httpStatus}; + } + return {0, httpStatus}; +} + +// Accumulate the response body into the caller's std::string. Unlike the PUT +// path (which discards responses), _bulk can return 200 with per-item failures +// inside the body, so the caller must be able to inspect it. +std::size_t captureResponse(char* ptr, std::size_t size, std::size_t nmemb, + void* userdata) { + static_cast(userdata)->append(ptr, size * nmemb); + return size * nmemb; +} + +// One attempt with a caller-chosen verb. Same bounded-I/O, auth and TLS setup +// as attemptPut (see the comments there); differs in caller-chosen content +// type (_bulk sends NDJSON, everything else JSON) and in capturing the +// response body — _bulk can return 200 with per-item failures, _search's +// whole point is the body, and index creation must inspect a 400 to tell +// "already exists" from a real rejection. For that reason a non-2xx status is +// NOT logged here: callers that treat some of them as benign log their own +// failures (attemptPost keeps the unconditional log for its callers). +PutAttempt attemptRequest(const char* method, const std::string& url, + const std::string& user, const std::string& password, + const std::string& body, const char* contentType, + std::string& responseBody) { + responseBody.clear(); + + CURL* curl = curl_easy_init(); + if (!curl) { + backtest_log::error("ElasticPublisher: curl_easy_init failed"); + return {1, 0}; + } + + struct curl_slist* headers = nullptr; + headers = curl_slist_append( + headers, (std::string{"Content-Type: "} + contentType).c_str()); + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + if (!user.empty()) { + curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_easy_setopt(curl, CURLOPT_USERNAME, user.c_str()); + curl_easy_setopt(curl, CURLOPT_PASSWORD, password.c_str()); + } + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(body.size())); + + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 5L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, captureResponse); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseBody); + + curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); + + const CURLcode rc = curl_easy_perform(curl); + + long httpStatus = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + if (rc != CURLE_OK) { + backtest_log::error(std::string("ElasticPublisher: ") + method + + " failed: " + curl_easy_strerror(rc)); + return {2, httpStatus}; + } + if (httpStatus < 200 || httpStatus >= 300) { + return {3, httpStatus}; + } + return {0, httpStatus}; +} + +// One POST attempt — attemptRequest plus the unconditional non-2xx log every +// pre-existing caller (_bulk chunks, _search) relies on. +PutAttempt attemptPost(const std::string& url, const std::string& user, + const std::string& password, const std::string& body, + const char* contentType, std::string& responseBody) { + const PutAttempt attempt = attemptRequest("POST", url, user, password, body, + contentType, responseBody); + if (attempt.code == 3) { + backtest_log::error("ElasticPublisher: HTTP " + + std::to_string(attempt.httpStatus) + " from " + url); + } + return attempt; +} + +// The one transient-failure retry policy shared by every HTTP entry point +// (putDocument, _bulk chunks, _search): up to kMaxRetryAttempts attempts; +// transport errors (code 2), HTTP 429 backpressure and 5xx are transient, +// everything else (other 4xx: mapping conflicts, auth) would fail identically +// every time and is permanent; back off 1s after the first failure, 3s after +// later ones. +constexpr int kMaxRetryAttempts = 3; + +bool isTransientFailure(const PutAttempt& attempt) { + return attempt.code == 2 || + (attempt.code == 3 && + (attempt.httpStatus == 429 || attempt.httpStatus >= 500)); +} + +void retryBackoff(const int attemptNumber) { + std::this_thread::sleep_for(std::chrono::seconds(attemptNumber == 1 ? 1 : 3)); +} + +// Drives `tryOnce` through the policy above and returns the final attempt +// (code 0 on success). For requests whose whole outcome is one PutAttempt; +// postBulkChunk keeps its own loop because _bulk can partially fail per item. +PutAttempt attemptWithRetries(const std::function& tryOnce) { + PutAttempt attempt{}; + for (int i = 1; i <= kMaxRetryAttempts; ++i) { + attempt = tryOnce(); + if (attempt.code == 0 || !isTransientFailure(attempt) || + i == kMaxRetryAttempts) { + break; + } + retryBackoff(i); + } + return attempt; +} + +// Append an undeliverable document as one NDJSON line so a sweep's computed +// results survive an Elastic outage and can be replayed later. `body` is +// already-serialised JSON, so it embeds verbatim. Serialised by a mutex: +// publishes happen concurrently on pool workers and lines must not interleave. +void deadLetter(const std::string& index, const std::string& docId, + const std::string& body) { + static std::mutex fileMutex; + const std::string path = + env::getOr("ELASTIC_DEADLETTER_PATH", "elastic_deadletter.ndjson"); + const std::lock_guard lock{fileMutex}; + std::ofstream out{path, std::ios::app}; + if (!out) { + backtest_log::error("ElasticPublisher: cannot open dead-letter file " + + path); + return; + } + out << "{\"ts\":\"" << elastic::nowIsoUtc() << "\",\"index\":\"" << index + << "\",\"docId\":\"" << docId << "\",\"doc\":" << body << "}\n"; +} + +// NDJSON body for one _bulk chunk: an action line naming the _id, then the +// pre-serialised document, one pair per doc. The trailing newline after the +// last line is mandatory — Elasticsearch rejects the request without it. The +// action line goes through nlohmann so the _id is JSON-escaped. +std::string buildBulkBody(const std::vector& docs) { + std::size_t bytes = 0; + for (const auto* doc : docs) { + bytes += doc->id.size() + doc->body.size() + 32; + } + std::string body; + body.reserve(bytes); + for (const auto* doc : docs) { + body += nlohmann::json{{"index", {{"_id", doc->id}}}}.dump(); + body += '\n'; + body += doc->body; + body += '\n'; + } + return body; +} + +// POST one chunk (already within the doc/byte caps), retrying transient +// failures with the same backoff policy as putDocument. _bulk can partially +// fail — HTTP 2xx with "errors":true and a per-item status array that aligns +// 1:1 with the actions sent — so item-level 429/5xx rejections form a smaller +// retry batch while other 4xx (mapping conflicts, auth — they would fail +// identically every time) are dead-lettered immediately. The stable _ids make +// replaying a whole chunk an idempotent overwrite of any docs that DID land. +// Returns 0 when every doc in the chunk was accepted, non-zero otherwise. +int postBulkChunk(const std::string& index, const std::string& url, + const std::string& user, const std::string& password, + std::vector pending) { + bool anyDeadLettered = false; + std::string response; + PutAttempt attempt{}; + for (int i = 1; i <= kMaxRetryAttempts; ++i) { + attempt = attemptPost(url, user, password, buildBulkBody(pending), + "application/x-ndjson", response); + + if (attempt.code == 0) { + std::vector retry; + try { + const nlohmann::json resp = nlohmann::json::parse(response); + if (!resp.value("errors", false)) { + if (!backtest_log::is_quiet()) { + std::cout << "ElasticPublisher: bulk POST " << url + << " (" << pending.size() << " docs, HTTP " + << attempt.httpStatus << ")" << std::endl; + } + return anyDeadLettered ? 3 : 0; + } + const nlohmann::json& items = resp.at("items"); + for (std::size_t k = 0; k < pending.size(); ++k) { + long status = 0; + if (k < items.size() && items[k].is_object()) { + const nlohmann::json& action = + items[k].contains("index") ? items[k]["index"] + : items[k]; + if (action.is_object()) { + status = action.value("status", 0); + } + } + if (status >= 200 && status < 300) { + continue; // this doc landed + } + if (status == 429 || status >= 500) { + retry.push_back(pending[k]); + } else { + // Permanent (or unrecognisable) item failure: keep a + // replayable local copy rather than retrying a doc + // that would be rejected identically every time. + deadLetter(index, pending[k]->id, pending[k]->body); + anyDeadLettered = true; + } + } + } catch (const std::exception& e) { + // 2xx but the response body defied inspection. Assume the + // chunk was delivered: dead-lettering here would duplicate + // docs that (most likely) landed. + backtest_log::error( + std::string("ElasticPublisher: _bulk response inspection " + "failed (") + e.what() + + "); assuming chunk delivered"); + return anyDeadLettered ? 3 : 0; + } + if (retry.empty()) { + // errors:true fully accounted for — everything either landed + // or was dead-lettered above. + if (anyDeadLettered) { + backtest_log::error("ElasticPublisher: _bulk to " + url + + " rejected document(s); dead-lettered"); + } + return anyDeadLettered ? 3 : 0; + } + pending = std::move(retry); + if (i == kMaxRetryAttempts) { + break; + } + } else { + if (!isTransientFailure(attempt) || i == kMaxRetryAttempts) { + break; + } + } + retryBackoff(i); + } + + // Retries exhausted or a permanent whole-request failure: same rationale + // as putDocument — the backtest already ran, so keep replayable copies. + backtest_log::error("ElasticPublisher: giving up on " + url + + " after retries; dead-lettering " + + std::to_string(pending.size()) + " document(s)"); + for (const auto* doc : pending) { + deadLetter(index, doc->id, doc->body); + } + return attempt.code != 0 ? attempt.code : 3; +} + +// Positive flush cadence from $ELASTIC_FLUSH_SECONDS. Best-effort like the +// rest of the publisher: junk falls back to the default with a logged warning +// instead of aborting the engine over a reporting knob. +std::chrono::seconds flushIntervalFromEnv() { + const std::string raw = env::getOr("ELASTIC_FLUSH_SECONDS", "30"); + int value = 0; + const auto [ptr, ec] = + std::from_chars(raw.data(), raw.data() + raw.size(), value); + if (ec != std::errc{} || ptr != raw.data() + raw.size() || value < 1) { + backtest_log::error("ElasticPublisher: invalid ELASTIC_FLUSH_SECONDS '" + + raw + "', using 30"); + return std::chrono::seconds{30}; + } + return std::chrono::seconds{value}; +} + +// Buffers documents per index and delivers them from one background thread in +// periodic _bulk batches, so a publish never blocks the thread that produced +// the document — a pool worker finishing a two-second backtest hands off its +// outcome docs and moves on; retries, backoff and dead-lettering all happen +// over here. Singleton via instance(): the magic static starts the thread on +// the first enqueue and its destructor (normal process exit) stops the thread +// and flushes the tail, so a direct `run` invocation's single result still +// lands before the process ends. +class DocumentBatcher { +public: + static DocumentBatcher& instance() { + // The constructor logs (flushIntervalFromEnv's invalid-value warning) + // DURING the magic static's construction — before run()/flush() and + // their guards exist. With the live-logs sink armed, that line would + // re-enter instance() while the static is still initialising: + // recursive static initialisation, which __cxa_guard_acquire turns + // into an abort. Suppress around the construction; the guard costs + // two thread_local writes on the (hot but cheap) steady-state path. + const backtest_log::SinkSuppression suppression; + static DocumentBatcher batcher; + return batcher; + } + + void enqueue(const std::string& index, elastic::BulkDoc doc) { + const std::lock_guard lock{mutex_}; + pending_[index].push_back(std::move(doc)); + } + + void enqueue(const std::string& index, std::vector docs) { + const std::lock_guard lock{mutex_}; + std::vector& queued = pending_[index]; + queued.insert(queued.end(), std::make_move_iterator(docs.begin()), + std::make_move_iterator(docs.end())); + } + + // Swap the buffer out under the lock, deliver outside it — enqueues keep + // landing (into the fresh buffer) while a flush is on the wire. + int flush() { + // flush also runs on caller threads (flushQueuedDocuments, the + // destructor's tail) — its failure lines must not re-enter the queue + // they describe, same doctrine as the no-putEngineException rule + // below. + const backtest_log::SinkSuppression suppression; + std::map> batch; + { + const std::lock_guard lock{mutex_}; + if (pending_.empty()) { + return 0; + } + batch.swap(pending_); + } + int worst = 0; + for (const auto& [index, docs] : batch) { + // bulkIndexDocuments already logs and dead-letters every failure. + // Deliberately NO putEngineException here: it enqueues into this + // same batcher, so reporting a failed flush through it would feed + // the next flush a fresh document for as long as the outage lasts. + const int rc = elastic::bulkIndexDocuments(index, docs); + if (rc != 0) { + worst = rc; + } + } + return worst; + } + + ~DocumentBatcher() { + // Static teardown begins here: any later log line shipping through + // the sink would enqueue into this dying batcher. Disarm first so + // other statics' destructors can keep logging to stderr safely. + backtest_log::setSink(nullptr); + { + const std::lock_guard lock{mutex_}; + stop_ = true; + } + cv_.notify_all(); + flusher_.join(); + flush(); // the tail: anything enqueued after the thread's last pass + } + + DocumentBatcher(const DocumentBatcher&) = delete; + DocumentBatcher& operator=(const DocumentBatcher&) = delete; + +private: + DocumentBatcher() : interval_(flushIntervalFromEnv()) { + // Complete curl's global-init guard before this constructor returns: + // statics destroy in reverse order of construction, so curl teardown + // then happens AFTER ~DocumentBatcher's tail flush. + ensureCurlInit(); + flusher_ = std::thread([this] { run(); }); + } + + void run() { + // The flusher's own log lines must never re-enter this queue (see + // flush() below) — suppress the live-logs sink for this thread's + // whole lifetime. + const backtest_log::SinkSuppression suppression; + std::unique_lock lock{mutex_}; + for (;;) { + cv_.wait_for(lock, interval_, [this] { return stop_; }); + if (stop_) { + return; // the destructor flushes the tail after the join + } + if (pending_.empty()) { + continue; // quiet interval — no POST for an empty buffer + } + lock.unlock(); + flush(); + lock.lock(); + } + } + + const std::chrono::seconds interval_; + std::mutex mutex_; + std::condition_variable cv_; + bool stop_ = false; + std::map> pending_; + std::thread flusher_; +}; + +} // namespace + +namespace elastic { + +std::string nowIsoUtc() { + const auto now = std::chrono::system_clock::to_time_t( + std::chrono::system_clock::now()); + std::tm tm_buf{}; + gmtime_r(&now, &tm_buf); + char buf[32]; + std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", &tm_buf); + return std::string{buf}; +} + +int putDocument(const std::string& index, const std::string& body, + const std::string& docId) { + // Publisher-origin failure lines must not ship through the live-logs + // sink back into this publisher (see backtestLog.hpp). + const backtest_log::SinkSuppression suppression; + // Allow runs to opt out of result reporting entirely (e.g. local backtests + // with no Elastic instance). On by default to preserve existing behaviour. + if (env::getOr("ELASTIC_ENABLED", "1") == "0") { + return 0; + } + + ensureCurlInit(); + + const std::string host = env::getOr("ELASTIC_HOST", "http://localhost:9200"); + const std::string user = env::getOr("ELASTIC_USER", ""); + const std::string password = env::getOr("ELASTIC_USER_PASSWORD", ""); + const std::string id = docId.empty() ? generateUuid() : docId; + const std::string url = host + "/" + index + "/_doc/" + id; + + // Retry transient failures (attemptWithRetries). A caller-supplied docId + // makes the replayed PUT idempotent — it overwrites the same _id rather + // than duplicating the document. + const PutAttempt attempt = attemptWithRetries( + [&] { return attemptPut(url, user, password, body); }); + if (attempt.code == 0) { + // Per-strategy success line is skipped under concurrent backtests. + if (!backtest_log::is_quiet()) { + std::cout << "ElasticPublisher: PUT " << url << " (HTTP " + << attempt.httpStatus << ")" << std::endl; + } + return 0; + } + + // The backtest that produced this document already ran; losing the doc + // would silently skew the sweep's accounting (results/failures/final + // counts drifting apart). Keep a replayable local copy instead. + backtest_log::error("ElasticPublisher: giving up on " + url + + " after retries; dead-lettering document"); + deadLetter(index, id, body); + return attempt.code; +} + +int bulkIndexDocuments(const std::string& index, + const std::vector& docs) { + // Same rationale as putDocument: delivery failures log, and those lines + // must not re-enter the queue they describe. + const backtest_log::SinkSuppression suppression; + if (docs.empty()) { + return 0; + } + // Same opt-out as putDocument: no Elastic instance, no reporting. + if (env::getOr("ELASTIC_ENABLED", "1") == "0") { + return 0; + } + + ensureCurlInit(); + + const std::string host = env::getOr("ELASTIC_HOST", "http://localhost:9200"); + const std::string user = env::getOr("ELASTIC_USER", ""); + const std::string password = env::getOr("ELASTIC_USER_PASSWORD", ""); + const std::string url = host + "/" + index + "/_bulk"; + + // Bound each request so a trade-heavy run cannot produce an oversized + // _bulk body (Elasticsearch's sweet spot is single-digit megabytes); the + // doc cap keeps the per-item response array cheap to walk. Chunks POST + // sequentially — this runs once at end of run, off the hot path. + constexpr std::size_t kMaxBulkDocs = 1000; + constexpr std::size_t kMaxBulkBytes = 4 * 1024 * 1024; + + int worst = 0; + std::vector chunk; + std::size_t chunkBytes = 0; + const auto flush = [&] { + if (chunk.empty()) { + return; + } + const int rc = postBulkChunk(index, url, user, password, chunk); + if (rc != 0) { + worst = rc; + } + chunk.clear(); + chunkBytes = 0; + }; + for (const BulkDoc& doc : docs) { + chunk.push_back(&doc); + chunkBytes += doc.id.size() + doc.body.size() + 32; // + action line + if (chunk.size() >= kMaxBulkDocs || chunkBytes >= kMaxBulkBytes) { + flush(); + } + } + flush(); + return worst; +} + +void enqueueDocument(const std::string& index, const std::string& body, + const std::string& docId) { + // Same opt-out as putDocument — checked at enqueue time so a disabled run + // never buffers anything (or starts the flusher thread at all). + if (env::getOr("ELASTIC_ENABLED", "1") == "0") { + return; + } + DocumentBatcher::instance().enqueue( + index, {docId.empty() ? generateUuid() : docId, body}); +} + +void enqueueDocuments(const std::string& index, std::vector docs) { + if (docs.empty() || env::getOr("ELASTIC_ENABLED", "1") == "0") { + return; + } + DocumentBatcher::instance().enqueue(index, std::move(docs)); +} + +int flushQueuedDocuments() { + if (env::getOr("ELASTIC_ENABLED", "1") == "0") { + return 0; + } + return DocumentBatcher::instance().flush(); +} + +int searchIndex(const std::string& index, const std::string& queryBody, + std::string& responseBody, long& httpStatus) { + responseBody.clear(); + httpStatus = 0; + + // Publisher-origin lines stay out of the live-logs sink here too — the + // caller's own logging (e.g. liveWinners' failure lines) still ships. + const backtest_log::SinkSuppression suppression; + + // A disabled reporter has nothing to read from — surface it as a failure + // (unlike the write paths, where "disabled" means a successful no-op). + if (env::getOr("ELASTIC_ENABLED", "1") == "0") { + backtest_log::error( + "ElasticPublisher: _search requested but ELASTIC_ENABLED=0"); + return 4; + } + + ensureCurlInit(); + + const std::string host = env::getOr("ELASTIC_HOST", "http://localhost:9200"); + const std::string user = env::getOr("ELASTIC_USER", ""); + const std::string password = env::getOr("ELASTIC_USER_PASSWORD", ""); + const std::string url = host + "/" + index + "/_search"; + + // Same transient-failure policy as putDocument, minus the dead-letter: + // a failed read leaves nothing to replay, the caller just re-queries. + // responseBody ends up holding the final attempt's body either way + // (attemptPost clears it per attempt). + const PutAttempt attempt = attemptWithRetries([&] { + return attemptPost(url, user, password, queryBody, "application/json", + responseBody); + }); + httpStatus = attempt.httpStatus; + return attempt.code; +} + +int ensureIndexExists(const std::string& index) { + const backtest_log::SinkSuppression suppression; + if (env::getOr("ELASTIC_ENABLED", "1") == "0") { + return 0; + } + + ensureCurlInit(); + + const std::string host = env::getOr("ELASTIC_HOST", "http://localhost:9200"); + const std::string user = env::getOr("ELASTIC_USER", ""); + const std::string password = env::getOr("ELASTIC_USER_PASSWORD", ""); + const std::string url = host + "/" + index; + + std::string response; + const PutAttempt attempt = attemptWithRetries([&] { + return attemptRequest("PUT", url, user, password, "{}", + "application/json", response); + }); + if (attempt.code == 0) { + if (!backtest_log::is_quiet()) { + std::cout << "ElasticPublisher: created index " << index + << std::endl; + } + return 0; + } + // The weekly load PUTs the same index name on every invocation; the index + // already being there is the desired state, not a failure. + if (attempt.httpStatus == 400 && + response.find("resource_already_exists_exception") != std::string::npos) { + return 0; + } + backtest_log::error("ElasticPublisher: creating index " + index + + " failed (HTTP " + std::to_string(attempt.httpStatus) + + "): " + response.substr(0, 500)); + return attempt.code; +} + +int repointAlias(const std::string& alias, const std::string& index) { + const backtest_log::SinkSuppression suppression; + if (env::getOr("ELASTIC_ENABLED", "1") == "0") { + return 0; + } + + ensureCurlInit(); + + const std::string host = env::getOr("ELASTIC_HOST", "http://localhost:9200"); + const std::string user = env::getOr("ELASTIC_USER", ""); + const std::string password = env::getOr("ELASTIC_USER_PASSWORD", ""); + const std::string url = host + "/_aliases"; + + // Built field-by-field: nested brace-init of nlohmann objects is ambiguous + // between object and array forms. + nlohmann::json remove; + remove["remove"]["index"] = "*"; + remove["remove"]["alias"] = alias; + remove["remove"]["must_exist"] = false; + nlohmann::json add; + add["add"]["index"] = index; + add["add"]["alias"] = alias; + nlohmann::json actions; + actions["actions"] = nlohmann::json::array({remove, add}); + + std::string response; + const PutAttempt attempt = attemptWithRetries([&] { + return attemptPost(url, user, password, actions.dump(), + "application/json", response); + }); + if (attempt.code != 0) { + backtest_log::error("ElasticPublisher: repointing alias " + alias + + " -> " + index + " failed (HTTP " + + std::to_string(attempt.httpStatus) + "): " + + response.substr(0, 500)); + return attempt.code; } - // Per-strategy success line is skipped under concurrent backtests (quiet). if (!backtest_log::is_quiet()) { - std::cout << "ElasticPublisher: PUT " << url << " (HTTP " << httpStatus - << ")" << std::endl; + std::cout << "ElasticPublisher: alias " << alias << " -> " << index + << std::endl; } return 0; } -int putEngineException(const EngineException& ex) noexcept { +void putEngineException(const EngineException& ex) noexcept { // Callers report from inside catch handlers, so this must never throw back // out. nlohmann's dump() throws type_error.316 on invalid UTF-8, and the // message is arbitrary exception text — guard it explicitly. try { - return putDocument("engine_exceptions", nlohmann::json(ex).dump()); + enqueueDocument("engine_exceptions", nlohmann::json(ex).dump()); } catch (const std::exception& e) { backtest_log::error( std::string("ElasticPublisher: failed to report engine exception: ") + e.what()); - return -1; } catch (...) { - return -1; } } diff --git a/source/run/reporting/elasticPublisher.hpp b/source/run/reporting/elasticPublisher.hpp index 32b417e..6d5cc2d 100644 --- a/source/run/reporting/elasticPublisher.hpp +++ b/source/run/reporting/elasticPublisher.hpp @@ -7,31 +7,117 @@ #pragma once #include +#include #include "run/reporting/engineException.hpp" -// Minimal Elasticsearch HTTP client — PUT-only, for indexing run outcomes and -// engine exceptions. Host is read from $ELASTIC_HOST (default -// http://localhost:9200) with optional HTTP basic auth from $ELASTIC_USER / -// $ELASTIC_USER_PASSWORD. Reporting can be disabled entirely with -// $ELASTIC_ENABLED=0. +// Minimal Elasticsearch HTTP client — indexing (PUT/_bulk) for run outcomes +// and engine exceptions, a _search read path (live winner selection), and +// index/alias admin for the weekly outcome indices (load; outcomeIndices.hpp). +// Host is read from $ELASTIC_HOST (default http://localhost:9200) with +// optional HTTP basic auth from $ELASTIC_USER / $ELASTIC_USER_PASSWORD. +// Reporting can be disabled entirely with $ELASTIC_ENABLED=0. +// +// Two write paths: putDocument/bulkIndexDocuments deliver synchronously on the +// calling thread; enqueueDocument(s) hand the document to a background flusher +// that batches everything into periodic _bulk requests, so the caller never +// blocks on the network. Report-only documents (run outcomes, engine +// exceptions, live trade audits) go through the queue — fast sweeps were +// producing one PUT per finished run, a request storm Elasticsearch answered +// with 429s. // // Lives in shared/ (as a plain header, not a module) so both the run path // (elasticClient) and the shared redis-consumer path (redisRunner, drainRuns, // which import nothing) can report through one implementation. namespace elastic { -// PUT one JSON document into `index`, with a freshly generated UUID id. +// PUT one JSON document into `index`. `docId` names the Elasticsearch _id so +// retries are idempotent (a replayed PUT overwrites the same document instead +// of duplicating it); when empty, a fresh UUID is generated. Transient +// failures (transport errors, HTTP 429/5xx) are retried with backoff; a +// document that still cannot be delivered is appended to a local NDJSON +// dead-letter file ($ELASTIC_DEADLETTER_PATH, default +// elastic_deadletter.ndjson) for later replay. // Returns 0 on success (or when reporting is disabled), non-zero otherwise. -int putDocument(const std::string& index, const std::string& body); +int putDocument(const std::string& index, const std::string& body, + const std::string& docId = ""); // ISO-8601 UTC timestamp ("YYYY-MM-DDTHH:MM:SSZ"). std::string nowIsoUtc(); -// Convenience: serialise and PUT an engine exception into "engine_exceptions". -// noexcept by contract — every caller invokes this from a catch handler where a -// throw would mask the original failure (and, on a worker, abort the drain -// loop). Serialisation/transport errors are logged and swallowed. -int putEngineException(const EngineException& ex) noexcept; +// One document destined for a _bulk request: the deterministic _id plus the +// pre-serialised JSON body. The id must be non-empty — bulk retries replay +// whole chunks, so only a stable _id keeps them idempotent overwrites. +struct BulkDoc { + std::string id; + std::string body; +}; + +// POST `docs` into `index` via the Elasticsearch _bulk API, in chunks (doc- and +// byte-bounded). Same env config, retry/backoff and dead-letter behaviour as +// putDocument; documents rejected item-by-item inside an otherwise-successful +// bulk response (mapping conflicts, etc.) are dead-lettered too. Returns 0 when +// every document was accepted (or reporting is disabled), non-zero when any +// document had to be dead-lettered. +int bulkIndexDocuments(const std::string& index, const std::vector& docs); + +// Queue one JSON document for the background flusher and return immediately — +// no network I/O on the calling thread. A dedicated thread delivers everything +// queued (across all indices) through bulkIndexDocuments — same retry/backoff +// and dead-letter treatment — every $ELASTIC_FLUSH_SECONDS (default 30), +// skipping intervals where nothing is buffered; a final flush runs at normal +// process exit. The trade-off: documents ride in memory for up to one +// interval, so an abnormal termination (signal, crash) loses that tail — +// nothing is dead-lettered for documents never attempted. `docId` as in +// putDocument; when empty a UUID is minted at enqueue time (the _bulk path +// needs a stable _id to keep retries idempotent). +void enqueueDocument(const std::string& index, const std::string& body, + const std::string& docId = ""); + +// Queue a pre-built batch for `index` in one lock acquisition (the per-trade +// documents arrive hundreds at a time). Same delivery semantics as +// enqueueDocument; every doc must already carry a non-empty id. +void enqueueDocuments(const std::string& index, std::vector docs); + +// Deliver everything currently queued, synchronously, on the calling thread. +// The periodic flusher and process exit call this automatically; exposed for +// callers that need documents visible before the next interval. Returns 0 when +// every document was accepted (or nothing was queued / reporting disabled). +int flushQueuedDocuments(); + +// POST `queryBody` to {ELASTIC_HOST}/{index}/_search and capture the response +// body. Read path: transient failures (transport errors, HTTP 429/5xx) are +// retried with the same backoff as putDocument, but nothing is dead-lettered — +// there is no document to replay; the caller simply re-queries. Returns 0 on +// HTTP 2xx; otherwise the attempt code (1 curl init, 2 transport, 3 non-2xx +// HTTP, 4 reporting disabled via $ELASTIC_ENABLED=0). `httpStatus` is the last +// HTTP status seen (0 when no HTTP exchange happened) so callers can tell a +// permanent 4xx (bad query / mapping) from an outage. +int searchIndex(const std::string& index, const std::string& queryBody, + std::string& responseBody, long& httpStatus); + +// PUT {ELASTIC_HOST}/{index} — create the index if it does not exist yet. +// Elasticsearch answering HTTP 400 resource_already_exists_exception counts +// as success: the load command re-runs this for every invocation in a week +// and the index already being there is exactly the state it wants. Same +// transient-failure retry policy as putDocument (nothing to dead-letter). +// Returns 0 on success or when reporting is disabled via $ELASTIC_ENABLED=0. +int ensureIndexExists(const std::string& index); + +// Atomically repoint `alias` at exactly `index` via one POST /_aliases action +// set: remove the alias from EVERY index currently holding it (must_exist: +// false, so the very first swap — and a hand-parked bootstrap alias on a +// legacy index — are handled), then add it to `index`. Elasticsearch applies +// the set atomically, so readers never observe the alias missing or doubled. +// `index` must already exist (ensureIndexExists). Returns 0 on success or +// when reporting is disabled. +int repointAlias(const std::string& alias, const std::string& index); + +// Convenience: serialise and queue an engine exception for "engine_exceptions" +// (enqueueDocument semantics — delivered by the background flusher). noexcept +// by contract — every caller invokes this from a catch handler where a throw +// would mask the original failure (and, on a worker, abort the drain loop). +// Serialisation errors are logged and swallowed. +void putEngineException(const EngineException& ex) noexcept; } // namespace elastic diff --git a/source/run/reporting/outcomeIndices.cpp b/source/run/reporting/outcomeIndices.cpp new file mode 100644 index 0000000..9eca980 --- /dev/null +++ b/source/run/reporting/outcomeIndices.cpp @@ -0,0 +1,42 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "run/reporting/outcomeIndices.hpp" + +#include "shared/utilities/env.hpp" + +namespace outcome_index { + +std::string weeklyIndex(const std::string_view base, const std::string_view batch) { + if (batch.empty()) { + return std::string{base}; + } + return std::string{base} + "-" + std::string{batch}; +} + +std::string currentAlias(const std::string_view base) { + return std::string{base} + "-current"; +} + +std::string isoWeekLabel(const std::time_t utc) { + std::tm tm_buf{}; + gmtime_r(&utc, &tm_buf); + // %G/%V are the ISO-8601 week-based year and week number (C99 strftime); + // gmtime_r fills the tm_wday/tm_yday they derive from. + char buf[16]; + std::strftime(buf, sizeof(buf), "%G-%V", &tm_buf); + return std::string{buf}; +} + +std::string currentBatchLabel() { + const std::string pinned = env::getOr("BACKTEST_BATCH", ""); + if (!pinned.empty()) { + return pinned; + } + return isoWeekLabel(std::time(nullptr)); +} + +} // namespace outcome_index diff --git a/source/run/reporting/outcomeIndices.hpp b/source/run/reporting/outcomeIndices.hpp new file mode 100644 index 0000000..7dcb1f5 --- /dev/null +++ b/source/run/reporting/outcomeIndices.hpp @@ -0,0 +1,77 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include + +// Naming for the outcome indices in Elasticsearch: `backtesting-` prefix, +// dashes throughout, and a per-week batch suffix so each weekly sweep lands in +// its own index ("backtesting-results-2026-28") while the history of earlier +// weeks stays untouched and searchable via the backtesting-* pattern. The +// batch label is minted ONCE at load time (currentBatchLabel) and rides inside +// every run configuration — never recomputed at write time — so a run's +// documents always target the same index no matter when the flusher delivers +// them: the deterministic doc _id (RUN_ID:UUID) keeps retries and dead-letter +// replays idempotent only within a stable index, and the ladder's terminal +// {9,0} run, drained days after the seed, still lands in the seed week. +// +// Each base also has a rolling "-current" alias, atomically repointed at the +// newest weekly index by the load command (elastic::repointAlias) — readers of +// "this week's run" (live winner selection, dashboards) follow the alias and +// never race the calendar. The alias name must differ from every concrete +// index name, which the batch suffix guarantees. +// +// The legacy trading_* indices are a frozen archive: an empty batch label +// (payloads queued before the field existed, hand-run configs) falls back to +// the bare base name, not to the old names. +namespace outcome_index { + +// Gate-cleared screening runs — every ladder rung EXCEPT the terminal +// full-history window (those go to kWinnersBase, see elasticClient). +inline constexpr std::string_view kResultsBase = "backtesting-results"; +// Gate-cleared full-history ({9,0}) runs — exactly the population live's +// winner selection trades, kept apart so live boots against a small index. +inline constexpr std::string_view kWinnersBase = "backtesting-winners"; +// Terminal record for EVERY run (completed / underperformed / cut off). +inline constexpr std::string_view kFinalBase = "backtesting-final"; +// Runs cut off by the loss limit (when REPORT_FAILURES). +inline constexpr std::string_view kFailuresBase = "backtesting-failures"; +// One document per closed trade (opt-in, high volume) — renamed with the +// convention but deliberately NOT rotated weekly. +inline constexpr std::string_view kTradesIndex = "backtesting-trades"; +// One aggregate document per experiment x symbol group (the `analysis` +// worker). Weekly + -current alias like the outcome bases, but deliberately +// NOT in kWeeklyBases: `load` must not create empty experiment indices — +// `experiments` prepares this base itself. +inline constexpr std::string_view kExperimentsBase = "backtesting-experiments"; + +// Every base the load command prepares each week (index + -current alias). +inline constexpr std::array kWeeklyBases{ + kResultsBase, kWinnersBase, kFinalBase, kFailuresBase}; + +// "backtesting-results-2026-28"; the bare base when `batch` is empty, so +// pre-batch configs keep writing somewhere sensible without alias admin. +std::string weeklyIndex(std::string_view base, std::string_view batch); + +// "backtesting-results-current" — the rolling alias for the newest batch. +std::string currentAlias(std::string_view base); + +// ISO-8601 year and week of `utc`, dash-separated and zero-padded +// ("2026-28"), so lexical order is chronological order. The ISO year can +// differ from the calendar year around New Year — that is the point: every +// day of one ISO week maps to one label. +std::string isoWeekLabel(std::time_t utc); + +// The batch label a load mints: $BACKTEST_BATCH when set and non-empty +// (re-runs, tests, pinning a multi-invocation load to one label), otherwise +// the current UTC ISO week. +std::string currentBatchLabel(); + +} // namespace outcome_index diff --git a/source/run/reporting/resultsSummary.cppm b/source/run/reporting/resultsSummary.cppm index d3170d0..54eb3af 100644 --- a/source/run/reporting/resultsSummary.cppm +++ b/source/run/reporting/resultsSummary.cppm @@ -22,6 +22,15 @@ export class ResultsSummary { public: static TradingResultsStats collect(const TradeManager& tradeManager, const tradingDefinitions::Configuration& config); + // Core overload for callers that have no Configuration in scope (runLoop's + // performance gate). startingBalance and lastMonths feed the score's + // normalisation/annualisation; primaryPointsPerPip converts the aggregate + // max drawdown to pips (the Configuration overload derives it from the + // run's primary symbol). + static TradingResultsStats collect(const TradeManager& tradeManager, + boost::decimal::decimal64_t startingBalance, + int lastMonths, + int primaryPointsPerPip); static void summarise(const TradeManager& tradeManager, const tradingDefinitions::Configuration& config); }; @@ -46,12 +55,13 @@ namespace { // equity curve, in pips (>= 0). Guards leave every score field at 0 for an // unscoreable run (no closed trades, non-positive balance, no horizon). void computePerformanceScore(TradingResultsStats& stats, - const tradingDefinitions::Configuration& config, + const boost::decimal::decimal64_t startingBalanceDec, + const int lastMonths, boost::decimal::decimal64_t winPipSum, boost::decimal::decimal64_t lossPipSum, boost::decimal::decimal64_t maxDropPips) { - const double startingBalance = static_cast(config.STARTING_BALANCE); - if (stats.tradesClosed == 0 || startingBalance <= 0.0 || config.LAST_MONTHS <= 0) { + const double startingBalance = static_cast(startingBalanceDec); + if (stats.tradesClosed == 0 || startingBalance <= 0.0 || lastMonths <= 0) { return; // unscoreable — leave all score fields at their 0 defaults } @@ -62,43 +72,51 @@ void computePerformanceScore(TradingResultsStats& stats, return; // only breakeven trades — nothing to score } - // Win rate: a run with no losers wins by definition; otherwise the share of - // decisive trades that won. - double winRate = 0.0; - if (negative == 0.0) { - winRate = 1.0; - } else if (positive != 0.0) { - winRate = positive / totalSum; - } + // Win rate over ALL closed trades (winners + losers + breakevens), so it + // reads as "share of trades that won". A run of one winner and ninety-nine + // breakevens is 1%, not the forced 100% the old no-losers special case + // produced. Breakevens likewise weight the loss share below, so the + // expectancy is a true per-trade mean. + const double closedCount = static_cast(stats.tradesClosed); + const double winRate = positive / closedCount; + const double lossRate = negative / closedCount; const double averageWin = positive > 0.0 ? static_cast(winPipSum) / positive : 0.0; const double averageLoss = negative > 0.0 ? std::abs(static_cast(lossPipSum) / negative) : 0.0; // Profit-to-loss weighting; the C# special cases pin the degenerate ends. + // averageLoss == 0 with losers present means the losing pips were + // unmeasurable (unknown-scale trades) — pin to the cap rather than divide + // to Inf, which nlohmann would silently serialise as null. double tradeRatio = 0.0; - if (positive > 0.0 && negative > 0.0) { + if (positive == 0.0) { + tradeRatio = 0.0; + } else if (negative == 0.0 || averageLoss == 0.0) { + tradeRatio = 100.0; + } else { tradeRatio = (averageWin * positive) / (averageLoss * negative); } - if (positive == 0.0) tradeRatio = 0.0; - if (negative == 0.0) tradeRatio = 100.0; // Expectancy in pips, normalised to opening equity, then turned into a // Van Tharp SQN by the sqrt(trades) frequency factor and mapped so SQN 2.0 // ~= score 50. - const double expectancyPips = averageWin * winRate - averageLoss * (1.0 - winRate); + const double expectancyPips = averageWin * winRate - averageLoss * lossRate; const double expectancyPercent = expectancyPips / startingBalance * 100.0; const double tradeFreqFactor = std::sqrt(totalSum); const double systemQuality = expectancyPercent * tradeFreqFactor; const double expectancyScore = systemQuality * (50.0 / 2.0); // CAGR over the backtest horizon; finalPnl is already net profit in pips. - const double years = static_cast(config.LAST_MONTHS) / 12.0; + const double years = static_cast(lastMonths) / 12.0; const double finalPnl = static_cast(stats.finalPnl); const double cagrPercent = (finalPnl / startingBalance) / years * 100.0; - // Realized max drawdown as a percent of opening equity, floored at 2% so a - // near-flat curve cannot explode the Calmar ratio. Calmar 3.0 ~= score 50. + // Max drawdown "percent": pip drawdown over the balance READ AS A PIP + // BUDGET (the same convention as the loss floor in runLoop — there is no + // pip-value model, so this is an internally consistent ranking proxy, not + // a true percent of account value). Floored at 2% so a near-flat curve + // cannot explode the Calmar ratio. Calmar 3.0 ~= score 50. const double maxDrawdownPercent = static_cast(maxDropPips) / startingBalance * 100.0; const double adjustedDrawdown = std::max(maxDrawdownPercent, 2.0); const double calmarScore = (cagrPercent / adjustedDrawdown) * (50.0 / 3.0); @@ -115,19 +133,39 @@ void computePerformanceScore(TradingResultsStats& stats, const double rawPerformance = expectancyScore * 0.5 + calmarScore * 0.5; const double performance = rawPerformance * confidenceMultiplier; - stats.winRate = boost::decimal::decimal64_t{winRate}; - stats.tradeRatio = boost::decimal::decimal64_t{tradeRatio}; - stats.expectancyScore = boost::decimal::decimal64_t{expectancyScore}; - stats.calmarScore = boost::decimal::decimal64_t{calmarScore}; - stats.confidenceMultiplier = boost::decimal::decimal64_t{confidenceMultiplier}; - stats.maxDrawdownPercent = boost::decimal::decimal64_t{maxDrawdownPercent}; - stats.performanceScore = boost::decimal::decimal64_t{performance}; + // Belt-and-braces: a non-finite score must never reach the JSON layer, + // where nlohmann silently serialises NaN/Inf as null and the run's metrics + // vanish without an error. Substitute 0 and say so. + const auto finiteOr0 = [](double v, const char* name) { + if (std::isfinite(v)) return v; + backtest_log::error(std::string("ResultsSummary: non-finite ") + name + + " replaced with 0"); + return 0.0; + }; + + stats.winRate = boost::decimal::decimal64_t{finiteOr0(winRate, "winRate")}; + stats.tradeRatio = boost::decimal::decimal64_t{finiteOr0(tradeRatio, "tradeRatio")}; + stats.expectancyScore = boost::decimal::decimal64_t{finiteOr0(expectancyScore, "expectancyScore")}; + stats.calmarScore = boost::decimal::decimal64_t{finiteOr0(calmarScore, "calmarScore")}; + stats.confidenceMultiplier = boost::decimal::decimal64_t{finiteOr0(confidenceMultiplier, "confidenceMultiplier")}; + stats.maxDrawdownPercent = boost::decimal::decimal64_t{finiteOr0(maxDrawdownPercent, "maxDrawdownPercent")}; + stats.performanceScore = boost::decimal::decimal64_t{finiteOr0(performance, "performanceScore")}; } } // namespace TradingResultsStats ResultsSummary::collect(const TradeManager& tradeManager, const tradingDefinitions::Configuration& config) { + const std::string primarySymbol = + config.SYMBOLS.substr(0, config.SYMBOLS.find(',')); + return collect(tradeManager, config.STARTING_BALANCE, config.LAST_MONTHS, + symbol_scale::get(primarySymbol)); +} + +TradingResultsStats ResultsSummary::collect(const TradeManager& tradeManager, + const boost::decimal::decimal64_t startingBalance, + const int lastMonths, + const int primaryPointsPerPip) { const auto& activeTrades = tradeManager.getActiveTrades(); const auto& closedTrades = tradeManager.getClosedTrades(); @@ -147,25 +185,37 @@ TradingResultsStats ResultsSummary::collect(const TradeManager& tradeManager, std::size_t losers = 0; std::size_t breakeven = 0; std::size_t liquidated = 0; + std::size_t unmeasured = 0; boost::decimal::decimal64_t pnlSum{0}; // accumulated in pips boost::decimal::decimal64_t winPipSum{0}; // sum of winning trades, pips boost::decimal::decimal64_t lossPipSum{0}; // sum of losing trades, pips (<= 0) for (const auto& trade : closedTrades) { if (trade.direction == Direction::LONG) ++closedLong; else ++closedShort; - if (trade.pnl > 0) ++winners; - else if (trade.pnl < 0) ++losers; - else ++breakeven; if (trade.liquidated) ++liquidated; // trade.pnl is int64 points-per-lot; convert to pips using the trade's - // own points-per-pip so mixed-symbol runs sum correctly. - if (trade.scalingFactor != 0) { - const boost::decimal::decimal64_t pips = - boost::decimal::decimal64_t{trade.pnl} / trade.scalingFactor; - pnlSum += pips; - if (trade.pnl > 0) winPipSum += pips; - else if (trade.pnl < 0) lossPipSum += pips; + // own points-per-pip so mixed-symbol runs sum correctly. A trade whose + // symbol has no known scale (scalingFactor == 0) cannot be expressed in + // pips, so it is excluded from the win/loss counters AND the pip sums — + // classifying it while contributing zero pips would skew the averages + // (and could zero averageLoss into a divide-by-Inf tradeRatio). + if (trade.scalingFactor == 0) { + ++unmeasured; + continue; } + if (trade.pnl > 0) ++winners; + else if (trade.pnl < 0) ++losers; + else ++breakeven; + const boost::decimal::decimal64_t pips = + boost::decimal::decimal64_t{trade.pnl} / trade.scalingFactor; + pnlSum += pips; + if (trade.pnl > 0) winPipSum += pips; + else if (trade.pnl < 0) lossPipSum += pips; + } + if (unmeasured != 0) { + backtest_log::error("ResultsSummary: " + std::to_string(unmeasured) + + " closed trade(s) on unknown-scale symbols excluded" + " from pip metrics"); } // True mark-to-market max drawdown, tracked live in TradeManager (so it @@ -173,9 +223,6 @@ TradingResultsStats ResultsSummary::collect(const TradeManager& tradeManager, // converted to pips with the run's primary symbol's points-per-pip — exact // for single-asset-class runs, matching the loss-limit floor convention in // Operations::run. - const std::string primarySymbol = - config.SYMBOLS.substr(0, config.SYMBOLS.find(',')); - const int primaryPointsPerPip = symbol_scale::get(primarySymbol); boost::decimal::decimal64_t maxDropPips{0}; if (primaryPointsPerPip != 0) { maxDropPips = boost::decimal::decimal64_t{tradeManager.maxDrawdownPoints()} @@ -203,7 +250,8 @@ TradingResultsStats ResultsSummary::collect(const TradeManager& tradeManager, stats.avgPnl = pnlSum / boost::decimal::decimal64_t{static_cast(closedCount)}; } - computePerformanceScore(stats, config, winPipSum, lossPipSum, maxDropPips); + computePerformanceScore(stats, startingBalance, lastMonths, + winPipSum, lossPipSum, maxDropPips); return stats; } diff --git a/source/run/reporting/tradeDocument.cpp b/source/run/reporting/tradeDocument.cpp new file mode 100644 index 0000000..0c34ce0 --- /dev/null +++ b/source/run/reporting/tradeDocument.cpp @@ -0,0 +1,73 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "run/reporting/tradeDocument.hpp" + +#include +#include // POSIX gmtime_r + +std::string TradeDocument::isoUtcMillis(std::chrono::system_clock::time_point tp) { + const auto sinceEpoch = tp.time_since_epoch(); + const auto secs = std::chrono::duration_cast(sinceEpoch); + auto millis = + std::chrono::duration_cast(sinceEpoch - secs); + std::time_t t = secs.count(); + // duration_cast truncates toward zero, so a pre-epoch time point would + // yield a negative millisecond remainder; fold it into the seconds. + if (millis.count() < 0) { + t -= 1; + millis += std::chrono::milliseconds{1000}; + } + std::tm utc{}; + gmtime_r(&t, &utc); + char datePart[32]; + std::strftime(datePart, sizeof(datePart), "%Y-%m-%dT%H:%M:%S", &utc); + char buf[40]; + std::snprintf(buf, sizeof(buf), "%s.%03dZ", datePart, + static_cast(millis.count())); + return std::string{buf}; +} + +void to_json(nlohmann::json& j, const TradeDocument& t) { + j = nlohmann::json{ + {"RUN_ID", t.RUN_ID}, + {"@timestamp", t.timestamp}, + {"ingestedAt", t.ingestedAt}, + {"hostname", t.hostname}, + {"strategyUuid", t.strategyUuid}, + {"strategyName", t.strategyName}, + {"tradeId", t.tradeId}, + {"symbol", t.symbol}, + {"direction", t.direction}, + {"size", t.size}, + {"entryPrice", t.entryPrice}, + {"entryBid", t.entryBid}, + {"entryAsk", t.entryAsk}, + {"closePrice", t.closePrice}, + {"stopDistancePips", t.stopDistancePips}, + {"limitDistancePips", t.limitDistancePips}, + {"openTime", t.openTime}, + {"closeTime", t.closeTime}, + {"holdSeconds", t.holdSeconds}, + {"pnlPoints", t.pnlPoints}, + {"pnlPips", t.pnlPips}, + {"liquidated", t.liquidated}, + {"scalingFactor", t.scalingFactor}, + {"priceScale", t.priceScale}, + }; + // A zero distance means that exit leg was never armed — the precomputed + // trigger price is meaningless, so emit null rather than a fake level. + if (t.stopDistancePips != 0) { + j["stopPrice"] = t.stopPrice; + } else { + j["stopPrice"] = nullptr; + } + if (t.limitDistancePips != 0) { + j["limitPrice"] = t.limitPrice; + } else { + j["limitPrice"] = nullptr; + } +} diff --git a/source/run/reporting/tradeDocument.hpp b/source/run/reporting/tradeDocument.hpp new file mode 100644 index 0000000..f0e0d89 --- /dev/null +++ b/source/run/reporting/tradeDocument.hpp @@ -0,0 +1,65 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include + +#include + +// Wire shape for ONE closed trade, outcome_index::kTradesIndex — the opt-in +// per-trade companion to the per-run docs in tradingResults.hpp. Emitted only +// when $ELASTIC_TRADES_ENABLED=1, bulk-posted once at end of run. Plain header +// (not a module) for the same reason as tradingResults.hpp: nlohmann's ADL +// to_json does not resolve across module boundaries. +struct TradeDocument { + std::string RUN_ID; + // Serialised as `@timestamp` = SIMULATION closeTime, so Kibana plots + // trades across the historical backtest window. A freshly-run backtest + // therefore does NOT appear under "last 15 minutes" — widen the time range + // to the backtest window, or query `ingestedAt` (wall clock) instead. + std::string timestamp; + std::string ingestedAt; // wall clock when the batch was posted + std::string hostname; // machine that ran the backtest + std::string strategyUuid; // config.STRATEGY.UUID (one per param combo) + std::string strategyName; // config.STRATEGY.TRADING_VARIABLES.STRATEGY + std::string tradeId; // Trade.id ("T{n}"), unique per execution + std::string symbol; + std::string direction; // "LONG"/"SHORT" — static, keyword-aggregatable + std::int32_t size; + // Real decimal prices: raw INT32 points / symbol_scale::getPriceScale(). + // When the symbol's scale is unknown (priceScale == 0 below) the raw + // integer values are carried through unchanged. + double entryPrice; + double entryBid; + double entryAsk; + double closePrice; + // Exit trigger levels. Only meaningful when the matching distance is + // non-zero; serialised as JSON null when disarmed so Kibana doesn't plot + // fake levels (see to_json). + double stopPrice; + double limitPrice; + std::int32_t stopDistancePips; + std::int32_t limitDistancePips; + std::string openTime; // sim time, ISO-8601 UTC with milliseconds + std::string closeTime; // sim time, ISO-8601 UTC with milliseconds + double holdSeconds; // sim closeTime - openTime + std::int64_t pnlPoints; // exact realized PnL (points x size) + double pnlPips; // pnlPoints / (scalingFactor * size); 0 if unmeasurable + bool liquidated; // forced close by loss limit vs organic + int scalingFactor; // points-per-pip (pip math recoverable) + int priceScale; // raw->real divisor used; 0 = unknown, prices left raw + + // ISO-8601 UTC with milliseconds ("YYYY-MM-DDTHH:MM:SS.mmmZ"). Ticks are + // sub-second, so the seconds-only stamps used elsewhere would collapse + // trades onto the same instant in Kibana. Lives here (a normal TU) rather + // than in an import-std module since it needs POSIX gmtime_r. + static std::string isoUtcMillis(std::chrono::system_clock::time_point tp); +}; + +void to_json(nlohmann::json& j, const TradeDocument& t); diff --git a/source/run/reporting/tradingResults.cpp b/source/run/reporting/tradingResults.cpp index 78682a4..a43266a 100644 --- a/source/run/reporting/tradingResults.cpp +++ b/source/run/reporting/tradingResults.cpp @@ -68,11 +68,25 @@ nlohmann::json reportConfigJson(const tradingDefinitions::Configuration& c) { j["MAX_LOSS_PERCENT"] = decimalToJsonNumber(c.MAX_LOSS_PERCENT); const auto& v = c.STRATEGY.TRADING_VARIABLES; auto& tv = j["STRATEGY"]["TRADING_VARIABLES"]; - tv["STOP_DISTANCE_IN_PIPS"] = v.STOP_DISTANCE_IN_PIPS; - tv["LIMIT_DISTANCE_IN_PIPS"] = v.LIMIT_DISTANCE_IN_PIPS; + tv["STOP_DISTANCE_IN_ATR"] = v.STOP_DISTANCE_IN_ATR; + tv["LIMIT_DISTANCE_IN_ATR"] = v.LIMIT_DISTANCE_IN_ATR; tv["TRADING_SIZE"] = v.TRADING_SIZE; return j; } + +// Top-level copies of the batch identity (also present under config.*) so +// Kibana filters and aggregations don't have to reach into the config object. +// Omitted entirely for pre-batch configs — absent keys, not empty strings, so +// legacy documents keep their exact shape. +void appendBatchMetadata(nlohmann::json& j, + const tradingDefinitions::Configuration& c) { + if (!c.EXECUTION_TS.empty()) { + j["executionTimestamp"] = c.EXECUTION_TS; + } + if (!c.BATCH.empty()) { + j["batch"] = c.BATCH; + } +} } // namespace void to_json(nlohmann::json& j, const TradingResults& r) { @@ -84,6 +98,7 @@ void to_json(nlohmann::json& j, const TradingResults& r) { {"config", reportConfigJson(r.config)}, {"results", r.results}, }; + appendBatchMetadata(j, r.config); } void to_json(nlohmann::json& j, const TradingFailure& f) { @@ -92,10 +107,13 @@ void to_json(nlohmann::json& j, const TradingFailure& f) { {"@timestamp", f.timestamp}, {"durationSeconds", f.durationSeconds}, {"reason", f.reason}, + {"breachPnlPips", f.breachPnlPips}, + {"lossFloorPips", f.lossFloorPips}, {"hostname", f.hostname}, {"config", reportConfigJson(f.config)}, {"results", f.results}, }; + appendBatchMetadata(j, f.config); } void to_json(nlohmann::json& j, const TradeFinal& f) { @@ -104,7 +122,9 @@ void to_json(nlohmann::json& j, const TradeFinal& f) { {"@timestamp", f.timestamp}, {"durationSeconds", f.durationSeconds}, {"success", f.success}, + {"status", f.status}, {"hostname", f.hostname}, {"config", reportConfigJson(f.config)}, }; + appendBatchMetadata(j, f.config); } diff --git a/source/run/reporting/tradingResults.hpp b/source/run/reporting/tradingResults.hpp index 97122f8..4201f07 100644 --- a/source/run/reporting/tradingResults.hpp +++ b/source/run/reporting/tradingResults.hpp @@ -64,12 +64,17 @@ struct TradingResults { // Wire shape for a run that was cut off early (e.g. it breached the account // loss limit). Carries the stats as they stood at the cutoff so a failed run -// is still fully inspectable in Kibana; `reason` says why it was stopped. +// is still fully inspectable in Kibana. `reason` is a STATIC, low-cardinality +// string (it becomes reason.keyword — one distinct value per failure class, so +// terms aggregations work); the run-specific numbers travel in the dedicated +// numeric fields below, never interpolated into the string. struct TradingFailure { std::string RUN_ID; std::string timestamp; double durationSeconds; // wall-clock seconds until the cutoff - std::string reason; + std::string reason; // static failure class, aggregatable + double breachPnlPips; // account PnL at cutoff, pips of price movement + double lossFloorPips; // the configured pip budget that was breached std::string hostname; // machine that ran the backtest tradingDefinitions::Configuration config; TradingResultsStats results; @@ -79,12 +84,20 @@ struct TradingFailure { // the input Configuration plus the bare outcome flag, how long it took, and the // host that produced it. Unlike TradingResults/TradingFailure it carries no // per-trade stats — it is the at-a-glance "this run finished" signal. Lands in -// index "trading_final". +// the weekly outcome_index::kFinalBase index. struct TradeFinal { std::string RUN_ID; std::string timestamp; double durationSeconds; // wall-clock seconds for this run - int success; // 1 = completed, 0 = loss-limit cutoff + // 1 = performance gate cleared — the same population the results/winners + // indices receive; 0 = underperformed or loss-limit cutoff (see `status`). + int success; + // Terminal state as a keyword: "completed" (performance gate cleared — + // the run also reports to the results/winners index and may chain), + // "underperformed" + // (finished its ticks but failed the gate; this record is its only + // report), or "loss_limit_breached". Splits the success=0 class. + std::string status; std::string hostname; // machine that ran the backtest tradingDefinitions::Configuration config; diff --git a/source/run/trading/reviewStopAndLimit.cppm b/source/run/trading/reviewStopAndLimit.cppm index bf4a660..f8d21e9 100644 --- a/source/run/trading/reviewStopAndLimit.cppm +++ b/source/run/trading/reviewStopAndLimit.cppm @@ -27,7 +27,13 @@ export namespace trading { inline void reviewStopAndLimit(TradeManager& tradeManager, const PriceData& tick) { const Trade* trade = tradeManager.findActiveTrade(tick.symbol); if (trade == nullptr) return; - if (auto exitPrice = trading::exit_rules::checkExit(*trade, tick)) { + + // if-with-initializer over an optional: checkExit returns + // std::optional — nullopt (falsy) means no exit on this tick, a + // value is the close-side price the SL/TP fired at, and `*exitPrice` + // unwraps it for closeTrade to record as the trade's closePrice. const is + // free (it documents read-only intent, it doesn't change codegen). + if (const auto exitPrice = trading::exit_rules::checkExit(*trade, tick)) { tradeManager.closeTrade(tick.symbol, *exitPrice, tick); } } diff --git a/source/run/trading/runLoop.cppm b/source/run/trading/runLoop.cppm index 4b5b455..24b3bc5 100644 --- a/source/run/trading/runLoop.cppm +++ b/source/run/trading/runLoop.cppm @@ -13,11 +13,16 @@ module; export module runLoop; -import std; // replaces , +import std; // replaces , , , , + // , +import barStore; // bars::BarStore — shared per-symbol bar histories +import entryConditions; // conditions::check — pre-decide ATR gate +import marketHours; // market_hours::tradePermitted import tradeManager; // TradeManager import reviewStopAndLimit; // trading::reviewStopAndLimit import priceData; // PriceData import strategy; // IStrategy +import resultsSummary; // ResultsSummary::collect — the performance gate's score export namespace trading { @@ -29,18 +34,46 @@ struct RiskLimits { tradingDefinitions::DEFAULT_STARTING_BALANCE}; boost::decimal::decimal64_t maxLossPercent{0}; int maxOpenTrades{0}; + // Cap on trade entries within any sliding 60-second window of TICK time + // (backtests replay history, so wall clock would be meaningless). + // Disabled here by default like the other limits; production runs inherit + // RunConfiguration's default (60) via Operations::run. + int maxTradesPerMinute{0}; + // Peak-market-hours entry filter (market_hours::tradePermitted): when + // set, entries are allowed only inside the symbol's session window. + // Exits — reviewStopAndLimit, loss-limit liquidation, end-of-data + // closes, during() — are never gated. Off by default like the other + // limits. + bool peakHoursOnly{false}; // Points-per-pip of the run's symbol — converts the pip-denominated loss // floor into the integer points the PnL is tracked in. Defaults to 1 (floor // stays in raw points) for callers/tests that don't set it. Set by // Operations::run from the run's symbol; exact for single-asset-class runs. int pointsPerPip{1}; + // Post-run performance gate, evaluated once at the bottom of runTicks: + // a run that exhausts its ticks only counts as Completed when its + // performance score EXCEEDS minPerformanceScore AND its decisive trade + // count (winners + losers — breakevens carry no information and would + // pad the sample) EXCEEDS minDecisiveTrades; otherwise it returns + // Underperformed. Each threshold <= 0 disables that check, matching the + // other limits, so existing tests and unconstrained runs are unaffected. + boost::decimal::decimal64_t minPerformanceScore{0}; + int minDecisiveTrades{0}; + // Backtest horizon in months, required whenever minPerformanceScore is + // active: the score annualises returns over this window, and without it + // the score computes to 0 and the gate can never pass. Set by + // Operations::run from the run's LAST_MONTHS. + int lastMonths{0}; }; -// How a run ended: ran out of ticks, or was cut off because realized losses -// reached the account loss limit (the fail-fast path). +// How a run ended: ran out of ticks with the performance gate cleared +// (Completed), ran out of ticks but failed the gate — too weak a score or too +// thin a sample to be worth advancing (Underperformed) — or was cut off +// because losses reached the account loss limit (the fail-fast path). enum class RunStatus { Completed, LossLimitBreached, + Underperformed, }; // The per-tick backtest loop, factored out of Operations::run so it can be @@ -59,25 +92,60 @@ enum class RunStatus { // after the close phase, so a breaching run stops before opening anything // new. On breach every open trade is liquidated at its last marked price // (the close side of the most recent tick seen for its symbol), so the -// reported PnL is the true account PnL at the cutoff. +// reported PnL is the true account PnL at the cutoff. A run that exhausts its +// ticks closes any remaining open trades the same way — at their last marks — +// but as ordinary (non-liquidated) closes, so finalPnl/avgPnl account for +// every trade the run opened. A run that survives its ticks is then held to +// the RiskLimits performance gate before it may report Completed. +// `barStore`/`gateSeries` are the shared bar pipeline and the ATR entry +// conditions (see entryConditions). Production (Operations::run) passes the +// run's store — strategy timeframes and gate series registered — plus the +// gate, so entries get dynamic ATR-derived pip distances. Both default off: +// tests that script exact SL/TP geometry get a local empty store and the +// trading variables pass through to openTrade as literal pip distances, +// keeping the loop machinery testable without ATR warm-ups. inline RunStatus runTicks(TradeManager& tradeManager, IStrategy& strategy, - const std::vector& ticks, + const std::span ticks, const tradingDefinitions::TradingVariables& vars, - const RiskLimits& limits = {}) { + const RiskLimits& limits = {}, + bars::BarStore* barStore = nullptr, + const std::optional& gateSeries = + std::nullopt) { + constexpr boost::decimal::decimal64_t zero{0}; + + bars::BarStore localBarStore; + bars::BarStore& sharedBars = + barStore != nullptr ? *barStore : localBarStore; const bool lossLimitActive = limits.maxLossPercent > zero; - // Lowest equity the run may reach, in int64 points. balance * loss% gives - // the floor in pips; scaling by points-per-pip puts it in the same integer - // points the PnL is tracked in, e.g. 10000 at 5% on EURUSD (10 pts/pip) -> - // -500 pips -> -5000 points. Computed once; the per-tick check is integer. + + // Lowest equity the run may reach, in int64 points. NOTE: this is a PIP + // BUDGET, not a currency limit. The engine has no pip-value/notional model, + // so balance * loss% is read DIRECTLY as a pip count — 10000 at 5% means + // "500 pips", not "$500" (the two only coincide under the unstated + // assumption that one unit of size earns 1 currency unit per pip). Scaling + // by points-per-pip puts the budget in the same integer points the PnL is + // tracked in (EURUSD at 10 pts/pip -> -5000 points), and scaling by trade + // size keeps it a budget on PRICE MOVEMENT: PnL is points × size, so an + // unscaled floor would silently shrink to budget/size pips for any size + // above 1. Computed once; the per-tick check is integer. + const std::int64_t sizeScale = std::max(1, vars.TRADING_SIZE); const std::int64_t pnlFloorPoints = -static_cast( - limits.startingBalance * limits.maxLossPercent / 100 * limits.pointsPerPip); + limits.startingBalance * limits.maxLossPercent / 100 * limits.pointsPerPip) + * sizeScale; const auto lossLimitBreached = [&] { return lossLimitActive && tradeManager.calculatePnl() + tradeManager.unrealizedPnl() <= pnlFloorPoints; }; + // Timestamps of entries still inside the sliding one-minute rate window. + // Only pushed to while the cap is active, so it holds at most + // maxTradesPerMinute entries; ticks arrive time-ordered (ORDER BY + // timestamp), so evicting from the front is sufficient. + const bool tradeRateCapActive = limits.maxTradesPerMinute > 0; + std::deque recentOpens; + for (const auto& tick : ticks) { // Revalue this symbol's open trades at the new price so the equity @@ -89,26 +157,76 @@ inline RunStatus runTicks(TradeManager& tradeManager, // entry could race within the same tick. trading::reviewStopAndLimit(tradeManager, tick); + // Feed the shared bar pipeline on EVERY tick, ungated (a gap would + // corrupt the ATR and the strategies' histories) and BEFORE the + // entry gates: the ATR conditions and decide() judge this tick + // against bar state that already includes it. Deliberate trend- + // filter consequence (Ryan's call): the tick's own price counts as + // trend evidence — the filter reads "current price vs trailing + // EMA", not "previous tick vs EMA". + sharedBars.update(tick); + if (lossLimitBreached()) { tradeManager.closeAllTrades(tick); return RunStatus::LossLimitBreached; } - // Entry gates: at most one open trade per symbol, and (when capped) - // no more than maxOpenTrades positions across the whole run. The cap - // is checked first so a capped run skips decide() entirely. + // Entry gates: (when filtered) only inside the symbol's peak session + // window, at most one open trade per symbol, (when capped) no more + // than maxOpenTrades positions across the whole run, and (when + // capped) no more than maxTradesPerMinute entries in the sliding + // minute ending at this tick. The gates are checked first so a gated + // run skips decide() entirely — a skipped entry is skipped, never + // deferred to a later tick. markToMarket/reviewStopAndLimit above and + // during() below run on every tick regardless. const bool belowOpenTradeCap = limits.maxOpenTrades <= 0 || tradeManager.reviewAccount() < static_cast(limits.maxOpenTrades); - if (belowOpenTradeCap && + + bool belowTradeRateCap = true; + if (tradeRateCapActive) { + // The window is half-open, (tick - 60s, tick]: an entry exactly + // 60 seconds old has aged out and frees its slot on this tick. + while (!recentOpens.empty() && + tick.timestamp - recentOpens.front() >= std::chrono::minutes{1}) { + recentOpens.pop_front(); + } + belowTradeRateCap = + recentOpens.size() < + static_cast(limits.maxTradesPerMinute); + } + + const bool inSession = + !limits.peakHoursOnly || + market_hours::tradePermitted(tick.symbol, tick.timestamp); + + if (inSession && belowOpenTradeCap && belowTradeRateCap && !tradeManager.hasActiveTradeForSymbol(tick.symbol)) { - if (auto signal = strategy.decide(tick)) { - tradeManager.openTrade(tick, - vars.TRADING_SIZE, - *signal, - vars.STOP_DISTANCE_IN_PIPS, - vars.LIMIT_DISTANCE_IN_PIPS); + // ATR entry conditions (when wired), BEFORE decide(): spread vs + // ATR and the volatility floors, producing this entry's dynamic + // pip distances from the ATR multipliers. A failed check skips + // the entry — skipped, never deferred, same doctrine as the + // caps. Gate off (tests): the variables pass through literally. + std::optional distances; + const bool conditionsMet = + !gateSeries.has_value() || + (distances = conditions::check(sharedBars, *gateSeries, tick, + vars.STOP_DISTANCE_IN_ATR, + vars.LIMIT_DISTANCE_IN_ATR)) + .has_value(); + if (conditionsMet) { + if (auto signal = strategy.decide(tick, sharedBars)) { + tradeManager.openTrade( + tick, vars.TRADING_SIZE, *signal, + distances ? distances->stopPips + : vars.STOP_DISTANCE_IN_ATR, + distances ? distances->limitPips + : vars.LIMIT_DISTANCE_IN_ATR); + if (tradeRateCapActive) { + recentOpens.push_back(tick.timestamp); + } + } } } @@ -116,7 +234,7 @@ inline RunStatus runTicks(TradeManager& tradeManager, // (e.g. trailing stops, partial closes). The default // RandomStrategy implementation is a no-op now that exits are // handled by reviewStopAndLimit above. - strategy.during(tick, tradeManager); + strategy.during(tick, sharedBars, tradeManager); } // Entries and during() run after the per-tick check; catch a breach on @@ -127,6 +245,34 @@ inline RunStatus runTicks(TradeManager& tradeManager, } return RunStatus::LossLimitBreached; } + // Ran out of ticks with positions still open: close them at their last + // marked prices so finalPnl/avgPnl account for every trade the run opened + // (max drawdown and tradesOpened already see them; leaving them open would + // report a rosier finalPnl than the run's own equity curve). These are + // ordinary end-of-data closes, not loss-limit liquidations. + if (!ticks.empty()) { + tradeManager.closeAllTrades(ticks.back(), /*liquidated=*/false); + } + + // Performance gate (when configured — see RiskLimits): surviving the ticks + // is necessary but not sufficient. Evaluated after the closes above so the + // score sees every trade the run opened. ResultsSummary::collect is the + // same computation the reporting path stores, so this gate and the + // Elasticsearch documents can never disagree about a run's score. + if (limits.minPerformanceScore > zero || limits.minDecisiveTrades > 0) { + const auto stats = ResultsSummary::collect( + tradeManager, limits.startingBalance, limits.lastMonths, + limits.pointsPerPip); + const std::size_t decisiveTrades = stats.winners + stats.losers; + if (limits.minDecisiveTrades > 0 && + decisiveTrades <= static_cast(limits.minDecisiveTrades)) { + return RunStatus::Underperformed; + } + if (limits.minPerformanceScore > zero && + stats.performanceScore <= limits.minPerformanceScore) { + return RunStatus::Underperformed; + } + } return RunStatus::Completed; } diff --git a/source/run/trading/tradeManager.cppm b/source/run/trading/tradeManager.cppm index 51c28e0..99de2e2 100644 --- a/source/run/trading/tradeManager.cppm +++ b/source/run/trading/tradeManager.cppm @@ -57,6 +57,13 @@ private: public: TradeManager() = default; + // Entry-slippage stress toggle (config ENTRY_SLIPPAGE_TENTH_PIPS): an + // adverse haircut, in TENTHS of a pip, applied to what an entry PAYS + // (LONG fills above the ask, SHORT below the bid). 0 = off. The SL/TP + // anchors stay on the untouched tick — slippage moves the fill cost, + // not the market levels. Set by the backtest runner from the run + // config; the live runner's book-seeding managers keep the 0 default. + std::int32_t entrySlippageTenthPips{0}; std::string openTrade(const PriceData& tick, std::int32_t size, Direction direction, @@ -89,9 +96,12 @@ public: // Revalue open trades for this tick's symbol at its close-side price // (bid for LONG, ask for SHORT), updating their floating PnL. void markToMarket(const PriceData& tick); - // Liquidate every open trade at its last marked price (timestamped with - // `tick`), realizing the floating PnL — used when a run is cut off. - void closeAllTrades(const PriceData& tick); + // Close every open trade at its last marked price (timestamped with + // `tick`), realizing the floating PnL. `liquidated` distinguishes a + // loss-limit cutoff (the default, matching the historical call site) from + // an ordinary end-of-data close, so reporting's `liquidated` counter only + // counts forced closes. + void closeAllTrades(const PriceData& tick, bool liquidated = true); }; namespace { @@ -109,6 +119,7 @@ std::int64_t floatingPnlAt(const Trade& trade, std::int32_t mark) { void TradeManager::updateDrawdown() { const std::int64_t equity = closedPnl + openPnl; if (equity > peakEquity) peakEquity = equity; + const std::int64_t drop = peakEquity - equity; if (drop > maxDrawdown) maxDrawdown = drop; } @@ -120,6 +131,18 @@ std::string TradeManager::openTrade(const PriceData& tick, std::int32_t limitDistancePips) { auto price = (direction == Direction::LONG) ? tick.ask : tick.bid; Trade trade(price, size, direction, tick.symbol); + // Entry-slippage stress: worsen the PAID price only. scalingFactor is + // the symbol's points-per-pip (every table entry is a multiple of 10), + // so tenth-pips convert exactly — 3 tenths = 3 points FX, 300 metals. + // exitReferencePrice and the SL/TP anchors below stay on the raw tick, + // and the haircut flows into PnL through entryPrice alone. An unknown + // symbol's sentinel scale yields 0 slip — same fail-safe as elsewhere. + if (entrySlippageTenthPips > 0 && trade.scalingFactor > 0) { + const std::int32_t slipPoints = + entrySlippageTenthPips * trade.scalingFactor / 10; + trade.entryPrice += + (direction == Direction::LONG) ? slipPoints : -slipPoints; + } trade.openTime = tick.timestamp; // simulation time, not wall clock trade.entryBid = tick.bid; trade.entryAsk = tick.ask; @@ -159,10 +182,23 @@ std::string TradeManager::openTrade(const PriceData& tick, return it->second.id; } +// Revalue this tick's symbol's open trade at the new price so account equity +// (closedPnl + openPnl) reflects the current floating PnL rather than a stale +// mark. Called once per tick, before runTicks' loss-limit check. void TradeManager::markToMarket(const PriceData& tick) { + const auto it = activeTrades.find(std::string_view{tick.symbol}); if (it == activeTrades.end()) return; // no position: equity unchanged + + // A map iterator points at a pair: ->first is + // the key (symbol), ->second the mapped Trade. Binding a Trade& (not a + // copy) means the writes below land in the map's own stored entry. Trade& trade = it->second; + + // Value the position at the price that would CLOSE it: a LONG exits by + // selling at the bid, a SHORT by buying back at the ask. Fold the change + // since the last mark into openPnl — a member running sum across all open + // trades (one TradeManager per run) — as a delta, so no rescan of the map. const auto mark = (trade.direction == Direction::LONG) ? tick.bid : tick.ask; const auto updated = floatingPnlAt(trade, mark); openPnl += updated - trade.floatingPnl; @@ -171,17 +207,17 @@ void TradeManager::markToMarket(const PriceData& tick) { updateDrawdown(); // capture floating drawdown at this tick's mark } -void TradeManager::closeAllTrades(const PriceData& tick) { +void TradeManager::closeAllTrades(const PriceData& tick, bool liquidated) { // Snapshot symbols/prices first: closeTrade mutates activeTrades. Runs at - // most once per run (loss-limit breach), so the allocation is off the - // per-tick path. + // most once per run (loss-limit breach or end of data), so the allocation + // is off the per-tick path. std::vector> toClose; toClose.reserve(activeTrades.size()); for (const auto& [symbol, trade] : activeTrades) { toClose.emplace_back(symbol, trade.lastMarkPrice); } for (const auto& [symbol, price] : toClose) { - closeTrade(symbol, price, tick, /*liquidated=*/true); + closeTrade(symbol, price, tick, liquidated); } } @@ -230,9 +266,12 @@ bool TradeManager::closeTrade(std::string_view symbol, ts << std::put_time(&utc, "%Y-%m-%dT%H:%M:%SZ"); const char* side = (closed.direction == Direction::LONG) ? "BUY" : "SELL"; - // PnL is stored in points-per-lot; show it in pips for readability. - const double pnlPips = closed.scalingFactor != 0 - ? static_cast(closed.pnl) / closed.scalingFactor + // PnL is stored in points × size; normalise by both the symbol's + // points-per-pip and the trade size so the log shows pips of price + // movement (dividing by scalingFactor alone would print pips×size). + const double pnlPips = (closed.scalingFactor != 0 && closed.size != 0) + ? static_cast(closed.pnl) + / (static_cast(closed.scalingFactor) * closed.size) : 0.0; std::cout << ts.str() << ", Trade Closed, " << closed.symbol diff --git a/source/shared/aws/dynamoAuth.cpp b/source/shared/aws/dynamoAuth.cpp new file mode 100644 index 0000000..ce36ac1 --- /dev/null +++ b/source/shared/aws/dynamoAuth.cpp @@ -0,0 +1,126 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "shared/aws/dynamoAuth.hpp" + +#include +#include + +#include +#include +#include +#include + +#include "shared/utilities/backtestLog.hpp" + +namespace { + +// InitAPI must run once before any client exists; the matching ShutdownAPI +// runs at static destruction. Clients are constructed per pull (cheap next +// to the network hop, and never outliving the guard). +void ensureAwsInit() { + static const struct AwsGlobal { + Aws::SDKOptions options; + AwsGlobal() { Aws::InitAPI(options); } + ~AwsGlobal() { Aws::ShutdownAPI(options); } + } guard; + (void)guard; +} + +// One GetItem. `retryable` distinguishes a transport/service error (worth +// the C# single retry) from a definitive miss (item absent/incomplete). +std::optional pullOnce(const std::string& environment, + bool& retryable) { + retryable = false; + ensureAwsInit(); + + // Region + credentials resolve from the environment / profile via the + // SDK's default chain. + const Aws::Client::ClientConfiguration config; + const Aws::DynamoDB::DynamoDBClient client(config); + + Aws::DynamoDB::Model::GetItemRequest request; + request.SetTableName(Aws::String{aws_auth::kAuthTable}); + request.AddKey("id", Aws::DynamoDB::Model::AttributeValue().SetS( + aws_auth::authKey(environment).c_str())); + // The C# AuthDyanmoDBObject range key defaults to the literal "null". + request.AddKey("sort", Aws::DynamoDB::Model::AttributeValue().SetS("null")); + + const auto outcome = client.GetItem(request); + if (!outcome.IsSuccess()) { + retryable = true; + backtest_log::error("DynamoAuth: GetItem " + + aws_auth::authKey(environment) + " failed: " + + std::string{outcome.GetError().GetMessage()}); + return std::nullopt; + } + + const auto& item = outcome.GetResult().GetItem(); + if (item.empty()) { + backtest_log::error("DynamoAuth: no session item for " + + aws_auth::authKey(environment) + + " (has the login service written one?)"); + return std::nullopt; + } + + std::map fields; + for (const auto& [name, value] : item) { + const Aws::String& s = value.GetS(); // empty for non-string attrs + fields.emplace(std::string{name.c_str(), name.size()}, + std::string{s.c_str(), s.size()}); + } + auto auth = aws_auth::authFromItem(fields); + if (!auth) { + backtest_log::error("DynamoAuth: session item " + + aws_auth::authKey(environment) + + " is incomplete (need url, apikey, CST, " + "xSecurityToken)"); + } + return auth; +} + +} // namespace + +namespace aws_auth { + +std::string authKey(const std::string& environment) { + return "Auth#" + environment; +} + +std::optional authFromItem( + const std::map& item) { + const auto field = [&item](const char* name) -> std::string { + const auto it = item.find(name); + return it == item.end() ? std::string{} : it->second; + }; + ig_rest::Auth auth{ + .url = field("url"), + .apiKey = field("apikey"), + .cst = field("CST"), + .xSecurityToken = field("xSecurityToken"), + }; + if (auth.url.empty() || auth.apiKey.empty() || auth.cst.empty() || + auth.xSecurityToken.empty()) { + return std::nullopt; + } + return auth; +} + +std::optional pullAuthWithRetry(const std::string& environment) { + bool retryable = false; + if (auto auth = pullOnce(environment, retryable)) { + return auth; + } + if (!retryable) { + return std::nullopt; + } + // The C# PullWithRetry: exactly one more attempt after 100ms. + backtest_log::error("DynamoAuth: retrying credentials in 100ms"); + std::this_thread::sleep_for(std::chrono::milliseconds{100}); + return pullOnce(environment, retryable); +} + +} // namespace aws_auth diff --git a/source/shared/aws/dynamoAuth.hpp b/source/shared/aws/dynamoAuth.hpp new file mode 100644 index 0000000..e168253 --- /dev/null +++ b/source/shared/aws/dynamoAuth.hpp @@ -0,0 +1,51 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include + +#include "shared/ig/igRestClient.hpp" // ig_rest::Auth + +// IG session credentials from DynamoDB, mirroring the C# engine's +// Auth.PullWithRetry over the AuthDyanmoDBObject table: an external login +// service keeps one item per trading environment refreshed with the current +// CST / X-SECURITY-TOKEN (they expire), and every engine pulls it fresh per +// broker request: +// +// table "MarketDataLive", key { id: "Auth#", sort: "null" } +// fields: url, apikey, CST, xSecurityToken (strings; the C# object's +// date/authRoot are ignored here) +// +// AWS credentials and region come from the environment via the SDK's +// default chain (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / +// AWS_DEFAULT_REGION, or a profile). This header is AWS-free (the SDK stays +// behind dynamoAuth.cpp), same isolation pattern as tradeLocks.hpp, so +// module global module fragments can #include it alongside `import std`. +namespace aws_auth { + +inline constexpr std::string_view kAuthTable = "MarketDataLive"; + +// "Auth#" — the C# "Auth#" + TradingEnvironment key. The +// environment is expected lowercased ("live" / "demo"). +std::string authKey(const std::string& environment); + +// Pure item -> session mapping (exposed for tests): nullopt unless url, +// apikey, CST and xSecurityToken are all present and non-empty — a partial +// session is unusable and must read as "no auth". +std::optional authFromItem( + const std::map& item); + +// GetItem with ONE retry after 100ms on a transport/service error — the C# +// PullWithRetry. A missing item or an incomplete one is a definitive +// nullopt (no retry): the login service simply hasn't written a session for +// this environment. +std::optional pullAuthWithRetry(const std::string& environment); + +} // namespace aws_auth diff --git a/source/shared/experiments/chainMatcher.cppm b/source/shared/experiments/chainMatcher.cppm new file mode 100644 index 0000000..8e4faf2 --- /dev/null +++ b/source/shared/experiments/chainMatcher.cppm @@ -0,0 +1,780 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// chainMatcher — the pure half of the analysis worker (trackingReport style): +// counts non-overlapping occurrences of an experiment's activity chain in a +// tick stream. Everything here is I/O-free — ticks arrive as (mid, timestamp) +// pairs, the return values are plain data — so the unit tests need no servers. +// +// Matcher semantics (pinned by tests/chainMatcher.cpp): +// +// - Touch-within-window: a leg completes at the FIRST tick where its +// condition holds with tickTs - anchorTs <= WINDOW_SECONDS. "Drops 1% in +// 10m" = touches -1% at any tick inside the window. +// - Leg 1 is rolling, not anchored: tracked via trailing-window extremes +// (monotonic max/min deques with TIMESTAMP-based expiry — the +// rangeBarBuilder.cppm deque pattern adapted from tick-count to time +// expiry; legitimate here because experiments are explicitly time-window +// questions, and evaluation still advances only on tick timestamps — no +// clock anywhere). +// - Anchoring: leg N+1 anchors at leg N's completion tick (price = that +// tick's mid, window from its timestamp); evaluation of leg N+1 starts on +// the NEXT tick, so a chain can never telescope through one tick. +// - Expiry: the first tick past the window fails the whole chain (the +// condition can no longer be touched inside the window) — reset to +// scanning-for-leg-1 and re-evaluate leg 1 on that same tick. Leg-1 +// trackers are fed every tick, so no re-scan is needed. +// - Non-overlap: on completion, increment the count, clear ALL trackers, +// resume next tick. Lookback legs re-warm after each match (documented +// slight undercount). +// - Serial attempts: a single anchor in flight; new leg-1 triggers are +// picked up only after the current attempt fails (documented v1 +// undercount vs multi-anchor). +// - Per-attempt recorder (phase 2): every attempt tracks its excursion +// above/below the LEG-1 anchor mid from the anchor tick to resolution +// (the resolving tick included, pre-anchor and post-resolution prices +// excluded), the anchor->completion duration for completed attempts, and +// the ask-bid spread at the anchor tick. Samples are exact up to +// kMaxAttemptSamples per matcher, then sampling stops while counts stay +// exact (samplesTruncated flags it). A single-leg chain's attempts +// resolve on their own anchor tick, so their excursions/durations are +// truthfully zero. +// - Attempts/failures: every leg-1 completion counts an ATTEMPT (the +// denominator for P(chain | leg 1) — for a single-leg chain attempts == +// occurrences by construction); every expiry counts a failure attributed +// to the leg being sought (failuresByLeg; index 0 stays 0, leg 1 never +// expires). A chain that fails and immediately re-anchors on the failing +// tick counts BOTH the failure and the new attempt. Because attempts are +// serial single-anchor, completion rates derived from them are +// conservative in clustered/volatile periods (documented bias). +// - Gaps/warm-up: band/lookback legs require actual tick coverage — the +// tracker must have been fed for a full window with no inter-tick gap as +// long as the window itself (tracked by a third monotonic deque of +// inter-tick gaps; the first tick counts as an infinite gap, so coverage +// also encodes warm-up). A weekend gap therefore cannot satisfy "stays in +// band", and windows spanning gaps stay uncovered until a full fresh +// window of ticks has accumulated. +// - Arithmetic: mid = (int64(ask)+bid)/2; percent thresholds are +// precomputed at config time as pctScaled = llround(pct * 1e6); every +// per-tick test is pure int64, e.g. +// (rollMax - cur) * 100'000'000 >= rollMax * pctScaled +// (both sides of (rollMax-cur)/rollMax >= pct/100 multiplied out) — no +// per-tick floating point. +// - RangeRelativeMove basis: the trailing high-low mid range over +// LOOKBACK_SECONDS (monotonic deques), NOT bar-based atr::calculate — +// matches the rangeBarBuilder doctrine and keeps a bar builder out of the +// matcher. Threshold = max(1, range x ATR_MULTIPLE) points; the JSON +// field stays ATR_MULTIPLE. For anchored legs the threshold FREEZES at +// the anchor tick (the volatility the setup completed under); as leg 1 it +// is recomputed per tick, which is meaningful when LOOKBACK_SECONDS < +// WINDOW_SECONDS (a move measured against a shorter recent-range basis) — +// with ATR_MULTIPLE >= 1 and LOOKBACK >= WINDOW the basis always contains +// the move itself, so the leg can never fire (documented v1 shape). + +module; + +#include "shared/experiments/experimentConfig.hpp" + +export module chainMatcher; + +import std; +import priceData; // PriceData — evaluateExperiment's input stream + +namespace { + +using TimePoint = std::chrono::system_clock::time_point; +using Seconds = std::chrono::seconds; + +// Trailing time-window extremes plus tick coverage, the rangeBarBuilder +// monotonic-deque pattern with timestamp expiry. Entries are valid while +// entry.ts + window >= ts (boundary inclusive, matching the anchored legs' +// tickTs - anchorTs <= WINDOW test). +class Tracker { +public: + explicit Tracker(const Seconds window) : window_(window) {} + + // Feed one tick: expire, snapshot the pre-insert extremes (NewExtreme's + // strict "beats every PRIOR tick in the lookback" test), insert, and + // record the inter-tick gap for the coverage measure. + void feed(const TimePoint ts, const std::int32_t mid) { + // Expire entries (and gap records) older than the trailing window. + while (!maxDeque_.empty() && maxDeque_.front().ts + window_ < ts) { + maxDeque_.pop_front(); + } + while (!minDeque_.empty() && minDeque_.front().ts + window_ < ts) { + minDeque_.pop_front(); + } + while (!gapDeque_.empty() && gapDeque_.front().ts + window_ < ts) { + gapDeque_.pop_front(); + } + + preMax_ = maxDeque_.empty() ? std::optional{} + : maxDeque_.front().price; + preMin_ = minDeque_.empty() ? std::optional{} + : minDeque_.front().price; + + // Monotonic insert: popping equals keeps the newer entry, which + // survives expiry longer (rangeBarBuilder's rule). + while (!maxDeque_.empty() && maxDeque_.back().price <= mid) { + maxDeque_.pop_back(); + } + maxDeque_.push_back({ts, mid}); + while (!minDeque_.empty() && minDeque_.back().price >= mid) { + minDeque_.pop_back(); + } + minDeque_.push_back({ts, mid}); + + // Coverage: track the largest inter-tick gap still inside the window + // via its own monotonic max deque. The first tick ever fed counts as + // an infinite gap, so covered() only turns true once a full window of + // gap-free ticks has accumulated — warm-up and weekend gaps take the + // same path. + const auto gap = lastTick_ ? ts - *lastTick_ + : std::chrono::system_clock::duration::max(); + lastTick_ = ts; + while (!gapDeque_.empty() && gapDeque_.back().gap <= gap) { + gapDeque_.pop_back(); + } + gapDeque_.push_back({ts, gap}); + } + + [[nodiscard]] bool hasData() const { return !maxDeque_.empty(); } + [[nodiscard]] std::int32_t max() const { return maxDeque_.front().price; } + [[nodiscard]] std::int32_t min() const { return minDeque_.front().price; } + [[nodiscard]] std::optional preMax() const { return preMax_; } + [[nodiscard]] std::optional preMin() const { return preMin_; } + + // A full window of real tick history: no gap as long as the window itself + // remains inside it (the first-tick infinite gap covers warm-up). + [[nodiscard]] bool covered() const { + return !gapDeque_.empty() && gapDeque_.front().gap < window_; + } + + void reset() { + maxDeque_.clear(); + minDeque_.clear(); + gapDeque_.clear(); + lastTick_.reset(); + preMax_.reset(); + preMin_.reset(); + } + +private: + struct Entry { + TimePoint ts; + std::int32_t price; + }; + struct GapEntry { + TimePoint ts; + std::chrono::system_clock::duration gap; + }; + + Seconds window_; + std::deque maxDeque_; + std::deque minDeque_; + std::deque gapDeque_; + std::optional lastTick_; + std::optional preMax_; + std::optional preMin_; +}; + +} // namespace + +export namespace chain_matcher { + +// Per-matcher cap on attempt SAMPLES (excursions/durations). Counts are +// never capped — beyond this, quantiles come from a truncated population and +// the outcome's samplesTruncated flag says so. +inline constexpr std::size_t kMaxAttemptSamples = 100'000; + +// One resolved attempt's excursion record: max points above/below the leg-1 +// anchor over the attempt's lifetime, plus the anchor itself so percent +// variants can be derived at aggregation time (FP once per attempt at most, +// never per tick). +struct ExcursionSample { + std::int64_t abovePoints = 0; + std::int64_t belowPoints = 0; + std::int32_t anchorMid = 0; +}; + +struct MatchStats { + std::uint64_t occurrences = 0; + // Leg-1 completions, i.e. anchors created — the denominator for + // P(chain | leg 1). Serial single-anchor, so conservative in clustered + // periods (see the module comment). Equals occurrences for a + // single-leg chain. + std::uint64_t attempts = 0; + // Expiry counts attributed to the leg being sought when the chain died. + // Sized to the chain length; index 0 stays 0 (leg 1 never expires). + std::vector failuresByLeg; + // Raw attempt samples, completed and failed kept separate (stop-loss vs + // take-profit populations). completionSeconds is parallel to + // completedExcursions (pushed under the same cap gate). + std::vector completedExcursions; + std::vector failedExcursions; + std::vector completionSeconds; + // Ask-bid spread at every attempt's anchor tick — mean only, so a plain + // sum/count pair that never truncates. + std::int64_t spreadAtTriggerSum = 0; + std::uint64_t spreadAtTriggerCount = 0; + bool samplesTruncated = false; +}; + +// One symbol's matcher: feed every tick, in stream order. Pure state machine — +// see the module comment for the pinned semantics. +class ChainMatcher { +public: + // Validates the chain (throws std::invalid_argument on an empty chain or + // a leg whose primitive is missing the fields it reads) and precomputes + // every per-tick threshold as int64, so onTick never touches FP. + // maxAttemptSamples overrides the sample cap — a test knob; production + // callers take the default. + explicit ChainMatcher(const experiments::ExperimentConfig& config, + const std::size_t maxAttemptSamples = kMaxAttemptSamples) + : maxSamples_(maxAttemptSamples) { + if (config.CHAIN.empty()) { + throw std::invalid_argument("ChainMatcher: CHAIN must not be empty"); + } + legs_.reserve(config.CHAIN.size()); + for (std::size_t i = 0; i < config.CHAIN.size(); ++i) { + legs_.push_back(makeLeg(config.CHAIN[i], i)); + } + stats_.failuresByLeg.assign(legs_.size(), 0); + } + + void onTick(const std::int32_t mid, const std::int32_t spread, + const TimePoint ts) { + // All legs' rolling trackers are fed every tick (so a failed attempt + // can re-evaluate leg 1 with no re-scan, and a chained leg's basis is + // already warm when its turn comes); the state machine below consults + // only the active leg. + for (LegRuntime& leg : legs_) { + if (leg.moveTracker) { + leg.moveTracker->feed(ts, mid); + } + if (leg.basisTracker) { + leg.basisTracker->feed(ts, mid); + } + } + + if (active_ > 0) { + // The in-flight attempt's excursion vs its LEG-1 anchor — this + // tick included, whether it advances, resolves, or expires the + // attempt. + const auto cur = static_cast(mid); + attemptAbove_ = std::max(attemptAbove_, cur - attemptAnchorMid_); + attemptBelow_ = std::max(attemptBelow_, attemptAnchorMid_ - cur); + + const LegRuntime& leg = legs_[active_]; + if (ts - anchorTs_ > leg.window) { + // Expiry: the condition can no longer be touched inside the + // window — the whole chain fails, attributed to the leg + // being sought. Fall through to re-evaluate leg 1 on this + // same tick (which may itself anchor a fresh attempt). + recordAttempt(/*completed=*/false, ts); + ++stats_.failuresByLeg[active_]; + active_ = 0; + } else if (conditionHolds(active_, mid)) { + completeActiveLeg(mid, spread, ts); + return; + } else { + return; + } + } + + // Scanning for leg 1 (rolling, no anchor). + if (conditionHolds(0, mid)) { + completeActiveLeg(mid, spread, ts); + } + } + + [[nodiscard]] const MatchStats& stats() const { return stats_; } + + // 0-based index of the leg currently being sought — for tests. + [[nodiscard]] std::size_t activeLeg() const { return active_; } + +private: + struct LegRuntime { + experiments::ActivityType type; + Seconds window{0}; // completion window (anchored legs) + std::int64_t pctScaled = 0; // llround(|MOVE_PERCENT| * 1e6) + bool negativeMove = false; // DirectionalMove: drop vs rise + int direction = 0; // NewExtreme / RangeRelativeMove + std::int64_t atrScaled = 0; // llround(ATR_MULTIPLE * 1e6) + // Rolling extremes over WINDOW_SECONDS — leg 1's anchor substitute + // (DirectionalMove / RangeRelativeMove in first position only). + std::optional moveTracker; + // Trailing basis over the band/lookback window (StaysInBand, + // NewExtreme, RangeRelativeMove — any position). + std::optional basisTracker; + }; + + static LegRuntime makeLeg(const experiments::Activity& activity, + const std::size_t index) { + using experiments::ActivityType; + LegRuntime leg{.type = activity.TYPE, + .window = Seconds(activity.WINDOW_SECONDS)}; + const auto requirePositive = [&](const int value, const char* field) { + if (value < 1) { + throw std::invalid_argument( + std::format("ChainMatcher: leg {} ({}) requires {} >= 1", + index + 1, experiments::toString(activity.TYPE), + field)); + } + }; + const auto requireDirection = [&] { + if (activity.DIRECTION != 1 && activity.DIRECTION != -1) { + throw std::invalid_argument(std::format( + "ChainMatcher: leg {} ({}) requires DIRECTION of +1 or -1", + index + 1, experiments::toString(activity.TYPE))); + } + leg.direction = activity.DIRECTION; + }; + + switch (activity.TYPE) { + case ActivityType::DirectionalMove: { + if (activity.MOVE_PERCENT == 0.0) { + throw std::invalid_argument(std::format( + "ChainMatcher: leg {} (DirectionalMove) requires a " + "non-zero MOVE_PERCENT", + index + 1)); + } + requirePositive(activity.WINDOW_SECONDS, "WINDOW_SECONDS"); + leg.negativeMove = activity.MOVE_PERCENT < 0.0; + leg.pctScaled = + std::llround(std::abs(activity.MOVE_PERCENT) * 1e6); + if (index == 0) { + leg.moveTracker.emplace(leg.window); + } + break; + } + case ActivityType::StaysInBand: { + if (activity.MOVE_PERCENT == 0.0) { + throw std::invalid_argument(std::format( + "ChainMatcher: leg {} (StaysInBand) requires a " + "non-zero MOVE_PERCENT (band width)", + index + 1)); + } + // The band is measured over LOOKBACK_SECONDS when set, + // falling back to WINDOW_SECONDS — so a leg-1 band needs no + // separate deadline field. + const int bandSeconds = activity.LOOKBACK_SECONDS > 0 + ? activity.LOOKBACK_SECONDS + : activity.WINDOW_SECONDS; + requirePositive(bandSeconds, "LOOKBACK_SECONDS (or WINDOW_SECONDS)"); + if (index > 0) { + requirePositive(activity.WINDOW_SECONDS, "WINDOW_SECONDS"); + } + leg.pctScaled = + std::llround(std::abs(activity.MOVE_PERCENT) * 1e6); + leg.basisTracker.emplace(Seconds(bandSeconds)); + break; + } + case ActivityType::NewExtreme: { + requirePositive(activity.LOOKBACK_SECONDS, "LOOKBACK_SECONDS"); + requireDirection(); + if (index > 0) { + requirePositive(activity.WINDOW_SECONDS, "WINDOW_SECONDS"); + } + leg.basisTracker.emplace(Seconds(activity.LOOKBACK_SECONDS)); + break; + } + case ActivityType::RangeRelativeMove: { + requirePositive(activity.LOOKBACK_SECONDS, "LOOKBACK_SECONDS"); + requirePositive(activity.WINDOW_SECONDS, "WINDOW_SECONDS"); + requireDirection(); + if (activity.ATR_MULTIPLE <= 0.0) { + throw std::invalid_argument(std::format( + "ChainMatcher: leg {} (RangeRelativeMove) requires " + "ATR_MULTIPLE > 0", + index + 1)); + } + leg.atrScaled = std::llround(activity.ATR_MULTIPLE * 1e6); + leg.basisTracker.emplace(Seconds(activity.LOOKBACK_SECONDS)); + if (index == 0) { + leg.moveTracker.emplace(leg.window); + } + break; + } + } + return leg; + } + + // max(1, range x ATR_MULTIPLE) in points — the >= 1 clamp stops a + // dead-flat basis (threshold 0) from firing on a zero move. + [[nodiscard]] static std::int64_t rangeThreshold(const std::int64_t range, + const std::int64_t atrScaled) { + return std::max(1, + (range * atrScaled + 500'000) / 1'000'000); + } + + // Does leg `index`'s condition hold at the current tick? Leg 1 (index 0) + // measures against its rolling trackers; anchored legs against + // anchorMid_. All int64, no per-tick FP. + [[nodiscard]] bool conditionHolds(const std::size_t index, + const std::int32_t mid) const { + using experiments::ActivityType; + const LegRuntime& leg = legs_[index]; + const auto cur = static_cast(mid); + + switch (leg.type) { + case ActivityType::DirectionalMove: { + if (index == 0) { + const Tracker& roll = *leg.moveTracker; + if (!roll.hasData()) { + return false; + } + // (extreme - cur) / extreme >= pct/100, multiplied out. + if (leg.negativeMove) { + const auto rollMax = static_cast(roll.max()); + return (rollMax - cur) * 100'000'000 >= + rollMax * leg.pctScaled; + } + const auto rollMin = static_cast(roll.min()); + return (cur - rollMin) * 100'000'000 >= + rollMin * leg.pctScaled; + } + const auto anchor = static_cast(anchorMid_); + if (leg.negativeMove) { + return (anchor - cur) * 100'000'000 >= anchor * leg.pctScaled; + } + return (cur - anchor) * 100'000'000 >= anchor * leg.pctScaled; + } + case ActivityType::StaysInBand: { + const Tracker& basis = *leg.basisTracker; + if (!basis.covered()) { + return false; // a gap/warm-up window can't satisfy a band + } + const auto range = static_cast(basis.max()) - + basis.min(); + return range * 100'000'000 <= cur * leg.pctScaled; + } + case ActivityType::NewExtreme: { + const Tracker& basis = *leg.basisTracker; + if (!basis.covered()) { + return false; // no fire before a full lookback of ticks + } + // Strict: the current mid must beat every PRIOR tick in the + // lookback (pre-insert snapshot), not merely equal it. + if (leg.direction > 0) { + return basis.preMax() && cur > *basis.preMax(); + } + return basis.preMin() && cur < *basis.preMin(); + } + case ActivityType::RangeRelativeMove: { + const Tracker& basis = *leg.basisTracker; + if (index == 0) { + if (!basis.covered()) { + return false; + } + const auto range = static_cast(basis.max()) - + basis.min(); + const std::int64_t threshold = + rangeThreshold(range, leg.atrScaled); + const Tracker& roll = *leg.moveTracker; + if (leg.direction < 0) { + return static_cast(roll.max()) - cur >= + threshold; + } + return cur - static_cast(roll.min()) >= + threshold; + } + // Anchored: threshold frozen at the anchor tick (see + // completeActiveLeg); invalid when the basis was uncovered at + // anchor time — the leg then never fires and the chain + // expires naturally. + if (!anchorThresholdValid_) { + return false; + } + const auto anchor = static_cast(anchorMid_); + if (leg.direction < 0) { + return anchor - cur >= anchorThreshold_; + } + return cur - anchor >= anchorThreshold_; + } + } + return false; + } + + // The resolved attempt's samples, under the shared cap gate (counts are + // never capped — only samples). completionSeconds stays parallel to + // completedExcursions because both are pushed here together. + void recordAttempt(const bool completed, const TimePoint ts) { + std::vector& samples = + completed ? stats_.completedExcursions : stats_.failedExcursions; + if (samples.size() >= maxSamples_) { + stats_.samplesTruncated = true; + return; + } + samples.push_back({attemptAbove_, attemptBelow_, + static_cast(attemptAnchorMid_)}); + if (completed) { + stats_.completionSeconds.push_back( + std::chrono::duration_cast( + ts - attemptAnchorTs_) + .count()); + } + } + + // The active leg's condition held at this tick: either the chain is done + // (count it, clear everything, resume next tick) or the next leg anchors + // here — and, since evaluation returns to the caller, it is first + // evaluated on the NEXT tick, never its own anchor tick. + void completeActiveLeg(const std::int32_t mid, const std::int32_t spread, + const TimePoint ts) { + if (active_ == 0) { + ++stats_.attempts; // a leg-1 completion IS an attempt + // Arm the attempt recorder at the leg-1 anchor: excursions are + // measured from HERE for the whole chain (anchorMid_ moves at + // every later leg; this baseline must not), and the spread is + // sampled at this tick only. + attemptAnchorMid_ = mid; + attemptAnchorTs_ = ts; + attemptAbove_ = 0; + attemptBelow_ = 0; + stats_.spreadAtTriggerSum += spread; + ++stats_.spreadAtTriggerCount; + } + if (active_ + 1 == legs_.size()) { + ++stats_.occurrences; + recordAttempt(/*completed=*/true, ts); + resetAfterMatch(); + return; + } + ++active_; + anchorMid_ = mid; + anchorTs_ = ts; + + // RangeRelativeMove anchored legs freeze their threshold from the + // basis range as it stood at the anchor — the volatility the setup + // completed under, not whatever the move itself later inflates it to. + const LegRuntime& next = legs_[active_]; + if (next.type == experiments::ActivityType::RangeRelativeMove) { + const Tracker& basis = *next.basisTracker; + anchorThresholdValid_ = basis.covered(); + anchorThreshold_ = + anchorThresholdValid_ + ? rangeThreshold(static_cast(basis.max()) - + basis.min(), + next.atrScaled) + : 0; + } + } + + // Non-overlap: after a full match, every tracker is cleared — a pre-match + // extreme can never seed the next occurrence, and lookback legs re-warm + // (documented slight undercount). + void resetAfterMatch() { + active_ = 0; + for (LegRuntime& leg : legs_) { + if (leg.moveTracker) { + leg.moveTracker->reset(); + } + if (leg.basisTracker) { + leg.basisTracker->reset(); + } + } + } + + std::vector legs_; + MatchStats stats_; + std::size_t maxSamples_ = kMaxAttemptSamples; + std::size_t active_ = 0; // index of the leg currently sought + std::int32_t anchorMid_ = 0; // active leg's anchor (legs > 0) + TimePoint anchorTs_{}; + std::int64_t anchorThreshold_ = 0; // frozen RangeRelativeMove threshold + bool anchorThresholdValid_ = false; + // The in-flight attempt's recorder state (leg-1 anchor baseline). + std::int64_t attemptAnchorMid_ = 0; + TimePoint attemptAnchorTs_{}; + std::int64_t attemptAbove_ = 0; + std::int64_t attemptBelow_ = 0; +}; + +// Aggregation helpers for evaluateExperiment — FP is fine here, this runs +// once per experiment after the scan, never per tick. +namespace detail { + +// Nearest-rank percentile over an already-sorted, non-empty vector. +inline double nearestRank(const std::vector& sorted, const double q) { + const auto rank = static_cast( + std::ceil(q * static_cast(sorted.size()))); + return sorted[std::max(rank, 1) - 1]; +} + +inline experiments::QuantilePair quantiles(std::vector& values) { + std::ranges::sort(values); + return {nearestRank(values, 0.5), nearestRank(values, 0.9)}; +} + +// Which excursion side counts as FAVORABLE (MFE), from the final leg's +// implied direction. StaysInBand has none — "up" is the documented default. +inline bool favorableUp(const experiments::Activity& finalLeg) { + switch (finalLeg.TYPE) { + case experiments::ActivityType::DirectionalMove: + return finalLeg.MOVE_PERCENT >= 0.0; + case experiments::ActivityType::NewExtreme: + case experiments::ActivityType::RangeRelativeMove: + return finalLeg.DIRECTION >= 0; + case experiments::ActivityType::StaysInBand: + return true; + } + return true; +} + +// Summarise one attempt population; nullopt when empty (the doc then reads +// null, never fake zeros). +inline std::optional summariseExcursions( + const std::vector& samples, const bool up) { + if (samples.empty()) { + return std::nullopt; + } + std::vector mfePoints, maePoints, mfePercent, maePercent; + mfePoints.reserve(samples.size()); + maePoints.reserve(samples.size()); + mfePercent.reserve(samples.size()); + maePercent.reserve(samples.size()); + for (const ExcursionSample& s : samples) { + const auto favorable = + static_cast(up ? s.abovePoints : s.belowPoints); + const auto adverse = + static_cast(up ? s.belowPoints : s.abovePoints); + mfePoints.push_back(favorable); + maePoints.push_back(adverse); + mfePercent.push_back(100.0 * favorable / s.anchorMid); + maePercent.push_back(100.0 * adverse / s.anchorMid); + } + return experiments::ExcursionStats{ + .samples = samples.size(), + .mfePoints = quantiles(mfePoints), + .maePoints = quantiles(maePoints), + .mfePercent = quantiles(mfePercent), + .maePercent = quantiles(maePercent), + }; +} + +} // namespace detail + +// Replays one experiment over a (possibly multi-symbol, UNION-ALL ordered) +// tick stream: the stream is demuxed into one ChainMatcher per symbol, so +// each symbol's chain is counted independently and the aggregate sums them. +// Throws (std::invalid_argument) on an invalid chain — the worker's +// per-experiment poison-pill path. +experiments::ExperimentOutcome evaluateExperiment( + const std::span ticks, + const experiments::ExperimentConfig& config) { + // Validate up front (even for an empty stream) so a malformed experiment + // is retired loudly by the worker, never reported as "0 occurrences". + [[maybe_unused]] const ChainMatcher validation(config); + + std::map matchers; + + // Per-symbol tick counts and spans, tracked alongside the matchers so a + // sparsely ticked symbol's rate is judged against its OWN coverage, not + // the shared stream's. + struct SymbolSpan { + std::uint64_t ticks = 0; + TimePoint minTs = TimePoint::max(); + TimePoint maxTs = TimePoint::min(); + }; + std::map spans; + + experiments::ExperimentOutcome outcome; + outcome.ticksScanned = ticks.size(); + outcome.failuresByLeg.assign(config.CHAIN.size(), 0); + + TimePoint minTs = TimePoint::max(); + TimePoint maxTs = TimePoint::min(); + for (const PriceData& tick : ticks) { + const auto mid = static_cast( + (static_cast(tick.ask) + tick.bid) / 2); + ChainMatcher& matcher = + matchers.try_emplace(tick.symbol, config).first->second; + // Delta-detection: a matcher completes at most one occurrence per + // tick, so an increment here identifies the completing tick — the + // calendar bucketing lives HERE, keeping the matcher core free of + // it. + const std::uint64_t before = matcher.stats().occurrences; + matcher.onTick(mid, tick.ask - tick.bid, tick.timestamp); + if (matcher.stats().occurrences > before) { + const auto day = + std::chrono::floor(tick.timestamp); + const std::chrono::year_month_day ymd{day}; + ++outcome.occurrencesByMonth[std::format( + "{:04}-{:02}", static_cast(ymd.year()), + static_cast(ymd.month()))]; + const auto hour = std::chrono::duration_cast( + tick.timestamp - day) + .count(); + ++outcome.occurrencesByHourUtc[static_cast(hour)]; + } + + SymbolSpan& span = spans[tick.symbol]; + ++span.ticks; + span.minTs = std::min(span.minTs, tick.timestamp); + span.maxTs = std::max(span.maxTs, tick.timestamp); + minTs = std::min(minTs, tick.timestamp); + maxTs = std::max(maxTs, tick.timestamp); + } + + if (!ticks.empty()) { + outcome.daysSpanned = + std::chrono::duration>(maxTs - minTs) + .count(); + } + // Merged attempt samples across the demuxed matchers (bounded by the + // per-matcher cap x symbol count). + std::vector completed; + std::vector failed; + std::vector completionSeconds; + std::int64_t spreadSum = 0; + std::uint64_t spreadCount = 0; + for (const auto& [symbol, matcher] : matchers) { + const MatchStats& stats = matcher.stats(); + outcome.occurrences += stats.occurrences; + outcome.attempts += stats.attempts; + for (std::size_t leg = 0; leg < stats.failuresByLeg.size(); ++leg) { + outcome.failuresByLeg[leg] += stats.failuresByLeg[leg]; + } + const SymbolSpan& span = spans.at(symbol); + outcome.perSymbol[symbol] = experiments::SymbolOutcome{ + .occurrences = stats.occurrences, + .ticksScanned = span.ticks, + .daysSpanned = + std::chrono::duration>(span.maxTs - + span.minTs) + .count(), + }; + outcome.occurrencesBySymbol[symbol] = stats.occurrences; // v1 shape + + completed.insert(completed.end(), stats.completedExcursions.begin(), + stats.completedExcursions.end()); + failed.insert(failed.end(), stats.failedExcursions.begin(), + stats.failedExcursions.end()); + for (const std::int64_t seconds : stats.completionSeconds) { + completionSeconds.push_back(static_cast(seconds)); + } + spreadSum += stats.spreadAtTriggerSum; + spreadCount += stats.spreadAtTriggerCount; + outcome.samplesTruncated = + outcome.samplesTruncated || stats.samplesTruncated; + } + + const bool up = detail::favorableUp(config.CHAIN.back()); + outcome.excursionOrientation = up ? "up" : "down"; + outcome.completedAttempts = detail::summariseExcursions(completed, up); + outcome.failedAttempts = detail::summariseExcursions(failed, up); + if (!completionSeconds.empty()) { + outcome.completionSeconds = detail::quantiles(completionSeconds); + } + if (spreadCount > 0) { + outcome.meanSpreadAtTriggerPoints = + static_cast(spreadSum) / static_cast(spreadCount); + } + return outcome; +} + +} // namespace chain_matcher diff --git a/source/shared/experiments/experimentConfig.hpp b/source/shared/experiments/experimentConfig.hpp new file mode 100644 index 0000000..835ddbc --- /dev/null +++ b/source/shared/experiments/experimentConfig.hpp @@ -0,0 +1,203 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +// Intentionally a plain header, NOT a .cppm module — see runConfiguration.hpp +// for why JSON-serializable structs stay GMF-includable headers in this repo. +// +// The experiment model: a chain of "activities" asked of the tick history — +// "price drops 1% over 10 minutes, then rises 0.5% in a further 10 minutes: +// how many times does that occur?" — without writing a full strategy. The +// producer (experimentsCommand) spans a parameter grid of these out to Redis; +// the analysis worker replays them against QuestDB ticks via chainMatcher. +namespace experiments { + +// The primitive an activity tests. Kept deliberately small for v1: +// - DirectionalMove: price moves MOVE_PERCENT (signed: negative = drop) +// within WINDOW_SECONDS. +// - StaysInBand: the trailing band window's mid range stays within +// MOVE_PERCENT (total width, sign ignored). +// - NewExtreme: price strictly exceeds the trailing LOOKBACK_SECONDS +// high (DIRECTION +1) or low (-1). +// - RangeRelativeMove: price moves ATR_MULTIPLE x the trailing +// LOOKBACK_SECONDS high-low range ("ATR"-style, but +// deque-tracked tick range, not bar ATR — see +// chainMatcher) in DIRECTION within WINDOW_SECONDS. +enum class ActivityType { + DirectionalMove, + StaysInBand, + NewExtreme, + RangeRelativeMove, +}; + +inline std::string toString(const ActivityType type) { + switch (type) { + case ActivityType::DirectionalMove: return "DirectionalMove"; + case ActivityType::StaysInBand: return "StaysInBand"; + case ActivityType::NewExtreme: return "NewExtreme"; + case ActivityType::RangeRelativeMove: return "RangeRelativeMove"; + } + throw std::invalid_argument("experiments::toString: invalid ActivityType"); +} + +// Unknown names throw — a payload naming a type this binary doesn't know is a +// poison pill the worker must retire loudly, never misread as a default. +inline ActivityType activityTypeFromString(const std::string& name) { + if (name == "DirectionalMove") return ActivityType::DirectionalMove; + if (name == "StaysInBand") return ActivityType::StaysInBand; + if (name == "NewExtreme") return ActivityType::NewExtreme; + if (name == "RangeRelativeMove") return ActivityType::RangeRelativeMove; + throw std::invalid_argument( + "experiments::activityTypeFromString: unknown ActivityType '" + name + + "'"); +} + +// One leg of the chain. Deliberately FLAT (every primitive's knobs in one +// struct, unused ones left at their defaults) so the sweep's leg-prefixed +// parameter names (LEG1_MOVE_PERCENT, ...) map on without any per-type +// machinery — mirroring how Combination is a flat map. +// Which fields each TYPE reads is validated by ChainMatcher's ctor. +struct Activity { + ActivityType TYPE = ActivityType::DirectionalMove; + double MOVE_PERCENT = 0.0; // signed for DirectionalMove; width for band + int WINDOW_SECONDS = 0; // completion window (and leg-1 trailing window) + int LOOKBACK_SECONDS = 0; // trailing basis window (band/extreme/range) + int DIRECTION = 0; // +1 / -1 for NewExtreme & RangeRelativeMove + double ATR_MULTIPLE = 0.0; // RangeRelativeMove threshold multiplier +}; + +// Hand-written (house style): TYPE strictly required — an activity without a +// type is meaningless — while every knob falls back to the struct default so +// payloads only carry the fields their primitive reads. +inline void to_json(nlohmann::json& j, const Activity& a) { + j = nlohmann::json{ + {"TYPE", toString(a.TYPE)}, + {"MOVE_PERCENT", a.MOVE_PERCENT}, + {"WINDOW_SECONDS", a.WINDOW_SECONDS}, + {"LOOKBACK_SECONDS", a.LOOKBACK_SECONDS}, + {"DIRECTION", a.DIRECTION}, + {"ATR_MULTIPLE", a.ATR_MULTIPLE}, + }; +} + +inline void from_json(const nlohmann::json& j, Activity& a) { + a.TYPE = activityTypeFromString(j.at("TYPE").get()); + const Activity defaults{}; + a.MOVE_PERCENT = j.value("MOVE_PERCENT", defaults.MOVE_PERCENT); + a.WINDOW_SECONDS = j.value("WINDOW_SECONDS", defaults.WINDOW_SECONDS); + a.LOOKBACK_SECONDS = j.value("LOOKBACK_SECONDS", defaults.LOOKBACK_SECONDS); + a.DIRECTION = j.value("DIRECTION", defaults.DIRECTION); + a.ATR_MULTIPLE = j.value("ATR_MULTIPLE", defaults.ATR_MULTIPLE); +} + +// One experiment: an ordered chain of activities counted non-overlapping +// against the tick stream. UUID is minted by the sweep factory (the payload +// key and the Elasticsearch _id both build on it); NAME is the sweep's name +// ("dipRecovery") echoed into the results doc for Kibana filtering. +struct ExperimentConfig { + std::string UUID; + std::string NAME; + std::vector CHAIN; +}; + +inline void to_json(nlohmann::json& j, const ExperimentConfig& c) { + j = nlohmann::json{ + {"UUID", c.UUID}, + {"NAME", c.NAME}, + {"CHAIN", c.CHAIN}, + }; +} + +inline void from_json(const nlohmann::json& j, ExperimentConfig& c) { + j.at("UUID").get_to(c.UUID); + j.at("CHAIN").get_to(c.CHAIN); + c.NAME = j.value("NAME", std::string{}); +} + +// Nearest-rank percentiles over an exact (possibly capped — see +// samplesTruncated) sample population. +struct QuantilePair { + double p50 = 0.0; + double p90 = 0.0; +}; + +// Excursion percentiles for one attempt population (completed and failed are +// kept SEPARATE — the failed-attempt MAE is the stop-loss question, the +// completed-attempt MFE is the take-profit question). MFE/MAE are oriented +// by the final leg's implied direction (ExperimentOutcome's +// excursionOrientation); percent variants are relative to each attempt's own +// leg-1 anchor mid. +struct ExcursionStats { + std::uint64_t samples = 0; + QuantilePair mfePoints; + QuantilePair maePoints; + QuantilePair mfePercent; + QuantilePair maePercent; +}; + +// One symbol's slice of an experiment outcome. Per-symbol tick counts and +// spans matter because the UNION-ALL stream's global numbers hide a sparsely +// ticked symbol — its rate would look artificially low against the shared +// denominator. +struct SymbolOutcome { + std::uint64_t occurrences = 0; + std::uint64_t ticksScanned = 0; + double daysSpanned = 0.0; +}; + +// What one experiment's replay over one symbol group's ticks produced — +// returned by chain_matcher::evaluateExperiment and echoed into the results +// document. Lives here (plain header, no JSON — the doc shape belongs to +// experimentResults.hpp) so the analysis worker's textual drain loop can name +// it without importing the chainMatcher module. +struct ExperimentOutcome { + std::uint64_t occurrences = 0; + std::uint64_t ticksScanned = 0; + double daysSpanned = 0.0; // max - min tick timestamp, in days + // Leg-1 completions (anchors) summed across symbols — the denominator + // for P(chain | leg 1); serial single-anchor, so conservative in + // clustered periods (see chainMatcher's module comment). + std::uint64_t attempts = 0; + // Chain deaths attributed to the leg being sought (index 0 stays 0), + // summed across symbols. Sized to the chain length. + std::vector failuresByLeg; + // Completion counts bucketed by the completing tick's UTC calendar slot: + // regime stability ("YYYY-MM" keys — absent months are absent keys, not + // zeros) and session dependence (hour-of-day histogram). + std::map occurrencesByMonth; + std::array occurrencesByHourUtc{}; + // Per-symbol breakdown; the flat map below is the v1 shape, kept so + // existing documents and dashboards keep their field. + std::map perSymbol; + std::map occurrencesBySymbol; + // Magnitude/timing/cost (phase 2). nullopt = empty population (the doc + // then reads null, never fake zeros — the completionRate doctrine). + // Excursions are measured vs each attempt's leg-1 anchor and oriented by + // the FINAL leg's implied direction: "up" = above-anchor counts as MFE, + // "down" = below-anchor does (StaysInBand finals default to "up"). + std::optional completedAttempts; + std::optional failedAttempts; + std::optional completionSeconds; // anchor -> chain done + std::optional meanSpreadAtTriggerPoints; // ask-bid at anchors + std::string excursionOrientation; // "up" | "down" + // Quantiles are exact until kMaxAttemptSamples per matcher, then samples + // stop while COUNTS stay exact — flagged so a truncated percentile is + // never mistaken for a full-population one. + bool samplesTruncated = false; +}; + +} // namespace experiments diff --git a/source/shared/experiments/experimentRunConfiguration.hpp b/source/shared/experiments/experimentRunConfiguration.hpp new file mode 100644 index 0000000..4317fd3 --- /dev/null +++ b/source/shared/experiments/experimentRunConfiguration.hpp @@ -0,0 +1,61 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include + +#include + +// Intentionally a plain header, NOT a .cppm module — see runConfiguration.hpp +// for why JSON-serializable structs stay GMF-includable headers in this repo. +namespace experiments { + +// Run-level descriptor for an experiment run: the unit of QuestDB tick data +// shared by every experiment in a sweep. Carried on +// BACKTESTING_QUEUE_EXPERIMENT_RUN and linked to its experiment payloads via +// RUN_ID (see queueKeys.hpp) — RunConfiguration's exact pattern, minus the +// trading risk knobs an occurrence count has no use for. +// +// The tick window is LAST_MONTHS long and ends OFFSET_MONTHS before now; each +// sweep builder declares its own pair (default 9/0 — see ExperimentSweepSpec). +// BATCH / EXECUTION_TS are the batch identity minted once per `experiments` +// invocation, naming the weekly backtesting-experiments index the results +// land in (same doctrine as RunConfiguration's fields). +struct ExperimentRunConfiguration { + std::string RUN_ID; + std::string SYMBOLS; + std::string BATCH; + std::string EXECUTION_TS; + int LAST_MONTHS = 0; + int OFFSET_MONTHS = 0; +}; + +// Hand-written (rather than NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE) so the core +// fields stay strictly required while the rest fall back to the struct +// defaults — RunConfiguration's serializer pattern. +inline void to_json(nlohmann::json& j, const ExperimentRunConfiguration& c) { + j = nlohmann::json{ + {"RUN_ID", c.RUN_ID}, + {"SYMBOLS", c.SYMBOLS}, + {"BATCH", c.BATCH}, + {"EXECUTION_TS", c.EXECUTION_TS}, + {"LAST_MONTHS", c.LAST_MONTHS}, + {"OFFSET_MONTHS", c.OFFSET_MONTHS}, + }; +} + +inline void from_json(const nlohmann::json& j, ExperimentRunConfiguration& c) { + j.at("RUN_ID").get_to(c.RUN_ID); + j.at("SYMBOLS").get_to(c.SYMBOLS); + j.at("LAST_MONTHS").get_to(c.LAST_MONTHS); + const ExperimentRunConfiguration defaults{}; + c.BATCH = j.value("BATCH", defaults.BATCH); + c.EXECUTION_TS = j.value("EXECUTION_TS", defaults.EXECUTION_TS); + c.OFFSET_MONTHS = j.value("OFFSET_MONTHS", defaults.OFFSET_MONTHS); +} + +} // namespace experiments diff --git a/source/shared/ig/igRestClient.cpp b/source/shared/ig/igRestClient.cpp new file mode 100644 index 0000000..54df18f --- /dev/null +++ b/source/shared/ig/igRestClient.cpp @@ -0,0 +1,175 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "shared/ig/igRestClient.hpp" + +#include +#include +#include +#include + +#include + +#include "shared/utilities/backtestLog.hpp" + +namespace { + +void ensureCurlInit() { + static const struct CurlGlobal { + CurlGlobal() { curl_global_init(CURL_GLOBAL_ALL); } + ~CurlGlobal() { curl_global_cleanup(); } + } guard; + (void)guard; +} + +std::size_t captureResponse(char* ptr, std::size_t size, std::size_t nmemb, + void* userdata) { + auto* body = static_cast(userdata); + body->append(ptr, size * nmemb); + return size * nmemb; +} + +bool caseInsensitiveEquals(const std::string_view a, const std::string_view b) { + if (a.size() != b.size()) { + return false; + } + for (std::size_t i = 0; i < a.size(); ++i) { + if (std::tolower(static_cast(a[i])) + != std::tolower(static_cast(b[i]))) { + return false; + } + } + return true; +} + +// One attempt. nullopt = no HTTP exchange happened (curl init or transport +// failure); a response with any status otherwise. +std::optional attempt( + const ig_rest::Auth& auth, const std::string& path, + const std::string& method, const std::string& jsonBody, + const ig_rest::Headers& extraHeaders) { + ensureCurlInit(); + CURL* curl = curl_easy_init(); + if (!curl) { + backtest_log::error("IGRestClient: curl_easy_init failed"); + return std::nullopt; + } + + // C# concatenates auth.url + path directly (the stored base URL ends + // without a slash and every path starts with one) — same contract here. + const std::string url = auth.url + path; + + struct curl_slist* headers = nullptr; + headers = curl_slist_append(headers, + "Content-Type: application/json; charset=UTF-8"); + headers = curl_slist_append(headers, "Accept: application/json"); + headers = curl_slist_append(headers, + ("X-IG-API-KEY: " + auth.apiKey).c_str()); + headers = curl_slist_append(headers, ("CST: " + auth.cst).c_str()); + headers = curl_slist_append( + headers, ("X-SECURITY-TOKEN: " + auth.xSecurityToken).c_str()); + // A caller-supplied Version wins outright — appending the versionFor + // default as well would put TWO Version headers on the wire. + if (!ig_rest::hasHeader(extraHeaders, "Version")) { + headers = curl_slist_append( + headers, ("Version: " + ig_rest::versionFor(extraHeaders)).c_str()); + } + for (const auto& [name, value] : extraHeaders) { + headers = curl_slist_append(headers, (name + ": " + value).c_str()); + } + + ig_rest::HttpResponse response; + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str()); + // Only attach a body when there is one: the confirms GET has none, and + // POSTFIELDS on a bodiless GET would ship a POST-flavoured request. + if (!jsonBody.empty()) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, jsonBody.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + static_cast(jsonBody.size())); + } + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, captureResponse); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response.body); + + // Bounded I/O on worker threads, same rationale as ElasticPublisher: + // NOSIGNAL because resolver-timeout signals are not thread-safe; 30s + // total matches the C# HttpClient timeout. + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 5L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L); + + curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); + + const CURLcode rc = curl_easy_perform(curl); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response.status); + + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + if (rc != CURLE_OK) { + backtest_log::error(std::string("IGRestClient: ") + method + " " + path + + " failed: " + curl_easy_strerror(rc)); + return std::nullopt; + } + return response; +} + +} // namespace + +namespace ig_rest { + +std::string versionFor(const Headers& extraHeaders) { + for (const auto& [name, value] : extraHeaders) { + if (caseInsensitiveEquals(name, "_method") + && caseInsensitiveEquals(value, "delete")) { + return "1"; + } + } + return "2"; +} + +bool hasHeader(const Headers& headers, const std::string_view name) { + for (const auto& [headerName, value] : headers) { + if (caseInsensitiveEquals(headerName, name)) { + return true; + } + } + return false; +} + +std::optional execute(const Auth& auth, const std::string& path, + const std::string& method, + const std::string& jsonBody, + const Headers& extraHeaders, + const int maxRetries) { + std::optional response; + for (int attemptNo = 0; attemptNo <= maxRetries; ++attemptNo) { + if (attemptNo > 0) { + // 2^attempt seconds — the C# Polly WaitAndRetry schedule (2s, 4s). + const auto backoff = std::chrono::seconds{1LL << attemptNo}; + backtest_log::error("IGRestClient: retry " + std::to_string(attemptNo) + + " for " + method + " " + path + " after " + + std::to_string(backoff.count()) + "s"); + std::this_thread::sleep_for(backoff); + } + response = attempt(auth, path, method, jsonBody, extraHeaders); + if (!response) { + continue; // transport failure — retry + } + const bool transient = response->status >= 500 || response->status == 408; + if (!transient) { + return response; // success or a definitive 4xx — caller decides + } + } + // Retries exhausted: hand back whatever the last attempt produced (a + // transient-status response, or nullopt when it never reached HTTP). + return response; +} + +} // namespace ig_rest diff --git a/source/shared/ig/igRestClient.hpp b/source/shared/ig/igRestClient.hpp new file mode 100644 index 0000000..cc4e05f --- /dev/null +++ b/source/shared/ig/igRestClient.hpp @@ -0,0 +1,69 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include + +// Minimal IG REST transport, mirroring the C# engine's IGMarketRequests +// executor: one bounded, retried HTTPS exchange with the IG session headers +// attached. No broker/business logic lives here — request gating (rate +// limit, duplicate suppression) is the caller's job (see the igRequests +// module) and response interpretation belongs to igMarkets. +// +// Same isolation pattern as elasticPublisher: plain header/.cpp pair (curl +// stays out of module global module fragments), Asio-free header. +namespace ig_rest { + +// One IG session's credentials. Pulled per request from DynamoDB (see +// shared/aws/dynamoAuth), where an external login service keeps the +// expiring CST / X-SECURITY-TOKEN pair refreshed. +struct Auth { + std::string url; // account API base, e.g. https://api.ig.com/gateway/deal + std::string apiKey; // X-IG-API-KEY + std::string cst; // CST session header + std::string xSecurityToken; // X-SECURITY-TOKEN session header +}; + +using Headers = std::vector>; + +struct HttpResponse { + long status{}; + std::string body; +}; + +// IG's per-endpoint Version header: "1" when the caller is tunnelling a +// DELETE through POST (an extra "_method: DELETE" header — IG's close- +// position API), otherwise "2". Free function so tests can pin the rule. +std::string versionFor(const Headers& extraHeaders); + +// True when `headers` already carries `name` (case-insensitive). execute() +// uses this to let a caller-supplied Version header (e.g. the confirms +// endpoint, which is Version 1 without being a tunnelled DELETE) REPLACE +// the versionFor default instead of duplicating it on the wire. +bool hasHeader(const Headers& headers, std::string_view name); + +// One exchange with retries: transient outcomes (transport error, HTTP 5xx +// or 408) are retried maxRetries times with exponential backoff (2s, 4s — +// the C# Polly policy at the default RETRY_COUNT=2), each attempt bounded by +// a 30s timeout. Returns the final response — INCLUDING a non-2xx one (the +// caller interprets status) — or nullopt when the final attempt still had no +// HTTP exchange at all. maxRetries = 0 sends exactly one attempt: required +// for the NON-IDEMPOTENT open POST, where a blind re-send after a lost ACK +// could double a live position (the caller resolves ambiguity through the +// confirms endpoint instead — see igRequests makeOpen). `extraHeaders` are +// sent verbatim (this is how "_method: DELETE" reaches IG) on top of the +// session headers, Version, Accept and Content-Type. +std::optional execute(const Auth& auth, const std::string& path, + const std::string& method, + const std::string& jsonBody, + const Headers& extraHeaders = {}, + int maxRetries = 2); + +} // namespace ig_rest diff --git a/source/shared/ipc/engineControl.hpp b/source/shared/ipc/engineControl.hpp index 600ce8b..59244bc 100644 --- a/source/shared/ipc/engineControl.hpp +++ b/source/shared/ipc/engineControl.hpp @@ -15,7 +15,8 @@ // monitor (source/backtesting-controller). The engine broadcasts how many // backtests are in flight; the monitor can flip a stop flag to ask for a graceful // drain-and-pause. Zero network, zero serialisation — both sides map the same -// 8-byte block. +// 12-byte block (see EngineState below: magic + active_jobs + stop_signal; the +// Python side must map all 12 bytes and read the fields at offsets 4 and 8). // // The block is a memory-mapped FILE at a fixed path (DEFAULT_PATH below), mapped // via Boost.Interprocess `file_mapping`. We use a file rather than @@ -55,7 +56,8 @@ static_assert(sizeof(EngineState) == 12, "EngineState must be a packed 12-byte b static_assert(alignof(EngineState) == 4, "EngineState fields must be 4-byte aligned"); // The default backing file. The Python controller defaults to the same path. -inline constexpr const char* DEFAULT_PATH = "/tmp/EngineControlShm"; +// Safe to use /tmp: Environment is a private, single-use machine +inline constexpr auto DEFAULT_PATH = "/tmp/EngineControlShm"; // NOSONAR // RAII owner of the mapped file. Construction creates and zeroes the file // (replacing any stale one left by a crashed run); destruction unmaps and removes diff --git a/source/ingest/tickPacket.cppm b/source/shared/net/tickPacket.cppm similarity index 96% rename from source/ingest/tickPacket.cppm rename to source/shared/net/tickPacket.cppm index 0b33941..59b2729 100644 --- a/source/ingest/tickPacket.cppm +++ b/source/shared/net/tickPacket.cppm @@ -24,7 +24,8 @@ // decodeTick turns those bytes into the engine's existing PriceData: it scales // the real bid/ask into the stored INT32 "points" via the symbol's price // multiplier (symbol_scale::getPriceScale) so the ingest writes exactly what the -// backtester's read path already expects. +// backtester's read path already expects. Shared by the ingest and live +// subcommands — both consume the same wire format, on different ports. export module tickPacket; @@ -33,7 +34,7 @@ import std; // , , , , , , import priceData; // PriceData import symbolScale; // symbol_scale::getPriceScale, kUnknown -export namespace ingest { +export namespace tick_packet { // Fixed packet geometry (see the header comment for the byte map). inline constexpr std::size_t kBidOffset = 0; @@ -140,4 +141,4 @@ template return PriceData(*askScaled, *bidScaled, timestamp, symbol); } -} // namespace ingest +} // namespace tick_packet diff --git a/source/shared/net/udpPorts.hpp b/source/shared/net/udpPorts.hpp new file mode 100644 index 0000000..431e8e7 --- /dev/null +++ b/source/shared/net/udpPorts.hpp @@ -0,0 +1,24 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include + +// The C# streamer (vortex/shared/UDPPorts.cs) fans its serialized feeds out to +// several UDP sinks. This engine consumes three of them: the persistence +// stream (ingest -> QuestDB) and the live stream (live -> strategy execution), +// both carrying 40-byte tick packets, plus the deal stream (tracking -> +// position updates) carrying 256-byte deal packets. Keep these in lockstep +// with UDPPorts.cs. Each bind port is also overridable at runtime (CLI arg / +// $INGEST_UDP_PORT / $LIVE_UDP_PORT / $TRACKING_UDP_PORT). +namespace udp_ports { + +inline constexpr std::uint16_t kSave = 11111; // == UDPPorts.PortSave +inline constexpr std::uint16_t kLive = 11110; // == UDPPorts.PortLive +inline constexpr std::uint16_t kTrade = 11112; // == UDPPorts.PortTrade + +} // namespace udp_ports diff --git a/source/ingest/udpReceiver.cpp b/source/shared/net/udpReceiver.cpp similarity index 89% rename from source/ingest/udpReceiver.cpp rename to source/shared/net/udpReceiver.cpp index 1664d7b..47e7a85 100644 --- a/source/ingest/udpReceiver.cpp +++ b/source/shared/net/udpReceiver.cpp @@ -9,7 +9,7 @@ // never reach the module boundary — mirroring how boostRedisImpl.cpp / // redisConnection.cpp isolate Boost.Asio from the rest of the build. -#include "ingest/udpReceiver.hpp" +#include "shared/net/udpReceiver.hpp" #include #include @@ -21,7 +21,7 @@ #include "shared/utilities/backtestLog.hpp" -namespace ingest { +namespace net { namespace asio = boost::asio; using asio::ip::udp; @@ -79,9 +79,16 @@ struct UdpReceiver::Impl { try { handler(std::span(buffer.data(), n)); } catch (const std::exception& e) { + // Suppress the live-logs ship: a handler throwing per + // datagram (bad_alloc under pressure) would ship one + // document per packet from the receive thread — + // stderr keeps the line; the minutely stats trace + // surfaces the outage off-box. + const backtest_log::SinkSuppression suppression; backtest_log::error(std::string("UdpReceiver: handler threw: ") + e.what()); } catch (...) { + const backtest_log::SinkSuppression suppression; backtest_log::error("UdpReceiver: handler threw non-std exception"); } } else if (errorCodes == asio::error::operation_aborted) { @@ -128,4 +135,4 @@ void UdpReceiver::stop() asio::post(impl_->ioc, [this] { impl_->shutdown(); }); } -} // namespace ingest +} // namespace net diff --git a/source/ingest/udpReceiver.hpp b/source/shared/net/udpReceiver.hpp similarity index 75% rename from source/ingest/udpReceiver.hpp rename to source/shared/net/udpReceiver.hpp index 41eaeda..4e9ec70 100644 --- a/source/ingest/udpReceiver.hpp +++ b/source/shared/net/udpReceiver.hpp @@ -13,12 +13,12 @@ #include #include -// A minimal UDP datagram receiver. The Boost.Asio implementation is hidden -// behind a pimpl so this header stays free of Asio's heavy templates — that -// keeps it safe to #include from a C++23 module's global module fragment -// (ingestCommand.cppm), the same isolation the repo uses for Boost.Redis -// (boostRedisImpl.cpp) and the Redis connection (redisConnection.hpp). -namespace ingest { +// A minimal UDP datagram receiver, shared by the ingest and live subcommands. +// The Boost.Asio implementation is hidden behind a pimpl so this header stays +// free of Asio's heavy templates — that keeps it safe to #include from a C++23 +// module's global module fragment, the same isolation the repo uses for +// Boost.Redis (boostRedisImpl.cpp) and the Redis connection (redisConnection.hpp). +namespace net { class UdpReceiver { public: @@ -48,4 +48,4 @@ class UdpReceiver { std::unique_ptr impl_; }; -} // namespace ingest +} // namespace net diff --git a/source/shared/questdb/connectionFactory.cppm b/source/shared/questdb/connectionFactory.cppm new file mode 100644 index 0000000..e4290b5 --- /dev/null +++ b/source/shared/questdb/connectionFactory.cppm @@ -0,0 +1,52 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// Single place that turns the process environment into a pg-wire +// DatabaseConnection: $QUESTDB_HOST (default 127.0.0.1), $QUESTDB_PORT +// (default 8812), and the stock QuestDB credentials (qdb/admin/quest). +// Callers that receive the host some other way (the run command takes it via +// argv) pass it as `hostOverride`; the port still comes from the environment +// so a non-default instance can be targeted without a rebuild. + +module; + +#include "shared/utilities/env.hpp" + +export module connectionFactory; + +import std; // replaces , , , , +import databaseConnection; // DatabaseConnection + +export namespace questdb { + +DatabaseConnection connectionFromEnv(std::string_view hostOverride = {}); + +} // namespace questdb + +namespace questdb { + +DatabaseConnection connectionFromEnv(std::string_view hostOverride) { + const std::string host = hostOverride.empty() + ? env::getOr("QUESTDB_HOST", "127.0.0.1") + : std::string(hostOverride); + + // from_chars over the full string rather than std::stoi: a misconfigured + // $QUESTDB_PORT fails with the offending value in the message instead of + // stoi's bare "invalid_argument", and trailing junk ("8812x") is rejected + // rather than silently truncated. + const std::string portText = env::getOr("QUESTDB_PORT", "8812"); + int port = 0; + const auto [ptr, ec] = + std::from_chars(portText.data(), portText.data() + portText.size(), port); + if (ec != std::errc{} || ptr != portText.data() + portText.size()) { + throw std::runtime_error( + std::format("QUESTDB_PORT is not a valid port: '{}'", portText)); + } + + return DatabaseConnection(host, port, "qdb", "admin", "quest"); +} + +} // namespace questdb diff --git a/source/shared/questdb/databaseConnection.cppm b/source/shared/questdb/databaseConnection.cppm index 33b4239..7ae352d 100644 --- a/source/shared/questdb/databaseConnection.cppm +++ b/source/shared/questdb/databaseConnection.cppm @@ -12,9 +12,10 @@ module; export module databaseConnection; -import std; // replaces , , , , , - // , , , -import priceData; // PriceData +import std; // replaces , , , , , + // , , , +import ohlcObject; // OhlcObject (queryOhlc result rows) +import priceData; // PriceData export class DatabaseConnection { private: @@ -29,13 +30,22 @@ public: std::vector executeQuery(const std::string& query) const; + std::vector queryOhlc(const std::string& query) const; + + // Runs a query expected to return exactly ONE row whose columns are all + // timestamps (e.g. the month-boundary row) and parses each column with + // fastParseTimestamp. Throws std::runtime_error on any other row count so a + // malformed boundary query can never be misread as data. + std::vector queryTimestampRow( + const std::string& query) const; + const std::string& getConnectionString() const { return connection_string; } }; -class InvalidTimestampFormatError : public std::runtime_error { +export class InvalidTimestampFormatError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; @@ -43,25 +53,52 @@ public: // Caches timegm per date — tick data is time-ordered so the date changes // rarely. The caller owns the cache, so each thread/query gets its own and // the parse stays safe if loading ever moves onto the ThreadPool. -struct DateCache { +// Exported (with the parser below) so the fractional-seconds handling is +// directly unit-testable. +export struct DateCache { std::string date; std::time_t epoch = 0; }; -static std::chrono::system_clock::time_point fastParseTimestamp(const char* ts, DateCache& cache) { +export std::chrono::system_clock::time_point fastParseTimestamp(const char* ts, DateCache& cache) { int year = 0; int month = 0; int day = 0; int hour = 0; int min = 0; int sec = 0; - int usec = 0; + int consumed = 0; + // %*1[T ] accepts either separator between date and time: pgwire text + // format uses a space, ISO-8601 literals (SqlManager::formatTimestamp) + // use 'T'. Assignment-suppressed, so parsedFields still counts 6. const int parsedFields = - std::sscanf(ts, "%4d-%2d-%2d %2d:%2d:%2d.%d", &year, &month, &day, &hour, &min, &sec, &usec); - if (parsedFields != 6 && parsedFields != 7) { + std::sscanf(ts, "%4d-%2d-%2d%*1[T ]%2d:%2d:%2d%n", &year, &month, &day, &hour, &min, &sec, &consumed); + if (parsedFields != 6) { throw InvalidTimestampFormatError("Invalid timestamp format: " + std::string(ts)); } + // Fractional seconds are scaled by the number of digits actually present: + // Postgres wire-format text trims trailing zeros, so ".5" means 500000 µs — + // reading the digits as a plain integer (the old "%d" parse) would have + // turned it into 5 µs, a 100000x error that corrupts tick ordering at bar + // boundaries. Digits beyond microsecond precision are truncated. + std::int64_t usec = 0; + if (ts[consumed] == '.') { + const char* p = ts + consumed + 1; + int digits = 0; + while (digits < 6 && p[digits] >= '0' && p[digits] <= '9') { + usec = usec * 10 + (p[digits] - '0'); + ++digits; + } + if (digits == 0) { + throw InvalidTimestampFormatError( + "Invalid fractional seconds in timestamp: " + std::string(ts)); + } + for (; digits < 6; ++digits) { + usec *= 10; + } + } + const std::string_view date(ts, 10); if (cache.date != date) { cache.date.assign(date); @@ -115,3 +152,55 @@ std::vector DatabaseConnection::executeQuery(const std::string& query return results; } + +std::vector DatabaseConnection::queryTimestampRow( + const std::string& query) const { + pqxx::connection conn(this->connection_string); + pqxx::nontransaction txn(conn); + pqxx::result result = txn.exec(query); + + if (result.size() != 1) { + throw std::runtime_error(std::format( + "Timestamp-row query returned {} rows, expected exactly 1", result.size())); + } + + const auto& row = result[0]; + std::vector timestamps(row.size()); + DateCache dateCache; + for (pqxx::row::size_type i = 0; i < row.size(); ++i) { + timestamps[i] = fastParseTimestamp(row[i].c_str(), dateCache); + } + return timestamps; +} + +// Maps rows of (timestamp, open, high, low, close) — the shape produced by a +// SAMPLE BY aggregation over a tick table — onto OhlcObject. Prices carry the +// same scaled INT32 fixed-point as the underlying ask/bid columns. Bars come +// back in delivered order, marked complete; ordering is the caller's business. +std::vector DatabaseConnection::queryOhlc(const std::string& query) const { + pqxx::connection conn(this->connection_string); + pqxx::nontransaction txn(conn); + pqxx::result result = txn.exec(query); + + std::vector bars(result.size()); + DateCache dateCache; + + for (std::size_t i = 0; i < result.size(); ++i) { + const auto& row = result[static_cast(i)]; + OhlcObject& bar = bars[i]; + bar.date = fastParseTimestamp(row[0].c_str(), dateCache); + int col = 1; + for (std::int32_t* field : {&bar.open, &bar.high, &bar.low, &bar.close}) { + const auto sv = row[col].view(); + const auto parsed = std::from_chars(sv.data(), sv.data() + sv.size(), *field); + if (parsed.ec != std::errc{}) { + throw std::runtime_error(std::format( + "Failed to parse OHLC column {}: '{}'", col, sv)); + } + ++col; + } + bar.complete = true; + } + + return bars; +} diff --git a/source/shared/questdb/sqlManager.cppm b/source/shared/questdb/sqlManager.cppm index 79509ed..5fe5786 100644 --- a/source/shared/questdb/sqlManager.cppm +++ b/source/shared/questdb/sqlManager.cppm @@ -13,33 +13,160 @@ import symbolScale; // symbol_scale::get / kUnknown export class SqlManager { public: - static std::vector loadPriceData(const DatabaseConnection& db, const std::vector& symbols, int LAST_MONTHS = 1); -}; + // The window is LAST_MONTHS long and ends OFFSET_MONTHS before now, so + // OFFSET_MONTHS = 0 loads the most recent data and e.g. (6, 12) loads the + // 6-month window that ended a year ago. + static std::vector loadPriceData(const DatabaseConnection& db, const std::vector& symbols, int LAST_MONTHS = 1, int OFFSET_MONTHS = 0); -std::vector SqlManager::loadPriceData(const DatabaseConnection& db, const std::vector& symbols, int LAST_MONTHS) { - if (symbols.empty()) { - return {}; - } + // Month boundaries for a tick superset, computed BY QUESTDB so they carry + // its exact dateadd('M', ...) semantics (month-end day clamping included) — + // never reimplemented locally. One statement evaluates now() once, so every + // column shares a single snapshot instant T: index 0 is T itself and index + // m is m calendar months before T. loadMonthBoundaries returns exactly + // months+1 instants. + static std::string buildMonthBoundariesQuery(int months); + static std::vector loadMonthBoundaries( + const DatabaseConnection& db, int months); + + // Tick load over an explicit [lower, upper) window, passed as literal + // timestamps rather than now()-relative dateadd expressions — so a superset + // query and the boundary row it was sliced from can never disagree about + // where a month starts. + static std::string buildPriceDataBetweenQuery( + const std::vector& symbols, + std::chrono::system_clock::time_point lower, + std::chrono::system_clock::time_point upper); + static std::vector loadPriceDataBetween( + const DatabaseConnection& db, + const std::vector& symbols, + std::chrono::system_clock::time_point lower, + std::chrono::system_clock::time_point upper); - // Symbols arrive via Redis payloads and are interpolated into the query - // below, so only accept names from the canonical table. This blocks SQL + // ISO-8601 with full microseconds ("2026-07-07T14:03:12.123456Z"): valid as + // a QuestDB timestamp literal, and round-trips exactly through + // fastParseTimestamp so boundary instants survive format -> parse unchanged. + static std::string formatTimestamp(std::chrono::system_clock::time_point tp); + +private: + // Symbols arrive via Redis payloads and are interpolated into query text, + // so only names from the canonical table are accepted. This blocks SQL // injection and catches typos before they become QuestDB errors. + static void validateSymbols(const std::vector& symbols); + + // Shared SELECT ... UNION ALL ... shape of both tick loads; the caller + // supplies the per-symbol WHERE clause. Ends with ORDER BY timestamp, plus + // a `, symbol` tie-break only for multi-symbol queries: equal-timestamp + // ticks across symbols otherwise come back in whatever order QuestDB + // merges them, while the tie-break on a single-symbol query would defeat + // the elided sort on the designated timestamp for no gain. + static std::string buildTickQuery(const std::vector& symbols, + const std::string& whereClause); +}; + +void SqlManager::validateSymbols(const std::vector& symbols) { for (const auto& symbol : symbols) { if (symbol_scale::get(symbol) == symbol_scale::kUnknown) { throw std::invalid_argument("Unknown symbol rejected: " + symbol); } } +} +std::string SqlManager::buildTickQuery(const std::vector& symbols, + const std::string& whereClause) { + // Columns are named explicitly (never `*`): executeQuery reads results + // positionally as (symbol, ask, bid, timestamp), so a tick table whose + // physical column order differs — or ever gains a column — must not be + // able to silently shift prices into the wrong fields. std::ostringstream query; for (std::size_t i = 0; i < symbols.size(); ++i) { if (i > 0) { query << " UNION ALL "; } - query << "SELECT '" << symbols[i] << "' as symbol, * FROM '" << symbols[i] - << "' WHERE timestamp >= dateadd('M', -" << LAST_MONTHS << ", now())"; + query << "SELECT '" << symbols[i] << "' as symbol, ask, bid, timestamp FROM '" + << symbols[i] << "' WHERE " << whereClause; } query << " ORDER BY timestamp"; + if (symbols.size() > 1) { + query << ", symbol"; + } + return query.str(); +} + +std::vector SqlManager::loadPriceData(const DatabaseConnection& db, const std::vector& symbols, int LAST_MONTHS, int OFFSET_MONTHS) { + if (symbols.empty()) { + return {}; + } + + validateSymbols(symbols); + + std::ostringstream where; + where << "timestamp >= dateadd('M', -" << (LAST_MONTHS + OFFSET_MONTHS) << ", now())"; + // Only cap the window when it is actually shifted back — with a zero + // offset the upper bound would just be now(), so leave it open and + // keep the default query identical to what it always was. + if (OFFSET_MONTHS > 0) { + where << " AND timestamp < dateadd('M', -" << OFFSET_MONTHS << ", now())"; + } + const std::string query = buildTickQuery(symbols, where.str()); + + std::cout << "Executing query: " << query << std::endl; + return db.executeQuery(query); +} + +std::string SqlManager::formatTimestamp(const std::chrono::system_clock::time_point tp) { + // %S on a microseconds-precision time prints the full fraction + // ("12.123456"), which fastParseTimestamp reads back to the same + // microsecond — the round-trip the boundary handling relies on. + return std::format("{:%Y-%m-%dT%H:%M:%S}Z", + std::chrono::floor(tp)); +} + +std::string SqlManager::buildMonthBoundariesQuery(const int months) { + if (months < 1) { + throw std::invalid_argument("Month boundaries need at least one month back"); + } + std::ostringstream query; + query << "SELECT now()"; + for (int m = 1; m <= months; ++m) { + query << ", dateadd('M', -" << m << ", now())"; + } + return query.str(); +} + +std::vector SqlManager::loadMonthBoundaries( + const DatabaseConnection& db, const int months) { + auto boundaries = db.queryTimestampRow(buildMonthBoundariesQuery(months)); + if (boundaries.size() != static_cast(months) + 1) { + throw std::runtime_error("Month boundary row returned " + + std::to_string(boundaries.size()) + " columns, expected " + + std::to_string(months + 1)); + } + return boundaries; +} + +std::string SqlManager::buildPriceDataBetweenQuery( + const std::vector& symbols, + const std::chrono::system_clock::time_point lower, + const std::chrono::system_clock::time_point upper) { + std::ostringstream where; + where << "timestamp >= cast('" << formatTimestamp(lower) + << "' as timestamp) AND timestamp < cast('" << formatTimestamp(upper) + << "' as timestamp)"; + return buildTickQuery(symbols, where.str()); +} + +std::vector SqlManager::loadPriceDataBetween( + const DatabaseConnection& db, + const std::vector& symbols, + const std::chrono::system_clock::time_point lower, + const std::chrono::system_clock::time_point upper) { + if (symbols.empty()) { + return {}; + } + + validateSymbols(symbols); - std::cout << "Executing query: " << query.str() << std::endl; - return db.executeQuery(query.str()); + const std::string query = buildPriceDataBetweenQuery(symbols, lower, upper); + std::cout << "Executing query: " << query << std::endl; + return db.executeQuery(query); } diff --git a/source/shared/redis/apiRequestGate.cpp b/source/shared/redis/apiRequestGate.cpp new file mode 100644 index 0000000..f55f69c --- /dev/null +++ b/source/shared/redis/apiRequestGate.cpp @@ -0,0 +1,134 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "shared/redis/apiRequestGate.hpp" + +#include +#include +#include + +#include "shared/redis/client/syncRedisClient.hpp" +#include "shared/utilities/backtestLog.hpp" + +namespace redis_api { + +std::string requestWindowKey(const int minuteOfHour) { + return "REQ#" + std::to_string(minuteOfHour); +} + +std::string dealRequestKey(const std::string& dealId) { + return "API#" + dealId; +} + +ApiRequestGate::ApiRequestGate(const std::string& host, const int port) + : client_(std::make_unique(host, port)) {} + +ApiRequestGate::~ApiRequestGate() = default; + +std::optional ApiRequestGate::requestsMade(const int minuteOfHour) { + const std::lock_guard guard{mutex_}; + std::string error; + const auto value = client_->run( + client_->operations().getString(requestWindowKey(minuteOfHour)), + error); + if (!value) { + // Distinguish "reached Redis, key missing" (a fresh minute — zero + // requests) from "could not reach Redis" (unknown; callers fail + // closed). run() yields nullopt only on failure; a missing key is a + // present optional holding nullopt. + if (!error.empty()) { + backtest_log::error("ApiRequestGate: reading " + + requestWindowKey(minuteOfHour) + " failed (" + + error + ")"); + } + return std::nullopt; + } + if (!value->has_value()) { + return 0; + } + int count = 0; + const std::string& text = **value; + const auto [ptr, ec] = + std::from_chars(text.data(), text.data() + text.size(), count); + if (ec != std::errc{} || count < 0) { + return 0; // garbage in the window key counts as an empty window + } + return count; +} + +bool ApiRequestGate::recordRequest(const int minuteOfHour) { + const std::optional current = requestsMade(minuteOfHour); + if (!current) { + return false; + } + const std::lock_guard guard{mutex_}; + std::string error; + // Unconditional SET with a fresh 1-minute TTL — the C# AddRequest. The + // TTL restarting on every write is fine: the key names its own minute, + // so it only needs to outlive that minute, and one extra minute of + // stale count in a key nobody reads any more is harmless. + const auto stored = client_->run( + client_->operations().setString(requestWindowKey(minuteOfHour), + std::to_string(*current + 1), + SetWhen::Always, + std::chrono::minutes{1}), + error); + if (!stored || !*stored) { + if (!error.empty()) { + backtest_log::error("ApiRequestGate: recording request in " + + requestWindowKey(minuteOfHour) + " failed (" + + error + ")"); + } + return false; + } + return true; +} + +std::optional ApiRequestGate::isDuplicateDeal(const std::string& dealId) { + if (dealId.empty()) { + return false; + } + const std::lock_guard guard{mutex_}; + std::string error; + const auto value = client_->run( + client_->operations().getString(dealRequestKey(dealId)), error); + if (!value) { + // UNKNOWN: Redis unreachable. Report it as such — the caller's + // policy decides (opens fail closed, risk-reducing closes fail open). + if (!error.empty()) { + backtest_log::error("ApiRequestGate: duplicate check for " + + dealRequestKey(dealId) + " failed (" + error + + "); reporting unknown"); + } + return std::nullopt; + } + return value->has_value(); +} + +bool ApiRequestGate::recordDealRequest(const std::string& dealId, + const std::chrono::seconds ttl) { + if (dealId.empty()) { + return true; + } + const std::lock_guard guard{mutex_}; + std::string error; + const auto stored = client_->run( + client_->operations().setString( + dealRequestKey(dealId), "1", SetWhen::Always, + std::chrono::duration_cast(ttl)), + error); + if (!stored || !*stored) { + if (!error.empty()) { + backtest_log::error("ApiRequestGate: recording " + + dealRequestKey(dealId) + " failed (" + error + + ")"); + } + return false; + } + return true; +} + +} // namespace redis_api diff --git a/source/shared/redis/apiRequestGate.hpp b/source/shared/redis/apiRequestGate.hpp new file mode 100644 index 0000000..b9b39f5 --- /dev/null +++ b/source/shared/redis/apiRequestGate.hpp @@ -0,0 +1,84 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include +#include + +namespace redis_util { +class SyncRedisClient; +} + +// Redis-backed broker API request bookkeeping, mirroring the C# engine's +// IGMarketRequests counters. Two key families, shared with the C# engine so +// both processes draw on ONE request budget against the broker's rate limit: +// +// REQ# -> requests made in that wall-clock minute (0-59), value a +// plain integer string, 1-minute TTL. The window budget. +// API# -> short-lived marker (30s) that a request for this deal +// key was just made; a second request while the marker +// lives is a duplicate and must not be sent (prevents +// accidental flooding of the same strategy+direction). +// +// Same isolation pattern as tradeLocks.hpp: Asio-free header, the connection +// machinery lives behind apiRequestGate.cpp, so module global module +// fragments can #include this alongside `import std`. +namespace redis_api { + +// Key builders — free functions so tests can pin the wire formats without a +// Redis server (drift would silently split the request budget between the +// two engines). +std::string requestWindowKey(int minuteOfHour); // "REQ#" +std::string dealRequestKey(const std::string& dealId); // "API#" + +class ApiRequestGate { +public: + // Connects lazily on first use via the shared Boost.Redis connection. + ApiRequestGate(const std::string& host, int port); + ~ApiRequestGate(); + ApiRequestGate(const ApiRequestGate&) = delete; + ApiRequestGate& operator=(const ApiRequestGate&) = delete; + + // GET REQ#, parsed as an integer (0 when the key is missing or + // holds garbage — matching the C# int.Parse-of-null -> 0 behaviour via + // its null guard). nullopt means UNKNOWN (Redis unreachable): callers + // gating outbound broker requests should fail closed and not send. + std::optional requestsMade(int minuteOfHour); + + // SET REQ# = current+1, PX 1 minute (the C# read-modify-write — + // benign raciness accepted there and here; the budget is a soft brake, + // the broker enforces the hard one). Returns false when Redis was + // unreachable. + bool recordRequest(int minuteOfHour); + + // GET API#: true when a request for this deal key is already in + // its suppression window, false when it is not, and nullopt when Redis + // was unreachable (UNKNOWN). The fail-closed/fail-open decision belongs + // to the caller: opens must treat unknown as duplicate (an unverifiable + // duplicate order is the exact accident this marker prevents), while a + // risk-reducing close must still go out (see igRequests' gatePolicy). + // Empty dealId -> false (nothing to deduplicate, C# behaviour). + std::optional isDuplicateDeal(const std::string& dealId); + + // SET API# PX(ttl). Empty dealId is a no-op (the C# random-key + // fallback recorded nothing useful — a key nobody will ever look up). + // Returns false when Redis was unreachable. + bool recordDealRequest(const std::string& dealId, + std::chrono::seconds ttl = std::chrono::seconds{30}); + +private: + std::unique_ptr client_; + // Serialises concurrent callers sharing one instance (one connection, + // one synchronous pump) — same pattern as TradeLocks; prefer one + // instance per thread (see IGMarketRequests' thread_local binder). + std::mutex mutex_; +}; + +} // namespace redis_api diff --git a/source/shared/redis/client/syncRedisClient.cpp b/source/shared/redis/client/syncRedisClient.cpp new file mode 100644 index 0000000..008c5e6 --- /dev/null +++ b/source/shared/redis/client/syncRedisClient.cpp @@ -0,0 +1,83 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "shared/redis/client/syncRedisClient.hpp" + +#include + +#include "shared/redis/connection/redisConnection.hpp" + +namespace redis_util { + +SyncRedisClient::SyncRedisClient(const std::string& host, const int port, + const std::chrono::seconds opTimeout, + const std::chrono::seconds failFastCooldown) + : opTimeout_(opTimeout), + failFastCooldown_(failFastCooldown), + conn_(makeRedisConnection(ioc_, host, port)), + ops_(conn_) {} + +SyncRedisClient::~SyncRedisClient() { + // Cancel the connection's detached async_run, then drain the context so + // it winds down cleanly before the io_context is destroyed. + conn_->cancel(); + ioc_.restart(); + ioc_.run(); +} + +bool SyncRedisClient::breakerOpen() const { + return std::chrono::steady_clock::now() < failFastUntil_; +} + +void SyncRedisClient::closeBreaker() { failFastUntil_ = {}; } + +bool SyncRedisClient::pumpUntil(const bool& done, std::string& error) { + const auto deadline = std::chrono::steady_clock::now() + opTimeout_; + ioc_.restart(); + while (!done) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + tripBreaker("timed out (Redis unreachable?)", error); + return false; + } + if (ioc_.run_one_for(deadline - now) == 0 && !done) { + // 0 means the wait timed out OR the context ran out of work + // (async_run died); disambiguate by the clock. + tripBreaker( + std::chrono::steady_clock::now() >= deadline + ? "timed out (Redis unreachable?)" + : "connection stopped before the command completed", + error); + return false; + } + } + return true; +} + +void SyncRedisClient::noteFailure(const std::exception_ptr failure, + std::string& error) { + try { + std::rethrow_exception(failure); + } catch (const std::exception& e) { + tripBreaker(e.what(), error); + } catch (...) { + tripBreaker("unknown error", error); + } +} + +void SyncRedisClient::tripBreaker(std::string reason, std::string& error) { + // A request issued while disconnected stays queued inside Boost.Redis + // (cancel_if_not_connected is off by default) and would execute when the + // connection comes back — for a SET NX gate that would mint a phantom + // lock for an order that was never placed, blocking re-entry for a full + // TTL after an outage. Cancel whatever is still pending so a failed call + // leaves nothing behind to fire late. Harmless when nothing is queued. + conn_->cancel(boost::redis::operation::exec); + failFastUntil_ = std::chrono::steady_clock::now() + failFastCooldown_; + error = std::move(reason); +} + +} // namespace redis_util diff --git a/source/shared/redis/client/syncRedisClient.hpp b/source/shared/redis/client/syncRedisClient.hpp new file mode 100644 index 0000000..222216c --- /dev/null +++ b/source/shared/redis/client/syncRedisClient.hpp @@ -0,0 +1,132 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "shared/redis/operations/redisOperations.hpp" + +// Synchronous, deadline-bounded facade over one Boost.Redis connection: an +// io_context the calling thread pumps, a connection kept alive by a detached +// async_run, the RedisOperations command layer, and a fail-fast circuit +// breaker. Grown out of the live trade-lock gate and shared here so every +// synchronous Redis consumer (the phase-2 broker position reader next) gets +// the same semantics instead of re-implementing the pump: +// +// - DEADLINE-BOUNDED: with Redis down, async_run sits in a reconnect loop +// and async_exec waits indefinitely for a connection — unbounded, that +// would wedge the calling thread instead of letting it fail. run() +// abandons the operation at the deadline. +// - FAIL-FAST BREAKER: after any failure, calls within the cooldown window +// return nullopt immediately (with an EMPTY error, so callers don't log +// a line per skipped probe) rather than paying the timeout again. +// - CANCEL ON FAILURE: an abandoned request stays queued inside Boost.Redis +// and would still execute when the connection comes back; every breaker +// trip cancels pending requests so nothing fires late (see tripBreaker). +// +// NOT thread-safe: one instance serves one caller at a time. Callers sharing +// an instance across threads must bring their own mutex (see TradeLocks); +// better, give each thread its own instance (see RedisTradeGate). +// +// This header pulls in Asio and Boost.Redis, so unlike tradeLocks.hpp it must +// stay OUT of module global module fragments — include it from plain .cpp +// implementation files only. +namespace redis_util { + +class SyncRedisClient { +public: + // Defaults: a 2s op deadline is generous for a localhost round trip but + // short enough that a caller blocked on a dead Redis fails quickly; a 3s + // cooldown stops a caller that retries at tick rate from paying the full + // deadline serially while Redis is down. + SyncRedisClient( + const std::string& host, int port, + std::chrono::seconds opTimeout = std::chrono::seconds{2}, + std::chrono::seconds failFastCooldown = std::chrono::seconds{3}); + ~SyncRedisClient(); // cancels the detached async_run and drains the context + + SyncRedisClient(const SyncRedisClient&) = delete; + SyncRedisClient& operator=(const SyncRedisClient&) = delete; + + // Command builders for run(), e.g. + // client.run(client.operations().setIfNotExists(key, value, ttl), error) + // The RedisOperations object is a member on purpose: coroutine frames + // capture its `this`, and an abandoned (timed-out) frame resumes during a + // LATER call's pump — the object it points back into must still be alive. + RedisOperations& operations() { return ops_; } + + // Runs one awaitable Redis operation to completion or deadline. Outcomes: + // the operation's own result, or nullopt when the breaker is open (empty + // `error`), the op timed out, the connection died, or the coroutine threw + // (reason in `error`). On timeout the coroutine is abandoned, so its + // completion state lives on the heap (shared_ptr) — a stale completion + // firing during a later call's pump writes into surviving storage, never + // a dead stack frame. + template + std::optional run(boost::asio::awaitable op, std::string& error) { + static_assert(!std::is_void_v, + "run() needs a value-returning operation; wrap a void " + "op in a coroutine that co_returns a flag"); + if (breakerOpen()) { + error.clear(); + return std::nullopt; + } + struct OpState { + bool done = false; + std::optional result; + std::exception_ptr failure; + }; + auto state = std::make_shared(); + boost::asio::co_spawn(ioc_, std::move(op), + [state](const std::exception_ptr e, T value) { + state->failure = e; + state->result = std::move(value); + state->done = true; + }); + if (!pumpUntil(state->done, error)) { + return std::nullopt; + } + if (state->failure) { + noteFailure(state->failure, error); + return std::nullopt; + } + closeBreaker(); + return std::move(state->result); + } + +private: + [[nodiscard]] bool breakerOpen() const; + void closeBreaker(); + + // Pumps the io_context until `done` flips or the deadline passes; trips + // the breaker on timeout or a dead connection and returns false. + bool pumpUntil(const bool& done, std::string& error); + + // Decodes a completed operation's exception into `error` and trips. + void noteFailure(std::exception_ptr failure, std::string& error); + + void tripBreaker(std::string reason, std::string& error); + + std::chrono::seconds opTimeout_; + std::chrono::seconds failFastCooldown_; + boost::asio::io_context ioc_; + std::shared_ptr conn_; + RedisOperations ops_; + std::chrono::steady_clock::time_point failFastUntil_{}; // epoch = closed +}; + +} // namespace redis_util diff --git a/source/shared/redis/connection/redisConnection.cpp b/source/shared/redis/connection/redisConnection.cpp index e7a867b..b34abaa 100644 --- a/source/shared/redis/connection/redisConnection.cpp +++ b/source/shared/redis/connection/redisConnection.cpp @@ -31,6 +31,13 @@ std::shared_ptr makeRedisConnection( // io_context thread; a firing health check would otherwise cancel the next // command ("Operation canceled"). A dead connection still surfaces as an // async_exec error, which the caller handles. + // + // KNOWN TRADE-OFF: with no health check there is also no liveness probe, + // so a HALF-OPEN connection (peer vanished without RST — network partition, + // NAT timeout) leaves async_exec suspended forever rather than erroring; + // the worker then hangs silently between runs. Re-enabling the check is + // only safe once the tick loads move off this io_context thread (they + // currently block it for longer than any sane PING timeout). cfg.health_check_interval = std::chrono::seconds::zero(); conn->async_run(cfg, {}, diff --git a/source/shared/redis/positionClustering.cpp b/source/shared/redis/positionClustering.cpp new file mode 100644 index 0000000..c4f6a7a --- /dev/null +++ b/source/shared/redis/positionClustering.cpp @@ -0,0 +1,405 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "shared/redis/positionClustering.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "shared/redis/client/syncRedisClient.hpp" +#include "shared/utilities/backtestLog.hpp" + +namespace redis_clusters { + +static_assert( + [] { + for (std::size_t i = 1; i < kClusterTable.size(); ++i) { + if (!(kClusterTable[i - 1].symbol < kClusterTable[i].symbol)) { + return false; + } + } + return true; + }(), + "redis_clusters::kClusterTable must stay sorted ascending by symbol — " + "groupsFor binary-searches it"); + +static_assert( + [] { + for (const SymbolClusters& entry : kClusterTable) { + if (entry.count == 0 || entry.count > entry.groups.size()) { + return false; + } + for (std::size_t i = 0; i < entry.groups.size(); ++i) { + if ((i < entry.count) != !entry.groups[i].empty()) { + return false; + } + } + } + return true; + }(), + "redis_clusters::kClusterTable counts must match the non-empty prefix " + "of each groups array"); + +static_assert( + [] { + for (std::size_t i = 1; i < kClusterRules.size(); ++i) { + if (!(kClusterRules[i - 1].cluster < kClusterRules[i].cluster)) { + return false; + } + } + return true; + }(), + "redis_clusters::kClusterRules must stay sorted ascending by cluster — " + "clusterLimit/isStrictCluster binary-search it"); + +std::string clusterKey(const std::string_view cluster) { + return "CG#" + std::string{cluster}; +} + +std::string clusterLockKey(const std::string_view cluster) { + return "CLUSTER_LOCK#" + std::string{cluster}; +} + +std::span groupsFor( + const std::string_view symbol) noexcept { + std::size_t lo = 0; + std::size_t hi = kClusterTable.size(); + while (lo < hi) { + const std::size_t mid = lo + ((hi - lo) >> 1); + const SymbolClusters& entry = kClusterTable[mid]; + if (entry.symbol < symbol) { + lo = mid + 1; + } else if (symbol < entry.symbol) { + hi = mid; + } else { + return {entry.groups.data(), entry.count}; + } + } + return {}; +} + +namespace { + +const ClusterRule* findRule(const std::string_view cluster) noexcept { + std::size_t lo = 0; + std::size_t hi = kClusterRules.size(); + while (lo < hi) { + const std::size_t mid = lo + ((hi - lo) >> 1); + const ClusterRule& rule = kClusterRules[mid]; + if (rule.cluster < cluster) { + lo = mid + 1; + } else if (cluster < rule.cluster) { + hi = mid; + } else { + return &rule; + } + } + return nullptr; +} + +// "symbol#strategyName#dealReference" -> (symbol, strategyName); nullopt for +// anything with fewer than two '#'-separated fields (the C# parts.Length >= 2 +// tolerance — a malformed member is skipped, never trusted). +std::optional> splitMember( + const std::string_view member) { + const std::size_t firstHash = member.find('#'); + if (firstHash == std::string_view::npos) { + return std::nullopt; + } + const std::string_view rest = member.substr(firstHash + 1); + const std::size_t secondHash = rest.find('#'); + const std::string_view strategy = + secondHash == std::string_view::npos ? rest : rest.substr(0, secondHash); + return std::make_pair(member.substr(0, firstHash), strategy); +} + +} // namespace + +int clusterLimit(const std::string_view cluster) noexcept { + const ClusterRule* rule = findRule(cluster); + return rule != nullptr ? rule->limit : 1; // the C# GroupLimits fallback +} + +bool isStrictCluster(const std::string_view cluster) noexcept { + const ClusterRule* rule = findRule(cluster); + return rule != nullptr && rule->strict; +} + +ClusterVerdict evaluateClusterGroup(const std::span members, + const std::string_view symbol, + const std::string_view strategyName, + const int limit, const bool strict) { + // A. Capacity — counts EVERY member, well-formed or not, like the C# + // members.Length check. + if (members.size() >= static_cast(limit)) { + return ClusterVerdict::GroupFull; + } + // B. Same (symbol, strategy) already open — signal stacking. + for (const std::string& member : members) { + if (const auto parts = splitMember(member); + parts && parts->first == symbol && parts->second == strategyName) { + return ClusterVerdict::SymbolStrategyStacked; + } + } + // C. Strategy diversity, strict clusters only: the same strategy on two + // correlated members is the same risk twice. + if (strict) { + for (const std::string& member : members) { + if (const auto parts = splitMember(member); + parts && parts->second == strategyName) { + return ClusterVerdict::StrategyNotDiverse; + } + } + } + return ClusterVerdict::Allowed; +} + +std::string clusterMemberString(const ClusterMember& member) { + return member.symbol + "#" + member.strategyName + "#" + + member.dealReference; +} + +std::map, std::less<>> +groupMembersByCluster(const std::span members, + const GroupsLookup& lookup) { + std::map, std::less<>> clusters; + for (const ClusterMember& member : members) { + const std::string value = clusterMemberString(member); + for (const std::string_view group : lookup(member.symbol)) { + clusters[std::string{group}].push_back(value); + } + } + // Dedup (the C# HashSet) + sort, so a rebuilt set's SADD sequence is + // deterministic regardless of the broker's position order. + for (auto& [group, values] : clusters) { + std::ranges::sort(values); + const auto duplicates = std::ranges::unique(values); + values.erase(duplicates.begin(), duplicates.end()); + } + return clusters; +} + +namespace { + +// Distinguishes this process's temp keys from another producer's mid-swap +// (the C# used a fresh Guid per swap; one nonce per process is enough here — +// calls are serialised by mutex_, so only ANOTHER process can collide, and +// only during a rollout overlap). +const std::string& processNonce() { + static const std::string nonce = [] { + std::random_device device; + return std::format("{:08x}{:08x}", device(), device()); + }(); + return nonce; +} + +} // namespace + +PositionClustering::PositionClustering(const std::string& host, const int port) + : client_(std::make_unique(host, port)) {} + +PositionClustering::~PositionClustering() = default; + +bool PositionClustering::isClusterBlocked(const std::string& symbol, + const std::string& strategyName, + const std::chrono::seconds lockTtl) { + const std::lock_guard guard{mutex_}; + + const std::span groups = groupsFor(symbol); + if (groups.empty()) { + // Unmapped symbol: fail-closed, matching the C# (a symbol with no + // risk cluster must not trade around the portfolio caps). + backtest_log::error("PositionClustering: " + symbol + + " has no cluster mapping; blocking entry"); + return true; + } + + // Phase 1 — the cooldown try-locks, all-or-nothing. A group already + // locked means another entry is mid-check (or just passed) somewhere in + // an overlapping cluster: back off. Locks this check DID acquire are + // deliberately left to their short TTL rather than rolled back with a + // DEL — the delete is value-blind, and another worker's markOpened can + // overwrite an acquired key with the 2-minute post-open cooldown between + // our SET NX and the rollback; deleting it would reopen the exact + // CG#-staleness gap markOpened closes (and a same-symbol overwrite makes + // even a value-compared delete unsafe). The cost of not rolling back is + // bounded and safe-side: a sibling entry is over-blocked for at most + // lockTtl seconds. + for (const std::string_view group : groups) { + const std::string key = clusterLockKey(group); + std::string error; + const std::optional stored = client_->run( + client_->operations().setIfNotExists( + key, symbol, + std::chrono::duration_cast( + lockTtl)), + error); + if (stored && *stored) { + continue; + } + if (!stored && !error.empty()) { + // Fail-closed: lock state unknowable. (Empty error = the breaker + // skipped the probe; the original failure was already logged.) + backtest_log::error("PositionClustering: cluster lock check " + "failed for " + key + " (" + error + + "); failing closed"); + } + return true; + } + + // Phase 2 — evaluate each cluster's members. On any block (or unknowable + // membership) the cooldown locks are left to expire, quieting the + // cluster either way — the C# does the same. + for (const std::string_view group : groups) { + std::string error; + const std::optional> members = client_->run( + client_->operations().setMembers(clusterKey(group)), error); + if (!members) { + if (!error.empty()) { + backtest_log::error("PositionClustering: reading " + + clusterKey(group) + " failed (" + error + + "); failing closed"); + } + return true; + } + // Rejections route through error() like every other line here: the + // plain header offers no format logger, and a portfolio-cap block is + // operationally notable either way. Tags mirror the C# messages. + switch (evaluateClusterGroup(*members, symbol, strategyName, + clusterLimit(group), + isStrictCluster(group))) { + case ClusterVerdict::Allowed: + break; + case ClusterVerdict::GroupFull: + backtest_log::error(std::format( + "PositionClustering: [Limit] {} rejected — {} is full " + "({}/{})", + symbol, group, members->size(), clusterLimit(group))); + return true; + case ClusterVerdict::SymbolStrategyStacked: + backtest_log::error(std::format( + "PositionClustering: [Stacking] {} rejected — {} already " + "has an open position on this symbol", + symbol, strategyName)); + return true; + case ClusterVerdict::StrategyNotDiverse: + backtest_log::error(std::format( + "PositionClustering: [Diversity] {} rejected — {} " + "already has {}", + symbol, group, strategyName)); + return true; + } + } + + return false; +} + +void PositionClustering::markOpened(const std::string& symbol, + const std::chrono::seconds ttl) { + const std::lock_guard guard{mutex_}; + + // Unconditional SET (not NX): the deal is live at the broker, so any + // short check-time lock a concurrent signal holds is superseded — the + // capacity it was probing for is taken. Best-effort per key: a failure + // leaves that cluster to the short cooldown and the producer's next + // sync; there is no entry to refuse here, so nothing fails closed. + for (const std::string_view group : groupsFor(symbol)) { + const std::string key = clusterLockKey(group); + std::string error; + const std::optional stored = client_->run( + client_->operations().setString( + key, symbol, SetWhen::Always, + std::chrono::duration_cast(ttl)), + error); + if ((!stored || !*stored) && !error.empty()) { + // Empty error = the breaker skipped the probe (already logged). + backtest_log::error("PositionClustering: post-open cooldown for " + + key + " failed (" + error + + "); the producer's sync is the backstop"); + } + } +} + +bool PositionClustering::syncAllClusters( + const std::span members, + const std::chrono::milliseconds ttl) { + const std::lock_guard guard{mutex_}; + + bool allSynced = true; + for (const auto& [cluster, values] : + groupMembersByCluster(members, groupsFor)) { + const std::string tempKey = + "temp_CG#" + cluster + "#" + processNonce(); + std::string error; + + // A leftover temp key (a crash mid-swap under this same nonce) would + // merge stale members into the rebuilt set — clear it first. + if (!client_->run(client_->operations().deleteKey(tempKey), error) + && !error.empty()) { + backtest_log::error("PositionClustering: clearing " + tempKey + + " failed (" + error + "); skipping " + + cluster + " this sync"); + allSynced = false; + continue; + } + + bool filled = true; + for (const std::string& value : values) { + if (!client_->run(client_->operations().setAdd(tempKey, value), + error)) { + if (!error.empty()) { + backtest_log::error("PositionClustering: filling " + + tempKey + " failed (" + error + + "); skipping " + cluster + + " this sync"); + } + filled = false; + break; + } + } + if (!filled) { + // Best-effort cleanup; a survivor is also cleared by the DEL at + // the start of the next sync. CG# keeps its previous contents. + std::string cleanupError; + client_->run(client_->operations().deleteKey(tempKey), + cleanupError); + allSynced = false; + continue; + } + + // groupMembersByCluster never emits an empty cluster, so the temp key + // exists here — RENAME cannot fire on a missing source. + const std::string key = clusterKey(cluster); + if (!client_->run(client_->operations().keyRename(tempKey, key), + error)) { + if (!error.empty()) { + backtest_log::error("PositionClustering: swapping " + tempKey + + " -> " + key + " failed (" + error + + ")"); + } + allSynced = false; + continue; + } + if (!client_->run(client_->operations().setTTL(key, ttl), error)) { + // The swap landed but the expiry did not: the set is correct now + // yet immortal if the producer dies — flag it; the next sync's + // swap re-arms the TTL. + if (!error.empty()) { + backtest_log::error("PositionClustering: expiring " + key + + " failed (" + error + ")"); + } + allSynced = false; + } + } + return allSynced; +} + +} // namespace redis_clusters diff --git a/source/shared/redis/positionClustering.hpp b/source/shared/redis/positionClustering.hpp new file mode 100644 index 0000000..27fc3c2 --- /dev/null +++ b/source/shared/redis/positionClustering.hpp @@ -0,0 +1,267 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace redis_util { +class SyncRedisClient; +} + +// Redis-backed position clustering, mirroring the C# engine's +// PositionClustering.CheckForCluster — the portfolio-level entry gate that +// caps correlated exposure. Every symbol maps to one or more risk clusters +// (US_Index, USD_Forex, EUR_Pairs, ...); an entry is blocked when any of its +// clusters is at capacity, already holds this exact (symbol, strategy), or — +// for the STRICT clusters (correlated indices/commodities) — already holds +// any deal from the same strategy name. +// +// Wire contract shared with the C# ecosystem (do not drift): +// +// CG# Redis SET of "symbol#strategyName#dealReference" +// members, rebuilt every minute by the position +// producer (the `positions` subcommand via +// syncAllClusters below — historically the external +// C# cron) from the broker's own book: a temp-key +// RENAME swap and a 5-minute TTL, so a cluster +// whose deals all closed simply expires. The entry +// gate READS the sets; syncAllClusters is the only +// writer. +// CLUSTER_LOCK# Cooldown try-lock (SET NX PX, 10s) taken on +// every cluster the symbol belongs to BEFORE the +// sets are read. Two near-simultaneous entries into +// overlapping clusters serialise on it, and the +// locks are left to expire whatever the verdict. +// When the broker then ACCEPTS the open, markOpened +// re-arms the locks for a cooldown sized to the +// producer's ~2-minute sync cadence, so no other +// member clears a capacity check against CG# sets +// that do not yet hold the fresh deal. A rejected +// or failed open arms nothing — the cluster stays +// on the short 10s cooldown and reopens promptly. +// +// Fail-closed doctrine (same as tradeLocks.hpp): any Redis failure — lock +// state or membership unknowable — blocks the entry. A missed entry is +// recoverable; an entry that busts a cluster cap is not. Unmapped symbols +// are blocked outright, matching the C#. +// +// Asio-free header on purpose (same isolation pattern as tradeLocks.hpp / +// positionManager.hpp): the connection machinery lives behind +// positionClustering.cpp so module global module fragments can #include +// this alongside `import std`. +namespace redis_clusters { + +// Key builders — free functions so tests can pin the wire format against the +// C# producer without a Redis server. +std::string clusterKey(std::string_view cluster); // "CG#" +std::string clusterLockKey(std::string_view cluster); // "CLUSTER_LOCK#" + +// One symbol's cluster memberships. `groups` is padded with empty views up +// to the widest membership (EURNOK: 4); `count` is the used prefix. +struct SymbolClusters { + std::string_view symbol; + std::array groups; + std::size_t count; +}; + +// MUST stay sorted ascending by symbol (static_assert in the .cpp) — +// groupsFor binary-searches it. Mirrors the C# BuildCluster map verbatim, +// including two symbols the engine does not trade (GBPAUD, NGASCMDUSD): +// the producer may still write them into shared clusters, and dropping them +// here would silently change this side's view of the map. +inline constexpr std::array kClusterTable{{ + {"AUDNZD", {"Commodity_FX", "Crosses", "AUD_Pairs", ""}, 3}, + {"AUDUSD", {"USD_Forex", "Commodity_FX", "AUD_Pairs", ""}, 3}, + {"AUSIDXAUD", {"Asian_Index", "", "", ""}, 1}, + {"BRENTCMDUSD", {"Energy", "", "", ""}, 1}, + {"COPPERCMDUSD", {"Industrial_Metals", "", "", ""}, 1}, + {"DEUIDXEUR", {"EU_Index", "", "", ""}, 1}, + {"EURAUD", {"Crosses", "EUR_Pairs", "AUD_Pairs", ""}, 3}, + {"EURCHF", {"Crosses", "EUR_Pairs", "", ""}, 2}, + {"EURGBP", {"Crosses", "EUR_Pairs", "GBP_Pairs", ""}, 3}, + {"EURJPY", {"Crosses", "EUR_Pairs", "JPY_Pairs", ""}, 3}, + {"EURNOK", {"Commodity_FX", "Crosses", "EUR_Pairs", "Scandi_Pairs"}, 4}, + {"EURUSD", {"USD_Forex", "EUR_Pairs", "", ""}, 2}, + {"FRAIDXEUR", {"EU_Index", "", "", ""}, 1}, + {"GBPAUD", {"Crosses", "GBP_Pairs", "AUD_Pairs", ""}, 3}, + {"GBPJPY", {"Crosses", "GBP_Pairs", "JPY_Pairs", ""}, 3}, + {"GBPUSD", {"USD_Forex", "GBP_Pairs", "", ""}, 2}, + {"GBRIDXGBP", {"UK_Index", "", "", ""}, 1}, + {"HKGIDXHKD", {"Asian_Index", "", "", ""}, 1}, + {"JPNIDXJPY", {"Asian_Index", "", "", ""}, 1}, + {"LIGHTCMDUSD", {"Energy", "", "", ""}, 1}, + {"NGASCMDUSD", {"Energy", "", "", ""}, 1}, + {"NZDUSD", {"USD_Forex", "Commodity_FX", "", ""}, 2}, + {"USA30IDXUSD", {"US_Index", "", "", ""}, 1}, + {"USA500IDXUSD", {"US_Index", "", "", ""}, 1}, + {"USATECHIDXUSD", {"US_Index", "", "", ""}, 1}, + {"USDCAD", {"USD_Forex", "Commodity_FX", "", ""}, 2}, + {"USDCHF", {"USD_Forex", "", "", ""}, 1}, + {"USDJPY", {"USD_Forex", "JPY_Pairs", "", ""}, 2}, + {"USDSEK", {"USD_Forex", "Scandi_Pairs", "", ""}, 2}, + {"XAGUSD", {"Precious_Metals", "", "", ""}, 1}, + {"XAUUSD", {"Precious_Metals", "", "", ""}, 1}, +}}; + +// Per-cluster rule: capacity, and whether the cluster enforces strategy +// DIVERSITY (strict = correlated assets where the same strategy on two +// members is the same risk twice). Mirrors the C# GroupLimits + +// StrictClusters. Sorted ascending by cluster name (static_assert in .cpp). +struct ClusterRule { + std::string_view cluster; + int limit; + bool strict; +}; + +inline constexpr std::array kClusterRules{{ + {"AUD_Pairs", 2, false}, + {"Asian_Index", 1, true}, + {"Commodity_FX", 2, false}, + {"Crosses", 2, false}, + {"EUR_Pairs", 2, false}, + {"EU_Index", 1, true}, + {"Energy", 1, true}, + {"GBP_Pairs", 2, false}, + {"Industrial_Metals", 1, true}, + {"JPY_Pairs", 2, false}, + {"Precious_Metals", 1, true}, + {"Scandi_Pairs", 1, false}, + {"UK_Index", 1, true}, + {"USD_Forex", 2, false}, + {"US_Index", 1, true}, +}}; + +// The clusters `symbol` belongs to (a view into kClusterTable); empty when +// the symbol is unmapped — the caller must then BLOCK (C# returns true). +[[nodiscard]] std::span groupsFor( + std::string_view symbol) noexcept; + +// Capacity of a cluster; 1 for a cluster missing from kClusterRules (the C# +// GroupLimits fallback). Strictness defaults to false for unknown clusters. +[[nodiscard]] int clusterLimit(std::string_view cluster) noexcept; +[[nodiscard]] bool isStrictCluster(std::string_view cluster) noexcept; + +// The decision for ONE cluster, given its current CG# members — pure logic +// (limit/strict passed in from the tables above), exported so tests can pin +// every branch without a server. Checks run in the C# order: capacity +// first, then same-(symbol, strategy) stacking, then (strict only) strategy +// diversity. Note the diversity branch is currently shadowed for the real +// tables — every strict cluster has limit 1, so any occupant reports +// GroupFull first — but it guards the day a strict cluster's limit is +// raised, exactly like the C#. Members are +// "symbol#strategyName#dealReference"; anything with fewer than two +// '#'-separated fields is skipped, like the C#. +enum class ClusterVerdict { + Allowed, + GroupFull, // members >= limit + SymbolStrategyStacked, // this exact (symbol, strategy) already open + StrategyNotDiverse, // strict cluster already holds this strategy +}; + +[[nodiscard]] ClusterVerdict evaluateClusterGroup( + std::span members, std::string_view symbol, + std::string_view strategyName, int limit, bool strict); + +// One open broker deal, as the position producer sees it — the raw material +// of a CG# member. `strategyName` (not the UUID) on purpose: the strict +// clusters' diversity rule compares strategy NAMES, and the C# collector fed +// them the same way. +struct ClusterMember { + std::string symbol; + std::string strategyName; + std::string dealReference; +}; + +// The CG# member wire format: "symbol#strategyName#dealReference" (the shape +// splitMember/evaluateClusterGroup parse back). Free so tests pin it against +// the C# producer without a server. +std::string clusterMemberString(const ClusterMember& member); + +// Lookup seam so tests can group against their own cluster table — the same +// injection pattern as live::MarketLookup. Production passes groupsFor. +using GroupsLookup = + std::function(std::string_view)>; + +// The C# SyncAllClusters grouping stage, pure: fan each member out to every +// cluster its symbol belongs to, dedup within a cluster (the C# HashSet) and +// sort for determinism. Members whose symbol is unmapped are dropped, like +// the C# TryGetValue skip — the ENTRY gate is where an unmapped symbol +// fails closed; a producer must still mirror the rest of the book. +std::map, std::less<>> +groupMembersByCluster(std::span members, + const GroupsLookup& lookup); + +class PositionClustering { +public: + // Connects lazily on first use via the shared Boost.Redis connection + // (host from $REDIS_HOST in the caller; port 6379 by convention). + PositionClustering(const std::string& host, int port); + ~PositionClustering(); + PositionClustering(const PositionClustering&) = delete; + PositionClustering& operator=(const PositionClustering&) = delete; + + // The C# CheckForCluster: TRUE = block the entry. Acquires the + // CLUSTER_LOCK cooldown on every cluster of `symbol` (backing off if any + // is already held — locks it did acquire are left to their short TTL, + // never DEL-rolled-back: a value-blind delete could destroy a concurrent + // markOpened cooldown, see the .cpp), then evaluates each cluster's + // CG# members. On a PASS the cooldown locks are left to expire (10s); + // on a verdict block they are too (the losing signal already paid the + // round trips — the cooldown quiets the cluster either way). Fail-closed + // on every Redis failure. `lockTtl` is parameterised for tests only. + bool isClusterBlocked(const std::string& symbol, + const std::string& strategyName, + std::chrono::seconds lockTtl = + std::chrono::seconds{10}); + + // Called AFTER the broker accepted an open on `symbol` (HTTP 200 + + // dealStatus resolved — the order channel's Accepted branch, never on + // Rejected/Failed): unconditionally re-arms CLUSTER_LOCK on every + // cluster the symbol belongs to for `ttl`. The new position will not + // appear in CG# until the external producer's next broker-book sync + // (~2-minute cadence), and the check-time 10s lock has usually expired + // by then — without this hold, a different strategy in the cluster + // clears a capacity check against sets that predate the deal. The SET + // is unconditional (not NX): the position exists, so stomping a + // concurrent checker's short lock only strengthens the guard. + // Best-effort: failures log and rely on the producer sync as backstop — + // the deal is already open, so there is nothing to fail closed FOR. + void markOpened(const std::string& symbol, + std::chrono::seconds ttl = std::chrono::minutes{2}); + + // The producer side (port of the C# SyncAllClusters), the class's only + // writer: for every NON-empty cluster the members map to, SADD them into + // a temp key and RENAME it over CG# (an atomic swap — old + // zombies vanish with it), then PEXPIRE `ttl`. Clusters with no current + // members are left to expire on their previous TTL, exactly like the C#. + // The RENAME only runs after every SADD succeeded, so it can never fire + // on a missing temp key. Returns false when any cluster's swap failed + // (its CG# keeps the previous contents; the next minute's sync repairs + // it). + bool syncAllClusters(std::span members, + std::chrono::milliseconds ttl = + std::chrono::minutes{5}); + +private: + std::unique_ptr client_; + // Serialises concurrent callers sharing one instance (one connection, + // one synchronous pump) — prefer one instance per worker thread (see + // brokerOrderSink::threadChannel). + std::mutex mutex_; +}; + +} // namespace redis_clusters diff --git a/source/shared/redis/positionManager.cpp b/source/shared/redis/positionManager.cpp new file mode 100644 index 0000000..c1e8738 --- /dev/null +++ b/source/shared/redis/positionManager.cpp @@ -0,0 +1,535 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "shared/redis/positionManager.hpp" + +#include +#include +#include +#include + +#include + +#include "shared/redis/client/syncRedisClient.hpp" +#include "shared/utilities/backtestLog.hpp" + +namespace { + +// Closed deals are kept for 60 days, matching the C# Remove(). +constexpr std::chrono::hours kHistoryTtl{24 * 60}; + +} // namespace + +namespace redis_positions { + +std::string positionListKey(const std::string& strategyId) { + return "PL#" + strategyId; +} + +std::string positionKey(const std::string& dealId) { return "PO#" + dealId; } + +std::string historyPositionKey(const std::string& dealId) { + return "PH#" + dealId; +} + +std::string dealReceiptKey(const std::string& dealReference, + const std::string& symbol) { + return "DealId#" + dealReference + "#" + symbol; +} + +std::string encodePositionList(const std::vector& dealIds) { + return nlohmann::json(dealIds).dump(); +} + +std::optional> decodePositionList( + const std::string& json) { + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(json); + } catch (const std::exception&) { + return std::nullopt; + } + if (!parsed.is_array()) { + return std::nullopt; + } + std::vector dealIds; + dealIds.reserve(parsed.size()); + for (const auto& item : parsed) { + if (!item.is_string()) { + return std::nullopt; + } + dealIds.push_back(item.get()); + } + return dealIds; +} + +std::optional decodePositionRecord(const std::string& json) { + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(json); + } catch (const std::exception&) { + return std::nullopt; + } + if (!parsed.is_object()) { + return std::nullopt; + } + PositionRecord record; + const auto readString = [&parsed](const char* key, std::string& out) { + if (const auto it = parsed.find(key); + it != parsed.end() && it->is_string()) { + out = it->get(); + } + }; + const auto readInt32 = [&parsed](const char* key, std::int32_t& out) { + if (const auto it = parsed.find(key); + it != parsed.end() && it->is_number()) { + out = it->get(); + } + }; + readString("dealId", record.dealId); + readString("dealReference", record.dealReference); + readString("symbol", record.symbol); + readString("epic", record.epic); + readString("direction", record.direction); + readString("strategyId", record.strategyId); + readString("strategyName", record.strategyName); + readInt32("level", record.level); + readInt32("stopLevel", record.stopLevel); + readInt32("limitLevel", record.limitLevel); + if (const auto it = parsed.find("size"); + it != parsed.end() && it->is_number()) { + record.size = it->get(); + } + if (const auto it = parsed.find("openedAt"); + it != parsed.end() && it->is_number()) { + record.openedAtMicros = it->get(); + } + // Only these three are load-bearing for the book sync (identity, which + // worker owns it, and which way it points); everything else may be + // absent from a producer-rewritten payload. + if (record.dealReference.empty() || record.symbol.empty() || + record.direction.empty()) { + return std::nullopt; + } + return record; +} + +std::string encodePositionRecord(const PositionRecord& record) { + // Hand-rolled to stay byte-compatible with buildPositionPayload + // (orderChannel) — same fields, same order. Every string is a broker or + // engine identifier (no user text), so no JSON escaping is needed, the + // same doctrine the writer relies on. + return std::format( + R"({{"dealId":"{}","dealReference":"{}","symbol":"{}","epic":"{}",)" + R"("direction":"{}","size":{},"level":{},"stopLevel":{},)" + R"("limitLevel":{},"strategyId":"{}","strategyName":"{}",)" + R"("openedAt":{}}})", + record.dealId, record.dealReference, record.symbol, record.epic, + record.direction, record.size, record.level, record.stopLevel, + record.limitLevel, record.strategyId, record.strategyName, + record.openedAtMicros); +} + +std::optional decodeDealReceipt(const std::string& json) { + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(json); + } catch (const std::exception&) { + return std::nullopt; + } + if (!parsed.is_object()) { + return std::nullopt; + } + DealReceipt receipt; + const auto readString = [&parsed](const char* key, std::string& out) { + if (const auto it = parsed.find(key); + it != parsed.end() && it->is_string()) { + out = it->get(); + } + }; + readString("strategyId", receipt.strategyId); + readString("dealId", receipt.dealId); + readString("strategyName", receipt.strategyName); + return receipt; +} + +PositionManager::PositionManager(const std::string& host, const int port) + : client_(std::make_unique(host, port)) {} + +PositionManager::~PositionManager() = default; + +// Expects mutex_ to be held by the calling public method. +PositionManager::ListState PositionManager::fetchList( + const std::string& strategyId, std::vector& dealIds) { + const std::string key = positionListKey(strategyId); + std::string error; + const auto value = client_->run(client_->operations().getString(key), error); + if (!value) { + // An empty error means the breaker skipped the probe (the original + // failure was already logged) — same convention as TradeLocks. + if (!error.empty()) { + backtest_log::error("PositionManager: reading " + key + " failed (" + + error + ")"); + } + return ListState::Failed; + } + if (!value->has_value()) { + return ListState::Missing; + } + auto decoded = decodePositionList(**value); + if (!decoded) { + // A corrupt list is indistinguishable from an unknown position count, + // so it reports Failed (callers fail closed) rather than "empty". + backtest_log::error("PositionManager: " + key + + " is not a JSON string array; treating the " + "position state as unknown"); + return ListState::Failed; + } + dealIds = std::move(*decoded); + return ListState::Ok; +} + +std::optional> PositionManager::getPositionsList( + const std::string& strategyId) { + const std::lock_guard guard{mutex_}; + std::vector dealIds; + switch (fetchList(strategyId, dealIds)) { + case ListState::Ok: + case ListState::Missing: + return dealIds; + case ListState::Failed: + break; + } + return std::nullopt; +} + +std::optional PositionManager::getPositionCount( + const std::string& strategyId) { + const auto payloads = getPositionPayloads(strategyId); + if (!payloads) { + return std::nullopt; + } + return static_cast(payloads->size()); +} + +bool PositionManager::savePosition(const std::string& dealId, + const std::string& payload, + const std::chrono::milliseconds ttl) { + const std::lock_guard guard{mutex_}; + const std::string key = positionKey(dealId); + std::string error; + const auto stored = client_->run( + client_->operations().setString(key, payload, SetWhen::Always, ttl), + error); + if (!stored) { + if (!error.empty()) { + backtest_log::error("PositionManager: saving " + key + " failed (" + + error + ")"); + } + return false; + } + return *stored; +} + +bool PositionManager::saveDealReceipt(const std::string& dealReference, + const std::string& symbol, + const std::string& payload, + const std::chrono::hours ttl) { + const std::lock_guard guard{mutex_}; + const std::string key = dealReceiptKey(dealReference, symbol); + std::string error; + const auto stored = client_->run( + client_->operations().setString( + key, payload, SetWhen::Always, + std::chrono::duration_cast(ttl)), + error); + if (!stored) { + if (!error.empty()) { + backtest_log::error("PositionManager: saving receipt " + key + + " failed (" + error + ")"); + } + return false; + } + return *stored; +} + +bool PositionManager::addPosition(const std::string& strategyId, + const std::string& dealId) { + const std::lock_guard guard{mutex_}; + std::vector dealIds; + if (fetchList(strategyId, dealIds) == ListState::Failed) { + return false; + } + if (std::ranges::find(dealIds, dealId) != dealIds.end()) { + return true; // already tracked — the C# add is idempotent too + } + dealIds.push_back(dealId); + + const std::string key = positionListKey(strategyId); + std::string error; + // No TTL on the list on purpose (matching C#): it is pruned by + // refreshPositionList as the PO# deals expire, not by its own expiry. + const auto stored = client_->run( + client_->operations().setString(key, encodePositionList(dealIds), + SetWhen::Always, std::nullopt), + error); + if (!stored) { + if (!error.empty()) { + backtest_log::error("PositionManager: adding " + dealId + " to " + + key + " failed (" + error + ")"); + } + return false; + } + return *stored; +} + +std::optional>> +PositionManager::getPositionPayloads(const std::string& strategyId) { + const std::lock_guard guard{mutex_}; + + std::vector dealIds; + switch (fetchList(strategyId, dealIds)) { + case ListState::Missing: + return std::vector>{}; + case ListState::Failed: + return std::nullopt; + case ListState::Ok: + break; + } + if (dealIds.empty()) { + return std::vector>{}; + } + + std::vector keys; + keys.reserve(dealIds.size()); + for (const std::string& dealId : dealIds) { + keys.push_back(positionKey(dealId)); + } + // Same MGET shape as refreshPositionList: missing keys (expired PO#) are + // simply absent from the result, and filtering the ORIGINAL list keeps + // the stored order. Read-only — the list itself is never rewritten here. + std::string error; + const auto liveDeals = client_->run( + client_->operations().getMultiple(std::move(keys)), error); + if (!liveDeals) { + if (!error.empty()) { + backtest_log::error("PositionManager: reading payloads for " + + positionListKey(strategyId) + " failed (" + + error + ")"); + } + return std::nullopt; + } + + std::vector> payloads; + payloads.reserve(dealIds.size()); + for (std::string& dealId : dealIds) { + if (const auto it = liveDeals->find(positionKey(dealId)); + it != liveDeals->end()) { + payloads.emplace_back(std::move(dealId), it->second); + } + } + return payloads; +} + +bool PositionManager::refreshPositionList(const std::string& strategyId) { + const std::lock_guard guard{mutex_}; + const std::string listKey = positionListKey(strategyId); + + std::vector dealIds; + switch (fetchList(strategyId, dealIds)) { + case ListState::Missing: + return true; // no list — nothing to refresh (C# skips too) + case ListState::Failed: + return false; + case ListState::Ok: + break; + } + + std::string error; + std::vector survivors; + if (!dealIds.empty()) { + std::vector keys; + keys.reserve(dealIds.size()); + for (const std::string& dealId : dealIds) { + keys.push_back(positionKey(dealId)); + } + // MGET drops keys without values, so the surviving deals are exactly + // the ones the broker (via the producer) still refreshes. Filtering + // the ORIGINAL list keeps the stored order deterministic. + const auto liveDeals = client_->run( + client_->operations().getMultiple(std::move(keys)), error); + if (!liveDeals) { + if (!error.empty()) { + backtest_log::error("PositionManager: refreshing " + listKey + + " failed (" + error + ")"); + } + return false; + } + for (std::string& dealId : dealIds) { + if (liveDeals->contains(positionKey(dealId))) { + survivors.push_back(std::move(dealId)); + } + } + } + + if (survivors.empty()) { + const auto removed = + client_->run(client_->operations().deleteKey(listKey), error); + if (!removed && !error.empty()) { + backtest_log::error("PositionManager: deleting emptied " + listKey + + " failed (" + error + ")"); + } + return removed.has_value(); + } + + const auto stored = client_->run( + client_->operations().setString(listKey, encodePositionList(survivors), + SetWhen::Always, std::nullopt), + error); + if (!stored) { + if (!error.empty()) { + backtest_log::error("PositionManager: rewriting " + listKey + + " failed (" + error + ")"); + } + return false; + } + return *stored; +} + +bool PositionManager::removePosition(const std::string& strategyId, + const std::string& dealId) { + const std::lock_guard guard{mutex_}; + std::string error; + + // Move the deal to history: GETDEL then SET PX(60d). Not atomic like the + // C# RENAME, but a deal that already expired is a normal case here (short + // TTL by design) and must not surface as a Redis error; losing the + // history copy to a crash between the two steps only costs debug data. + const auto payload = client_->run( + client_->operations().getDelete(positionKey(dealId)), error); + if (!payload) { + if (!error.empty()) { + backtest_log::error("PositionManager: closing " + + positionKey(dealId) + " failed (" + error + + ")"); + } + return false; + } + if (payload->has_value()) { + const auto archived = client_->run( + client_->operations().setString( + historyPositionKey(dealId), **payload, SetWhen::Always, + std::chrono::duration_cast( + kHistoryTtl)), + error); + if (!archived && !error.empty()) { + backtest_log::error("PositionManager: archiving " + + historyPositionKey(dealId) + " failed (" + + error + ")"); + } + } + + std::vector dealIds; + switch (fetchList(strategyId, dealIds)) { + case ListState::Missing: + return true; // no list — nothing to remove the id from + case ListState::Failed: + return false; + case ListState::Ok: + break; + } + std::erase(dealIds, dealId); + + // Written back even when unchanged or empty, matching the C# Remove(); + // refreshPositionList deletes an emptied list on its next pass. + const auto stored = client_->run( + client_->operations().setString(positionListKey(strategyId), + encodePositionList(dealIds), + SetWhen::Always, std::nullopt), + error); + if (!stored) { + if (!error.empty()) { + backtest_log::error("PositionManager: removing " + dealId + + " from " + positionListKey(strategyId) + + " failed (" + error + ")"); + } + return false; + } + return *stored; +} + +std::optional> PositionManager::getPositionPayload( + const std::string& dealId) { + const std::lock_guard guard{mutex_}; + const std::string key = positionKey(dealId); + std::string error; + auto value = client_->run(client_->operations().getString(key), error); + if (!value) { + if (!error.empty()) { + backtest_log::error("PositionManager: reading " + key + " failed (" + + error + ")"); + } + return std::nullopt; + } + return std::move(*value); +} + +std::optional> +PositionManager::getHistoryPositionPayload(const std::string& dealId) { + const std::lock_guard guard{mutex_}; + const std::string key = historyPositionKey(dealId); + std::string error; + auto value = client_->run(client_->operations().getString(key), error); + if (!value) { + if (!error.empty()) { + backtest_log::error("PositionManager: reading " + key + " failed (" + + error + ")"); + } + return std::nullopt; + } + return std::move(*value); +} + +std::optional PositionManager::getDealReceipt( + const std::string& dealReference, const std::string& symbol) { + const std::lock_guard guard{mutex_}; + const std::string key = dealReceiptKey(dealReference, symbol); + std::string error; + const auto value = client_->run(client_->operations().getString(key), error); + if (!value) { + if (!error.empty()) { + backtest_log::error("PositionManager: reading receipt " + key + + " failed (" + error + ")"); + } + return std::nullopt; + } + return *value; // inner nullopt (missing receipt) flattens to nullopt +} + +std::optional> PositionManager::listStrategyIds() { + const std::lock_guard guard{mutex_}; + static constexpr std::string_view kPrefix = "PL#"; + std::string error; + auto keys = client_->run( + client_->operations().getKeysByPattern(std::string{kPrefix} + "*"), + error); + if (!keys) { + if (!error.empty()) { + backtest_log::error("PositionManager: scanning PL#* failed (" + + error + ")"); + } + return std::nullopt; + } + std::vector strategyIds; + strategyIds.reserve(keys->size()); + for (std::string& key : *keys) { + strategyIds.push_back(key.substr(kPrefix.size())); + } + std::ranges::sort(strategyIds); + return strategyIds; +} + +} // namespace redis_positions diff --git a/source/shared/redis/positionManager.hpp b/source/shared/redis/positionManager.hpp new file mode 100644 index 0000000..c9045f5 --- /dev/null +++ b/source/shared/redis/positionManager.hpp @@ -0,0 +1,209 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace redis_util { +class SyncRedisClient; +} + +// Redis-backed broker position store, mirroring the C# engine's +// PositionManager. Three key families, a wire contract shared with the C# +// engine and the external position producer (which repopulates the PO# deals +// from the broker every ~2 minutes): +// +// PL# -> JSON array of open deal ids for that strategy +// PO# -> the live deal payload, SHORT TTL: written when the +// engine opens a position and refreshed by the external +// producer, so a deal the broker never confirms (or has +// closed) simply expires and falls out of the PL# list +// on the next refreshPositionList +// PH# -> closed deal kept 60 days for debugging/history +// +// Same isolation pattern as tradeLocks.hpp: this header is Asio-free so +// module global module fragments can #include it alongside `import std`; the +// connection machinery (redis_util::SyncRedisClient — the deadline-bounded, +// fail-fast pump) lives behind positionManager.cpp. +namespace redis_positions { + +// Key builders — free functions so tests can pin the formats (a drifted +// prefix would silently split the position space between the two engines and +// the producer) without a Redis server. +std::string positionListKey(const std::string& strategyId); // "PL#" +std::string positionKey(const std::string& dealId); // "PO#" +std::string historyPositionKey(const std::string& dealId); // "PH#" +// "DealId##" — the C# deal receipt written next to +// every accepted open, kept 60 days for traceability (maps a client deal +// reference back to strategy/deal metadata long after the position closed). +std::string dealReceiptKey(const std::string& dealReference, + const std::string& symbol); + +// The PL# value is a JSON array of deal-id strings (the C# side writes it +// with JsonSerializer.Serialize(List)). Codec exposed for the same +// reason as the key builders. decode returns nullopt for anything that is +// not a JSON array of strings. +std::string encodePositionList(const std::vector& dealIds); +std::optional> decodePositionList( + const std::string& json); + +// One decoded PO# payload — the shape orderChannel::buildPositionPayload +// writes when the engine opens a deal (and the external producer later +// overwrites from the broker book). +struct PositionRecord { + std::string dealId; // empty until the deal confirmed + std::string dealReference; // the PO#/PL# book key + std::string symbol; + std::string epic; + std::string direction; // "BUY" | "SELL" + double size{}; // broker units (post size-modifier) + std::int32_t level{}; // scaled INT32 points (see priceData) + std::int32_t stopLevel{}; + std::int32_t limitLevel{}; + std::string strategyId; + std::string strategyName; + std::int64_t openedAtMicros{}; +}; + +// Decode tolerantly — only dealReference, symbol and direction are required +// (a producer-rewritten payload may drop the rest); nullopt for non-objects +// or when a required field is missing/empty. +std::optional decodePositionRecord(const std::string& json); + +// The inverse: the exact PO# wire shape orderChannel::buildPositionPayload +// writes (same field order, hand-rolled — nlohmann sorts keys alphabetically, +// which would drift the stored shape from the C# JsonSerializer's). Used by +// the position producer to rewrite a record it just updated. +std::string encodePositionRecord(const PositionRecord& record); + +// The DealId## receipt payload, as written by +// orderChannel::buildDealReceipt (and the C# engine before it) — only the +// fields the position producer reads back. Decode is tolerant like +// decodePositionRecord: nullopt only for unparseable/non-object JSON; absent +// fields decode as empty strings (the caller substitutes "Unknown"). +struct DealReceipt { + std::string strategyId; + std::string dealId; + std::string strategyName; +}; +std::optional decodeDealReceipt(const std::string& json); + +class PositionManager { +public: + // Connects lazily on first use via the shared Boost.Redis connection + // (host from $REDIS_HOST in the caller; port 6379 by convention). + PositionManager(const std::string& host, int port); + ~PositionManager(); + PositionManager(const PositionManager&) = delete; + PositionManager& operator=(const PositionManager&) = delete; + + // GET PL#. A missing list key means "no open positions" and + // returns an empty vector. nullopt means UNKNOWN — Redis unreachable, + // timed out, or the stored value was not a JSON string array (logged) — + // so callers gating entries on the count can fail closed. + std::optional> getPositionsList( + const std::string& strategyId); + + // Number of PL# deals whose PO# still exists (getPositionPayloads' + // survivor rule), NOT the raw list size: nothing in THIS engine prunes + // PL# when the broker itself closes a deal (stop/limit), so a raw count + // would hold the strategy at MAX_OPEN_TRADES until the producer next + // rewrites the list — an expired PO# must stop counting the moment the + // book sync stops seeing it. 0 when the list key is missing; nullopt + // when the state is unknown (see getPositionsList). + std::optional getPositionCount(const std::string& strategyId); + + // PL# -> (dealReference, PO# payload) pairs in list order, + // READ-ONLY (never rewrites PL# — pruning is refreshPositionList's / + // the producer's contract). References whose PO# has expired are + // OMITTED: an expired deal is unconfirmed or closed at the broker (the + // PO#-TTL doctrine). Empty vector = no open positions; nullopt = + // UNKNOWN (Redis unreachable or a corrupt list). + std::optional>> + getPositionPayloads(const std::string& strategyId); + + // SET PO# payload PX(ttl) — records the deal the moment the + // engine opens it. The TTL is deliberately SHORT: the external producer + // refreshes the key every ~2 minutes from the broker's own book, so the + // default outlives two missed refresh cycles and an unconfirmed deal + // self-expires instead of counting against the strategy forever. + bool savePosition(const std::string& dealId, const std::string& payload, + std::chrono::milliseconds ttl = std::chrono::minutes{5}); + + // Appends dealId to the PL# list (creating it if absent); a dealId + // already present is left alone. Returns false when the list state is + // unknown or the write failed. + bool addPosition(const std::string& strategyId, const std::string& dealId); + + // SET DealId## payload PX(ttl) — the C# deal + // receipt (60-day default, matching the PH# history retention). + bool saveDealReceipt(const std::string& dealReference, + const std::string& symbol, const std::string& payload, + std::chrono::hours ttl = std::chrono::hours{24 * 60}); + + // Re-derives the PL# list from the PO# deals that still exist: MGET + // drops expired deals, survivors are written back in their original + // order, and a list with no survivors is deleted — the C# refresh loop. + // Returns false when Redis failed mid-refresh (the list is left as-is). + bool refreshPositionList(const std::string& strategyId); + + // GET PO#, raw. Outer nullopt = Redis failure (state UNKNOWN — + // the producer must not rebuild a possibly-live record from scratch); + // inner nullopt = key missing (a genuinely new/expired deal, the fresh- + // save path). + std::optional> getPositionPayload( + const std::string& dealId); + + // GET PH#, raw — the 60-day close-history record. Same contract + // as getPositionPayload: outer nullopt = Redis failure, inner nullopt = + // missing. The tracking consumer reads history first (a DELETED deal has + // usually already been archived) before falling back to the live PO#. + std::optional> getHistoryPositionPayload( + const std::string& dealId); + + // GET DealId##, raw. nullopt for missing AND for + // failed — the producer coalesces both to strategy "Unknown", exactly + // like the C# LookUpDealId's null propagation. + std::optional getDealReceipt(const std::string& dealReference, + const std::string& symbol); + + // Every strategyId with a PL# list, via SCAN PL#* (never KEYS), sorted + // for deterministic logs. The producer refreshes each — the C# iterated + // its deployed-strategies config, which this engine does not have; the + // keyspace itself is the equivalent source (and also covers lists whose + // strategy is no longer deployed). nullopt = Redis failure. + std::optional> listStrategyIds(); + + // Closes a deal: moves PO# to PH# with a 60-day TTL + // (GETDEL + SET rather than RENAME — a RENAME on an already-expired deal + // raises a Redis error, which would trip the client's circuit breaker for + // a perfectly normal case) and removes the id from the PL# list. + bool removePosition(const std::string& strategyId, + const std::string& dealId); + +private: + // Missing distinguishes "no PL# key" (a valid empty book) from Failed + // (unknown state): refresh skips the former and aborts on the latter. + enum class ListState { Ok, Missing, Failed }; + ListState fetchList(const std::string& strategyId, + std::vector& dealIds); + + std::unique_ptr client_; + // Serialises concurrent callers sharing one instance (one connection, one + // synchronous pump) — same pattern as TradeLocks; prefer one instance per + // thread (see RedisPositionCounter). + std::mutex mutex_; +}; + +} // namespace redis_positions diff --git a/source/shared/redis/tradeLocks.cpp b/source/shared/redis/tradeLocks.cpp new file mode 100644 index 0000000..7076c11 --- /dev/null +++ b/source/shared/redis/tradeLocks.cpp @@ -0,0 +1,144 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "shared/redis/tradeLocks.hpp" + +#include // gethostname + +#include +#include + +#include "shared/redis/client/syncRedisClient.hpp" +#include "shared/utilities/backtestLog.hpp" + +namespace { + +// Lock keys record who held them (aids debugging a stuck lock from +// redis-cli); the value plays no part in the NX semantics. +std::string lockOwner() { + char buf[256]; + if (::gethostname(buf, sizeof(buf)) != 0) { + return "live"; + } + buf[sizeof(buf) - 1] = '\0'; // POSIX leaves truncation unspecified + return buf; +} + +} // namespace + +namespace redis_locks { + +std::string lockKey(const std::string& strategyUuid, + const std::string& direction) { + return "LOCK#" + strategyUuid + "#" + direction; +} + +TradeLocks::TradeLocks(const std::string& host, const int port) + : client_(std::make_unique(host, port)) {} + +TradeLocks::~TradeLocks() = default; + +bool TradeLocks::isThereATradeLock(const std::string& strategyUuid, + const std::string& direction, + const std::chrono::seconds ttl) { + const std::lock_guard guard{mutex_}; + const std::string key = lockKey(strategyUuid, direction); + const auto now = std::chrono::steady_clock::now(); + + // A lock we acquired is held at least until the TTL we stored it with + // runs out — `now` is taken BEFORE the SET, so the cached instant can + // only under-estimate Redis's expiry, never overshoot it. While it is + // known-held, answer locally instead of paying a round trip per signal. + // (An external early release — e.g. a manual redis-cli DEL — goes + // unnoticed until the cached expiry; that can only delay an entry, never + // ungate one.) + if (const auto it = heldUntil_.find(key); it != heldUntil_.end()) { + if (now < it->second) { + return true; + } + heldUntil_.erase(it); + } + + std::string error; + // SET NX PX: true = key was absent and is now ours (no pre-existing lock). + const std::optional stored = client_->run( + client_->operations().setIfNotExists( + key, lockOwner(), + std::chrono::duration_cast(ttl)), + error); + if (!stored) { + // Fail-closed: with Redis unreachable the lock state is unknowable, so + // report "locked" — blocking an entry is recoverable, an ungated order + // is not. An empty error means the breaker skipped the probe (the + // original failure was already logged). + if (!error.empty()) { + backtest_log::error("TradeLocks: lock check failed for " + key + + " (" + error + "); failing closed"); + } + return true; + } + if (*stored) { + heldUntil_[key] = now + ttl; + return false; + } + return true; +} + +bool TradeLocks::extendLock(const std::string& strategyUuid, + const std::string& direction, + const std::chrono::seconds ttl) { + const std::lock_guard guard{mutex_}; + const std::string key = lockKey(strategyUuid, direction); + const auto now = std::chrono::steady_clock::now(); + + std::string error; + // Unconditional SET PX — no NX: extending must succeed whether or not the + // old TTL already lapsed, exactly like the C# extend. + const std::optional stored = client_->run( + client_->operations().setString( + key, lockOwner(), + SetWhen::Always, + std::chrono::duration_cast(ttl)), + error); + if (!stored || !*stored) { + if (!error.empty()) { + backtest_log::error("TradeLocks: extending " + key + " failed (" + + error + ")"); + } + // The cached expiry (if any) still under-estimates the surviving TTL, + // so it stays valid — leave it alone. + return false; + } + heldUntil_[key] = now + ttl; + return true; +} + +bool TradeLocks::releaseLock(const std::string& strategyUuid, + const std::string& direction) { + const std::lock_guard guard{mutex_}; + const std::string key = lockKey(strategyUuid, direction); + // Whatever DEL returns, this instance no longer considers the lock held: + // dropping the cached expiry forces the next isThereATradeLock through a + // real SET NX, which sees the truth either way (key gone -> reacquire; + // DEL failed and key survives -> still locked). + heldUntil_.erase(key); + + std::string error; + const std::optional removed = + client_->run(client_->operations().deleteKey(key), error); + if (!removed) { + if (!error.empty()) { + backtest_log::error("TradeLocks: releasing " + key + " failed (" + + error + ")"); + } + return false; + } + // DEL on an already-expired key reports false; the lock is gone either + // way, so that is still a successful release. + return true; +} + +} // namespace redis_locks diff --git a/source/shared/redis/tradeLocks.hpp b/source/shared/redis/tradeLocks.hpp new file mode 100644 index 0000000..be88912 --- /dev/null +++ b/source/shared/redis/tradeLocks.hpp @@ -0,0 +1,85 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include +#include + +namespace redis_util { +class SyncRedisClient; +} + +// Redis-backed trade locks for live trading, mirroring the C# engine's +// TradeLocks: one lock per (strategy UUID, direction) with a TTL, acquired +// atomically via SET NX PX. The lock is the live entry gate — while it is +// held, the same strategy may not fire another order in the same direction, +// which also throttles strategies that signal on every tick (RandomStrategy) +// to one order per direction per TTL window. +// +// This header is Asio-free on purpose (same isolation pattern as +// redisConnection.hpp / udpReceiver.hpp): the connection machinery lives in +// redis_util::SyncRedisClient behind tradeLocks.cpp, so module global module +// fragments can #include this safely alongside `import std`. +namespace redis_locks { + +// "LOCK##" — key format shared with the C# engine's +// TradeLocks; a free function so tests can pin the format without a server. +std::string lockKey(const std::string& strategyUuid, + const std::string& direction); + +class TradeLocks { +public: + // Connects lazily on first use via the shared Boost.Redis connection + // (host from $REDIS_HOST in the caller; port 6379 by convention). + TradeLocks(const std::string& host, int port); + ~TradeLocks(); + TradeLocks(const TradeLocks&) = delete; + TradeLocks& operator=(const TradeLocks&) = delete; + + // SET key NX PX(ttl). Returns TRUE when a lock ALREADY existed (do not + // trade); acquires the lock and returns false otherwise. FAIL-CLOSED: any + // Redis failure (unreachable, timeout, dead connection) is logged and + // reported as "locked" — in live trading a missed entry is recoverable, + // a duplicate or ungated order is not, so uncertainty must block. + bool isThereATradeLock(const std::string& strategyUuid, + const std::string& direction, + std::chrono::seconds ttl = std::chrono::seconds{30}); + + // SET key PX(ttl), UNCONDITIONAL — restarts the lock's TTL from now, + // mirroring the C# TradeLocks extend (a plain SET with expiry overwrites + // whatever TTL remained). For the broker order channel: keep the lock + // alive while an order is in flight so it cannot lapse mid-placement. + // Returns false when Redis was unreachable — the lock then simply lapses + // at its previous TTL (logged; recoverable). + bool extendLock(const std::string& strategyUuid, + const std::string& direction, + std::chrono::seconds ttl = std::chrono::seconds{30}); + + // DEL key — drops the lock early (C# TradeLocks release semantics), so + // the strategy may re-enter before the TTL runs out, e.g. once the broker + // confirms the order was rejected. Returns false when Redis was + // unreachable — the lock then expires on its own TTL (logged; safe, just + // slower to reopen). + bool releaseLock(const std::string& strategyUuid, + const std::string& direction); + +private: + std::unique_ptr client_; + // Locks THIS instance acquired, mapped to the steady-clock instant their + // TTL is guaranteed to still cover — while a lock is known-held the check + // answers locally instead of paying a Redis round trip per signal. + std::map heldUntil_; + // Serialises concurrent callers sharing one instance (one connection, one + // synchronous pump). The live gate avoids the contention entirely by + // giving each worker thread its own TradeLocks (see RedisTradeGate). + std::mutex mutex_; +}; + +} // namespace redis_locks diff --git a/source/shared/tradingDefinitions.hpp b/source/shared/tradingDefinitions.hpp index 3a80223..8a04289 100644 --- a/source/shared/tradingDefinitions.hpp +++ b/source/shared/tradingDefinitions.hpp @@ -8,6 +8,7 @@ #include "shared/tradingDefinitions/variables/ohlcVariables.hpp" #include "shared/tradingDefinitions/variables/ohlcRsiVariables.hpp" #include "shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp" +#include "shared/tradingDefinitions/variables/fvgVariables.hpp" #include "shared/tradingDefinitions/variables/tradingVariables.hpp" #include "shared/tradingDefinitions/variables/strategyVariables.hpp" #include "shared/tradingDefinitions/strategyConfig.hpp" diff --git a/source/shared/tradingDefinitions/config/configuration.hpp b/source/shared/tradingDefinitions/config/configuration.hpp index e20e351..13e7c76 100644 --- a/source/shared/tradingDefinitions/config/configuration.hpp +++ b/source/shared/tradingDefinitions/config/configuration.hpp @@ -18,16 +18,25 @@ // nlohmann's ADL-based to_json/from_json do not resolve reliably across a module // import boundary. See source/.../*.cppm GMFs and the module-migration notes. namespace tradingDefinitions { -// The risk fields (STARTING_BALANCE, MAX_LOSS_PERCENT, MAX_OPEN_TRADES, -// REPORT_FAILURES) mirror RunConfiguration — see the comments there. +// The window fields (LAST_MONTHS, OFFSET_MONTHS), batch identity (BATCH, +// EXECUTION_TS) and risk/simulation fields (STARTING_BALANCE, +// MAX_LOSS_PERCENT, MAX_OPEN_TRADES, MAX_TRADES_PER_MINUTE, REPORT_FAILURES, +// PEAK_HOURS_ONLY, ENTRY_SLIPPAGE_TENTH_PIPS) mirror RunConfiguration — see +// the comments there. struct Configuration { std::string RUN_ID; std::string SYMBOLS; + std::string BATCH; + std::string EXECUTION_TS; int LAST_MONTHS = 0; + int OFFSET_MONTHS = 0; boost::decimal::decimal64_t STARTING_BALANCE{DEFAULT_STARTING_BALANCE}; boost::decimal::decimal64_t MAX_LOSS_PERCENT{0}; int MAX_OPEN_TRADES{0}; + int MAX_TRADES_PER_MINUTE{60}; bool REPORT_FAILURES{true}; + bool PEAK_HOURS_ONLY{false}; + int ENTRY_SLIPPAGE_TENTH_PIPS{0}; StrategyConfig STRATEGY; }; @@ -38,11 +47,17 @@ inline void to_json(nlohmann::json& j, const Configuration& c) { j = nlohmann::json{ {"RUN_ID", c.RUN_ID}, {"SYMBOLS", c.SYMBOLS}, + {"BATCH", c.BATCH}, + {"EXECUTION_TS", c.EXECUTION_TS}, {"LAST_MONTHS", c.LAST_MONTHS}, + {"OFFSET_MONTHS", c.OFFSET_MONTHS}, {"STARTING_BALANCE", c.STARTING_BALANCE}, {"MAX_LOSS_PERCENT", c.MAX_LOSS_PERCENT}, {"MAX_OPEN_TRADES", c.MAX_OPEN_TRADES}, + {"MAX_TRADES_PER_MINUTE", c.MAX_TRADES_PER_MINUTE}, {"REPORT_FAILURES", c.REPORT_FAILURES}, + {"PEAK_HOURS_ONLY", c.PEAK_HOURS_ONLY}, + {"ENTRY_SLIPPAGE_TENTH_PIPS", c.ENTRY_SLIPPAGE_TENTH_PIPS}, {"STRATEGY", c.STRATEGY}, }; } @@ -53,10 +68,18 @@ inline void from_json(const nlohmann::json& j, Configuration& c) { j.at("LAST_MONTHS").get_to(c.LAST_MONTHS); j.at("STRATEGY").get_to(c.STRATEGY); const Configuration defaults{}; + c.BATCH = j.value("BATCH", defaults.BATCH); + c.EXECUTION_TS = j.value("EXECUTION_TS", defaults.EXECUTION_TS); + c.OFFSET_MONTHS = j.value("OFFSET_MONTHS", defaults.OFFSET_MONTHS); c.STARTING_BALANCE = j.value("STARTING_BALANCE", defaults.STARTING_BALANCE); c.MAX_LOSS_PERCENT = j.value("MAX_LOSS_PERCENT", defaults.MAX_LOSS_PERCENT); c.MAX_OPEN_TRADES = j.value("MAX_OPEN_TRADES", defaults.MAX_OPEN_TRADES); + c.MAX_TRADES_PER_MINUTE = + j.value("MAX_TRADES_PER_MINUTE", defaults.MAX_TRADES_PER_MINUTE); c.REPORT_FAILURES = j.value("REPORT_FAILURES", defaults.REPORT_FAILURES); + c.PEAK_HOURS_ONLY = j.value("PEAK_HOURS_ONLY", defaults.PEAK_HOURS_ONLY); + c.ENTRY_SLIPPAGE_TENTH_PIPS = + j.value("ENTRY_SLIPPAGE_TENTH_PIPS", defaults.ENTRY_SLIPPAGE_TENTH_PIPS); } }; diff --git a/source/shared/tradingDefinitions/config/runConfiguration.hpp b/source/shared/tradingDefinitions/config/runConfiguration.hpp index 0610d4b..31ab575 100644 --- a/source/shared/tradingDefinitions/config/runConfiguration.hpp +++ b/source/shared/tradingDefinitions/config/runConfiguration.hpp @@ -29,17 +29,51 @@ inline constexpr boost::decimal::decimal64_t DEFAULT_STARTING_BALANCE{10000}; // equity loss reaches MAX_LOSS_PERCENT of it. <= 0 disables the cutoff. // - MAX_OPEN_TRADES: cap on simultaneously open positions per run; entries // are skipped while at the cap. <= 0 means unlimited. +// - MAX_TRADES_PER_MINUTE: cap on trade entries within any sliding 60-second +// window of TICK time (backtests replay history, so wall clock would be +// meaningless); entries are skipped while at the cap. <= 0 means +// unlimited. Defaults to 60 — unlike the other risk knobs this one is ON +// by default, as a runaway-strategy brake. // - REPORT_FAILURES: when false, runs cut off by the loss limit are NOT // reported to Elasticsearch (silencer for large sweeps where liquidated // runs are expected noise). Completed runs always report. +// - PEAK_HOURS_ONLY: when true, entries are allowed only inside the +// symbol's peak session window (market_hours::tradePermitted — weekend +// block plus per-session hours). Exits are never gated. Defaults to +// false so payloads and winners written before the field existed keep +// their behaviour. +// - ENTRY_SLIPPAGE_TENTH_PIPS: slippage stress toggle, in TENTHS of a pip +// (3 = 0.3 pip). Every backtest entry fills that much AGAINST the trade +// (LONG above the ask, SHORT below the bid); SL/TP anchors stay on the +// raw tick. 0 (the default) is off, so existing payloads and unstressed +// sweeps are unchanged. Backtest-only — live fills belong to the broker. +// +// The tick window is LAST_MONTHS long and ends OFFSET_MONTHS before now, so +// OFFSET_MONTHS = 0 (the default) tests the most recent data while e.g. +// LAST_MONTHS = 6, OFFSET_MONTHS = 12 replays the 6-month window that ended a +// year ago — rolling historical windows without touching the data. +// +// BATCH / EXECUTION_TS are the batch identity minted once per load (see +// outcomeIndices.hpp): BATCH is the ISO week label ("2026-28") that names the +// weekly Elasticsearch outcome indices, EXECUTION_TS the seed wall clock the +// whole batch shares. They ride through Redis and every rolling-window rung +// unchanged so a run's documents always target the seed week's index. Empty +// (payloads written before the fields existed, hand-run configs) means +// unsuffixed index names and no batch metadata on the documents. struct RunConfiguration { std::string RUN_ID; std::string SYMBOLS; + std::string BATCH; + std::string EXECUTION_TS; int LAST_MONTHS = 0; + int OFFSET_MONTHS = 0; boost::decimal::decimal64_t STARTING_BALANCE{DEFAULT_STARTING_BALANCE}; boost::decimal::decimal64_t MAX_LOSS_PERCENT{0}; int MAX_OPEN_TRADES{0}; + int MAX_TRADES_PER_MINUTE{60}; bool REPORT_FAILURES{true}; + bool PEAK_HOURS_ONLY{false}; + int ENTRY_SLIPPAGE_TENTH_PIPS{0}; }; // Hand-written (rather than NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE) so the @@ -49,11 +83,17 @@ inline void to_json(nlohmann::json& j, const RunConfiguration& c) { j = nlohmann::json{ {"RUN_ID", c.RUN_ID}, {"SYMBOLS", c.SYMBOLS}, + {"BATCH", c.BATCH}, + {"EXECUTION_TS", c.EXECUTION_TS}, {"LAST_MONTHS", c.LAST_MONTHS}, + {"OFFSET_MONTHS", c.OFFSET_MONTHS}, {"STARTING_BALANCE", c.STARTING_BALANCE}, {"MAX_LOSS_PERCENT", c.MAX_LOSS_PERCENT}, {"MAX_OPEN_TRADES", c.MAX_OPEN_TRADES}, + {"MAX_TRADES_PER_MINUTE", c.MAX_TRADES_PER_MINUTE}, {"REPORT_FAILURES", c.REPORT_FAILURES}, + {"PEAK_HOURS_ONLY", c.PEAK_HOURS_ONLY}, + {"ENTRY_SLIPPAGE_TENTH_PIPS", c.ENTRY_SLIPPAGE_TENTH_PIPS}, }; } @@ -62,10 +102,18 @@ inline void from_json(const nlohmann::json& j, RunConfiguration& c) { j.at("SYMBOLS").get_to(c.SYMBOLS); j.at("LAST_MONTHS").get_to(c.LAST_MONTHS); const RunConfiguration defaults{}; + c.BATCH = j.value("BATCH", defaults.BATCH); + c.EXECUTION_TS = j.value("EXECUTION_TS", defaults.EXECUTION_TS); + c.OFFSET_MONTHS = j.value("OFFSET_MONTHS", defaults.OFFSET_MONTHS); c.STARTING_BALANCE = j.value("STARTING_BALANCE", defaults.STARTING_BALANCE); c.MAX_LOSS_PERCENT = j.value("MAX_LOSS_PERCENT", defaults.MAX_LOSS_PERCENT); c.MAX_OPEN_TRADES = j.value("MAX_OPEN_TRADES", defaults.MAX_OPEN_TRADES); + c.MAX_TRADES_PER_MINUTE = + j.value("MAX_TRADES_PER_MINUTE", defaults.MAX_TRADES_PER_MINUTE); c.REPORT_FAILURES = j.value("REPORT_FAILURES", defaults.REPORT_FAILURES); + c.PEAK_HOURS_ONLY = j.value("PEAK_HOURS_ONLY", defaults.PEAK_HOURS_ONLY); + c.ENTRY_SLIPPAGE_TENTH_PIPS = + j.value("ENTRY_SLIPPAGE_TENTH_PIPS", defaults.ENTRY_SLIPPAGE_TENTH_PIPS); } }; diff --git a/source/shared/tradingDefinitions/strategyConfig.hpp b/source/shared/tradingDefinitions/strategyConfig.hpp index 5f9263f..072c7bd 100644 --- a/source/shared/tradingDefinitions/strategyConfig.hpp +++ b/source/shared/tradingDefinitions/strategyConfig.hpp @@ -10,6 +10,7 @@ #include #include "shared/tradingDefinitions/variables/tradingVariables.hpp" #include "shared/tradingDefinitions/variables/ohlcVariables.hpp" +#include "shared/tradingDefinitions/variables/rangeBarVariables.hpp" #include "shared/tradingDefinitions/variables/strategyVariables.hpp" namespace tradingDefinitions { @@ -18,12 +19,32 @@ struct StrategyConfig { std::string UUID; TradingVariables TRADING_VARIABLES; std::vector OHLC_VARIABLES; + std::vector RANGE_VARIABLES; StrategyVariables STRATEGY_VARIABLES; }; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(StrategyConfig, - UUID, - TRADING_VARIABLES, - OHLC_VARIABLES, - STRATEGY_VARIABLES -); -} + +// Hand-written (rather than NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE) so the +// original fields stay strictly required while RANGE_VARIABLES falls back to +// empty — every ES winner document, queued Redis payload and test fixture +// written before range bars existed still parses (the macro's j.at would +// reject them all, and liveWinners' per-hit guard would then silently drop +// every winner). Same doctrine as RunConfiguration's serializers. +inline void to_json(nlohmann::json& j, const StrategyConfig& c) { + j = nlohmann::json{ + {"UUID", c.UUID}, + {"TRADING_VARIABLES", c.TRADING_VARIABLES}, + {"OHLC_VARIABLES", c.OHLC_VARIABLES}, + {"RANGE_VARIABLES", c.RANGE_VARIABLES}, + {"STRATEGY_VARIABLES", c.STRATEGY_VARIABLES}, + }; +} + +inline void from_json(const nlohmann::json& j, StrategyConfig& c) { + j.at("UUID").get_to(c.UUID); + j.at("TRADING_VARIABLES").get_to(c.TRADING_VARIABLES); + j.at("OHLC_VARIABLES").get_to(c.OHLC_VARIABLES); + c.RANGE_VARIABLES = + j.value("RANGE_VARIABLES", std::vector{}); + j.at("STRATEGY_VARIABLES").get_to(c.STRATEGY_VARIABLES); +} +} diff --git a/source/shared/tradingDefinitions/variables/fvgVariables.hpp b/source/shared/tradingDefinitions/variables/fvgVariables.hpp new file mode 100644 index 0000000..3a086e3 --- /dev/null +++ b/source/shared/tradingDefinitions/variables/fvgVariables.hpp @@ -0,0 +1,43 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +namespace tradingDefinitions { +// Parameters for the Fair Value Gap strategy. LOOKBACK_BARS bounds how far +// back the closed-bar scan may reach on the FVG timeframe. MIN_GAP_PIPS is +// the smallest tradeable gap in PIPS — the strategy converts it to points per +// symbol via symbol_scale::get, so one value means the same real-world size +// on every symbol scale (renamed from MIN_GAP_POINTS, which was raw sub-pip +// points; old configs carrying the points field parse to the 0 default and +// the ctor rejects them loudly). HTF_SMA_PERIOD is the SMA length over the +// higher timeframe's closed closes used as the trend filter. +// MIN_GAP_AGE_BARS behaves as a MAXIMUM age despite its name (preserved from +// the C# original): 0 = only the newest closed 3-bar pattern is examined, +// larger values widen the scan backward, clamped by LOOKBACK_BARS. +// MAX_TRADE_DURATION_MINUTES is the during() time cap (strategy_exits:: +// closeIfPastCap): a trade open strictly longer closes at the exit-side +// price; <= 0 disables — and the 0 default is what winner configs persisted +// before the field existed parse to (uncapped, their original behaviour). +// WITH_DEFAULT so winner configs persisted before a field existed parse with +// the in-class defaults instead of throwing; the strategy ctor rejects the +// zero defaults loudly, so an absent required field still fails fast — at +// construction, not at parse. +struct FVGStrategyVariables { + int LOOKBACK_BARS = 0; + int MIN_GAP_PIPS = 0; + int HTF_SMA_PERIOD = 0; + int MIN_GAP_AGE_BARS = 0; + int MAX_TRADE_DURATION_MINUTES = 0; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(FVGStrategyVariables, + LOOKBACK_BARS, + MIN_GAP_PIPS, + HTF_SMA_PERIOD, + MIN_GAP_AGE_BARS, + MAX_TRADE_DURATION_MINUTES); +} diff --git a/source/shared/tradingDefinitions/variables/keltnerFadeVariables.hpp b/source/shared/tradingDefinitions/variables/keltnerFadeVariables.hpp new file mode 100644 index 0000000..e28f395 --- /dev/null +++ b/source/shared/tradingDefinitions/variables/keltnerFadeVariables.hpp @@ -0,0 +1,34 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +namespace tradingDefinitions { +// Parameters for the Keltner-band mean-reversion strategy. BAND_SMA_PERIOD is +// the lookback over the signal timeframe's CLOSED closes for both the band +// centre (SMA) and its width (ATR over the same closed bars). The band +// half-width is BAND_ATR_MULT_TENTHS x ATR / 10 — tenths so the sweep can +// express fractional multipliers (15 = 1.5x ATR) while the engine stays in +// integer arithmetic. MAX_TRADE_DURATION_MINUTES caps a trade's lifetime +// exactly like the session breakouts': during() closes the symbol's trade +// once it has been open STRICTLY longer than this; <= 0 disables the cap. +// For a fade the cap is the thesis clock — a stretch that has not snapped +// back within the window is a failed reversion, not a position to sit in. +// WITH_DEFAULT so winner configs persisted before a field existed parse with +// the in-class defaults instead of throwing; the strategy ctor rejects the +// zero band defaults loudly, so an absent required field still fails fast — +// at construction, not at parse (an absent cap is simply disabled). +struct KeltnerFadeVariables { + int BAND_SMA_PERIOD = 0; + int BAND_ATR_MULT_TENTHS = 0; + int MAX_TRADE_DURATION_MINUTES = 0; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(KeltnerFadeVariables, + BAND_SMA_PERIOD, + BAND_ATR_MULT_TENTHS, + MAX_TRADE_DURATION_MINUTES); +} diff --git a/source/shared/tradingDefinitions/variables/liquiditySweepReversalVariables.hpp b/source/shared/tradingDefinitions/variables/liquiditySweepReversalVariables.hpp new file mode 100644 index 0000000..3c26d64 --- /dev/null +++ b/source/shared/tradingDefinitions/variables/liquiditySweepReversalVariables.hpp @@ -0,0 +1,46 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +namespace tradingDefinitions { +// Parameters for the liquidity sweep reversal strategy. PIVOT_BARS is the +// fractal wing: a closed bar is a swing high/low when its extreme STRICTLY +// beats the PIVOT_BARS closed bars on each side (ties disqualify), so a pivot +// only exists once its right wing has fully closed — no lookahead. +// LOOKBACK_BARS bounds how far back pivots are scanned. MIN_SWEEP_PIPS is the +// minimum wick excursion beyond a swept level (converted to integer points +// via symbol_scale::get) for the touch to count as a sweep rather than a +// shallow tap; 0 accepts any strict poke. DISPLACEMENT_ATR_TENTHS demands the +// rejection bar's body be at least tenths x ATR(10) / 10 against the sweep — +// 0 disables the gate, so the sweep itself A/Bs whether displacement carries +// signal. VALID_BARS is the rejection's freshness window (the squeeze +// breakout idiom): a setup older than this many closed bars leaves the scan. +// MAX_TRADE_DURATION_MINUTES is the during() time cap (strategy_exits:: +// closeIfPastCap): a trade open strictly longer closes at the exit-side +// price; <= 0 disables — and the 0 default is what winner configs persisted +// before the field existed parse to (uncapped, their original behaviour). +// WITH_DEFAULT so winner configs persisted before a field existed parse with +// the in-class defaults instead of throwing; the strategy ctor rejects zero +// PIVOT_BARS / LOOKBACK_BARS / VALID_BARS loudly, so an absent required field +// still fails fast — at construction, not at parse. +struct LiquiditySweepReversalVariables { + int PIVOT_BARS = 0; + int LOOKBACK_BARS = 0; + int MIN_SWEEP_PIPS = 0; + int DISPLACEMENT_ATR_TENTHS = 0; + int VALID_BARS = 0; + int MAX_TRADE_DURATION_MINUTES = 0; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(LiquiditySweepReversalVariables, + PIVOT_BARS, + LOOKBACK_BARS, + MIN_SWEEP_PIPS, + DISPLACEMENT_ATR_TENTHS, + VALID_BARS, + MAX_TRADE_DURATION_MINUTES); +} diff --git a/source/shared/tradingDefinitions/variables/nyOpenRangeBreakoutVariables.hpp b/source/shared/tradingDefinitions/variables/nyOpenRangeBreakoutVariables.hpp new file mode 100644 index 0000000..ba6be8c --- /dev/null +++ b/source/shared/tradingDefinitions/variables/nyOpenRangeBreakoutVariables.hpp @@ -0,0 +1,38 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +namespace tradingDefinitions { +// Parameters for the New York open-range breakout strategy. RANGE_HOURS is +// the depth of the pre-open range, in whole hours counted back from the NY +// equities open (13:30 UTC under EDT, else 14:30) — 13 approximates the full +// overnight session, 4 a tight pre-open coil; it is swept rather than +// hardcoded so the data decides which range definition carries the edge. +// BUFFER_PIPS pads the range high/low breakout levels (converted to integer +// points via symbol_scale::get) — a noise filter against marginal pokes. +// ENTRY_WINDOW_MINUTES bounds how long after the open entries may fire; the +// NewYork peak-hours session is three hours from the open, so values above +// 180 buy nothing when PEAK_HOURS_ONLY is on. MAX_TRADE_DURATION_MINUTES caps +// a trade's lifetime exactly like the session range breakout's: during() +// closes the symbol's trade once it has been open STRICTLY longer than this; +// <= 0 disables the cap. WITH_DEFAULT so winner configs persisted before a +// field existed parse with the in-class defaults instead of throwing; the +// strategy ctor rejects zero RANGE_HOURS / ENTRY_WINDOW_MINUTES loudly, so an +// absent required field still fails fast — at construction, not at parse. +struct NyOpenRangeBreakoutVariables { + int RANGE_HOURS = 0; + int BUFFER_PIPS = 0; + int ENTRY_WINDOW_MINUTES = 0; + int MAX_TRADE_DURATION_MINUTES = 0; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(NyOpenRangeBreakoutVariables, + RANGE_HOURS, + BUFFER_PIPS, + ENTRY_WINDOW_MINUTES, + MAX_TRADE_DURATION_MINUTES); +} diff --git a/source/shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp b/source/shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp index 6302567..d7ad76f 100644 --- a/source/shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp +++ b/source/shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp @@ -12,8 +12,16 @@ namespace tradingDefinitions { // closed-candle high/low breakout levels: price must clear the level by this // many pips (converted to integer points via symbol_scale::get) before a // signal fires — a noise filter against marginal pokes through the range. +// MAX_TRADE_DURATION_MINUTES caps a trade's lifetime: during() closes the +// symbol's trade once it has been open STRICTLY longer than this; <= 0 +// disables the cap. WITH_DEFAULT so winner configs persisted before the +// field existed parse with the in-class defaults (absent = 0 = disabled) +// instead of throwing — which also makes BUFFER_PIPS tolerant of absence. struct OHLCBreakoutVariables { int BUFFER_PIPS = 0; + int MAX_TRADE_DURATION_MINUTES = 0; }; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(OHLCBreakoutVariables, BUFFER_PIPS); +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(OHLCBreakoutVariables, + BUFFER_PIPS, + MAX_TRADE_DURATION_MINUTES); } diff --git a/source/shared/tradingDefinitions/variables/ohlcVariables.hpp b/source/shared/tradingDefinitions/variables/ohlcVariables.hpp index 2593035..7f2564f 100644 --- a/source/shared/tradingDefinitions/variables/ohlcVariables.hpp +++ b/source/shared/tradingDefinitions/variables/ohlcVariables.hpp @@ -5,12 +5,37 @@ // --------------------------------------- #pragma once +#include + #include namespace tradingDefinitions { +// One (bar count, bar minutes) pair for a strategy's OHLC feed. {0, 0} is the +// "unused" sentinel carried by strategies that build no bars (RandomStrategy); +// a strategy that actually builds bars must have both >= 1 — calculateOHLC +// throws on a non-positive duration, since a zero-minute bar would otherwise +// roll a new bar on every tick and grow without bound. struct OHLCVariables { - int OHLC_COUNT; - int OHLC_MINUTES; + int OHLC_COUNT = 0; + int OHLC_MINUTES = 0; }; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(OHLCVariables, OHLC_COUNT, OHLC_MINUTES); -} + +inline void to_json(nlohmann::json& j, const OHLCVariables& v) { + j = nlohmann::json{ + {"OHLC_COUNT", v.OHLC_COUNT}, + {"OHLC_MINUTES", v.OHLC_MINUTES}, + }; +} + +inline void from_json(const nlohmann::json& j, OHLCVariables& v) { + j.at("OHLC_COUNT").get_to(v.OHLC_COUNT); + j.at("OHLC_MINUTES").get_to(v.OHLC_MINUTES); + // Zero is the documented "unused" sentinel; anything negative is garbage + // that should fail the parse (a poison-pill payload) rather than reach the + // bar builder. + if (v.OHLC_COUNT < 0 || v.OHLC_MINUTES < 0) { + throw std::invalid_argument( + "OHLCVariables: OHLC_COUNT/OHLC_MINUTES must be >= 0, got " + j.dump()); + } +} +} diff --git a/source/shared/tradingDefinitions/variables/rangeBarVariables.hpp b/source/shared/tradingDefinitions/variables/rangeBarVariables.hpp new file mode 100644 index 0000000..e56a3db --- /dev/null +++ b/source/shared/tradingDefinitions/variables/rangeBarVariables.hpp @@ -0,0 +1,50 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +#include + +namespace tradingDefinitions { +// One range-bar series spec for a strategy's feed. A range bar completes when +// price travels a threshold distance; the threshold is a percentage of the +// rolling high-low range over the last RANGE_ATR_TICK_WINDOW ticks — a pure +// tick-count measure with no clock dependency, so it adapts to a volatility +// spike on the tick it happens rather than when a time bucket rolls (see +// rangeBarBuilder). All-zeros is the "unused" sentinel mirroring +// OHLCVariables' {0, 0}; a strategy that actually builds range bars must have +// every field >= 1 — RangeSeries throws on anything less, since a zero tick +// window can never warm and a zero threshold percent would roll a bar per +// tick. +struct RangeBarVariables { + int RANGE_ATR_TICK_WINDOW = 0; // rolling tick window for the range measure + int RANGE_ATR_PERCENT = 0; // threshold = windowRange * pct / 100 + int RANGE_COUNT = 0; // bars kept AND the prepopulate depth +}; + +inline void to_json(nlohmann::json& j, const RangeBarVariables& v) { + j = nlohmann::json{ + {"RANGE_ATR_TICK_WINDOW", v.RANGE_ATR_TICK_WINDOW}, + {"RANGE_ATR_PERCENT", v.RANGE_ATR_PERCENT}, + {"RANGE_COUNT", v.RANGE_COUNT}, + }; +} + +inline void from_json(const nlohmann::json& j, RangeBarVariables& v) { + j.at("RANGE_ATR_TICK_WINDOW").get_to(v.RANGE_ATR_TICK_WINDOW); + j.at("RANGE_ATR_PERCENT").get_to(v.RANGE_ATR_PERCENT); + j.at("RANGE_COUNT").get_to(v.RANGE_COUNT); + // Zero is the documented "unused" sentinel; anything negative is garbage + // that should fail the parse (a poison-pill payload) rather than reach the + // bar builder — same doctrine as OHLCVariables. + if (v.RANGE_ATR_TICK_WINDOW < 0 || v.RANGE_ATR_PERCENT < 0 || + v.RANGE_COUNT < 0) { + throw std::invalid_argument( + "RangeBarVariables: all fields must be >= 0, got " + j.dump()); + } +} +} diff --git a/source/shared/tradingDefinitions/variables/rangeVelocityVariables.hpp b/source/shared/tradingDefinitions/variables/rangeVelocityVariables.hpp new file mode 100644 index 0000000..944cf01 --- /dev/null +++ b/source/shared/tradingDefinitions/variables/rangeVelocityVariables.hpp @@ -0,0 +1,41 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +namespace tradingDefinitions { +// Parameters for the range-bar velocity momentum strategy. Range bars all +// cover ~equal price travel (see rangeBarBuilder), so the time each bar took +// to form IS a momentum reading — the clock as a signal, never as a sampling +// basis. RUN_BARS is K: the number of consecutive same-direction closed range +// bars required. SPEED_LOOKBACK_BARS is M: the bars immediately preceding the +// run whose median formation duration is the speed baseline (median, not +// mean, so weekend-gap bars don't poison it). SPEED_RATIO_PERCENT gates each +// run bar: it passes when duration x 100 <= median x this — 100 means "at the +// norm", below 100 demands genuinely faster-than-normal formation. +// EXIT_RUN_BARS is E: a run of this many closed bars AGAINST the open +// position closes it from during() (no speed filter — momentum dying is +// enough). MAX_TRADE_DURATION_MINUTES is the time cap; 0 disables (the +// OhlcBreakout idiom). +// WITH_DEFAULT so winner configs persisted before a field existed parse with +// the in-class defaults instead of throwing; the strategy ctor rejects zeros +// loudly, so an absent required field still fails fast — at construction, not +// at parse. +struct RangeVelocityVariables { + int RUN_BARS = 0; + int SPEED_LOOKBACK_BARS = 0; + int SPEED_RATIO_PERCENT = 0; + int EXIT_RUN_BARS = 0; + int MAX_TRADE_DURATION_MINUTES = 0; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(RangeVelocityVariables, + RUN_BARS, + SPEED_LOOKBACK_BARS, + SPEED_RATIO_PERCENT, + EXIT_RUN_BARS, + MAX_TRADE_DURATION_MINUTES); +} diff --git a/source/shared/tradingDefinitions/variables/sessionRangeBreakoutVariables.hpp b/source/shared/tradingDefinitions/variables/sessionRangeBreakoutVariables.hpp new file mode 100644 index 0000000..0a19eae --- /dev/null +++ b/source/shared/tradingDefinitions/variables/sessionRangeBreakoutVariables.hpp @@ -0,0 +1,33 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +namespace tradingDefinitions { +// Parameters for the session open-range breakout strategy. BUFFER_PIPS pads +// the Asian-session high/low breakout levels (converted to integer points via +// symbol_scale::get) — a noise filter against marginal pokes through the +// range. ENTRY_WINDOW_MINUTES bounds how long after the London open entries +// may fire: the open-range edge decays through the session, so the window is +// a first-class swept parameter. MAX_TRADE_DURATION_MINUTES caps a trade's +// lifetime exactly like the OHLC breakout's: during() closes the symbol's +// trade once it has been open STRICTLY longer than this; <= 0 disables the +// cap (the session strategy uses it to avoid riding a London entry into New +// York chop). WITH_DEFAULT so winner configs persisted before a field existed +// parse with the in-class defaults instead of throwing; the strategy ctor +// rejects a zero ENTRY_WINDOW_MINUTES loudly, so an absent required field +// still fails fast — at construction, not at parse. +struct SessionRangeBreakoutVariables { + int BUFFER_PIPS = 0; + int ENTRY_WINDOW_MINUTES = 0; + int MAX_TRADE_DURATION_MINUTES = 0; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(SessionRangeBreakoutVariables, + BUFFER_PIPS, + ENTRY_WINDOW_MINUTES, + MAX_TRADE_DURATION_MINUTES); +} diff --git a/source/shared/tradingDefinitions/variables/squeezeBreakoutVariables.hpp b/source/shared/tradingDefinitions/variables/squeezeBreakoutVariables.hpp new file mode 100644 index 0000000..76b9bb7 --- /dev/null +++ b/source/shared/tradingDefinitions/variables/squeezeBreakoutVariables.hpp @@ -0,0 +1,40 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +namespace tradingDefinitions { +// Parameters for the volatility-squeeze breakout strategy. NR_LOOKBACK picks +// the contraction pattern: 0 = inside bar (the newest closed bar's range sits +// entirely within its predecessor's), N >= 2 = NR-N (the bar's high-low range +// is STRICTLY the narrowest of the last N closed bars). 1 is rejected by the +// strategy ctor — "narrowest of the last one" matches every bar. VALID_BARS +// is how many closed bars back a matched pattern may sit and still supply +// breakout levels (1 = the newest closed bar only); the pattern's own +// high/low are the levels. BUFFER_PIPS pads those levels (converted to +// integer points via symbol_scale::get) — a noise filter against marginal +// pokes. MAX_TRADE_DURATION_MINUTES is the during() time cap +// (strategy_exits::closeIfPastCap): a trade open strictly longer closes at +// the exit-side price; <= 0 disables — and the 0 default is what winner +// configs persisted before the field existed parse to (uncapped, their +// original behaviour). WITH_DEFAULT so winner configs persisted before a +// field existed parse with the in-class defaults instead of throwing; +// NR_LOOKBACK's 0 default is the (valid) inside-bar mode, and the ctor +// rejects the zero VALID_BARS loudly, so a wholly absent group still fails +// fast — at construction, not at parse. +struct SqueezeBreakoutVariables { + int NR_LOOKBACK = 0; + int VALID_BARS = 0; + int BUFFER_PIPS = 0; + int MAX_TRADE_DURATION_MINUTES = 0; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(SqueezeBreakoutVariables, + NR_LOOKBACK, + VALID_BARS, + BUFFER_PIPS, + MAX_TRADE_DURATION_MINUTES); +} diff --git a/source/shared/tradingDefinitions/variables/strategyVariables.cpp b/source/shared/tradingDefinitions/variables/strategyVariables.cpp index 9780e60..0aaa38b 100644 --- a/source/shared/tradingDefinitions/variables/strategyVariables.cpp +++ b/source/shared/tradingDefinitions/variables/strategyVariables.cpp @@ -18,6 +18,41 @@ void to_json(nlohmann::json& j, const StrategyVariables& s) { } else { j["OHLC_BREAKOUT_VARIABLES"] = nullptr; } + if (s.FVG_STRATEGY_VARIABLES) { + j["FVG_STRATEGY_VARIABLES"] = *s.FVG_STRATEGY_VARIABLES; + } else { + j["FVG_STRATEGY_VARIABLES"] = nullptr; + } + if (s.KELTNER_FADE_VARIABLES) { + j["KELTNER_FADE_VARIABLES"] = *s.KELTNER_FADE_VARIABLES; + } else { + j["KELTNER_FADE_VARIABLES"] = nullptr; + } + if (s.SESSION_RANGE_BREAKOUT_VARIABLES) { + j["SESSION_RANGE_BREAKOUT_VARIABLES"] = *s.SESSION_RANGE_BREAKOUT_VARIABLES; + } else { + j["SESSION_RANGE_BREAKOUT_VARIABLES"] = nullptr; + } + if (s.SQUEEZE_BREAKOUT_VARIABLES) { + j["SQUEEZE_BREAKOUT_VARIABLES"] = *s.SQUEEZE_BREAKOUT_VARIABLES; + } else { + j["SQUEEZE_BREAKOUT_VARIABLES"] = nullptr; + } + if (s.NY_OPEN_RANGE_BREAKOUT_VARIABLES) { + j["NY_OPEN_RANGE_BREAKOUT_VARIABLES"] = *s.NY_OPEN_RANGE_BREAKOUT_VARIABLES; + } else { + j["NY_OPEN_RANGE_BREAKOUT_VARIABLES"] = nullptr; + } + if (s.LIQUIDITY_SWEEP_REVERSAL_VARIABLES) { + j["LIQUIDITY_SWEEP_REVERSAL_VARIABLES"] = *s.LIQUIDITY_SWEEP_REVERSAL_VARIABLES; + } else { + j["LIQUIDITY_SWEEP_REVERSAL_VARIABLES"] = nullptr; + } + if (s.RANGE_VELOCITY_VARIABLES) { + j["RANGE_VELOCITY_VARIABLES"] = *s.RANGE_VELOCITY_VARIABLES; + } else { + j["RANGE_VELOCITY_VARIABLES"] = nullptr; + } } void from_json(const nlohmann::json& j, StrategyVariables& s) { @@ -31,6 +66,51 @@ void from_json(const nlohmann::json& j, StrategyVariables& s) { } else { s.OHLC_BREAKOUT_VARIABLES = std::nullopt; } + if (j.contains("FVG_STRATEGY_VARIABLES") && !j.at("FVG_STRATEGY_VARIABLES").is_null()) { + s.FVG_STRATEGY_VARIABLES = j.at("FVG_STRATEGY_VARIABLES").get(); + } else { + s.FVG_STRATEGY_VARIABLES = std::nullopt; + } + if (j.contains("KELTNER_FADE_VARIABLES") && !j.at("KELTNER_FADE_VARIABLES").is_null()) { + s.KELTNER_FADE_VARIABLES = j.at("KELTNER_FADE_VARIABLES").get(); + } else { + s.KELTNER_FADE_VARIABLES = std::nullopt; + } + if (j.contains("SESSION_RANGE_BREAKOUT_VARIABLES") && + !j.at("SESSION_RANGE_BREAKOUT_VARIABLES").is_null()) { + s.SESSION_RANGE_BREAKOUT_VARIABLES = + j.at("SESSION_RANGE_BREAKOUT_VARIABLES").get(); + } else { + s.SESSION_RANGE_BREAKOUT_VARIABLES = std::nullopt; + } + if (j.contains("SQUEEZE_BREAKOUT_VARIABLES") && + !j.at("SQUEEZE_BREAKOUT_VARIABLES").is_null()) { + s.SQUEEZE_BREAKOUT_VARIABLES = + j.at("SQUEEZE_BREAKOUT_VARIABLES").get(); + } else { + s.SQUEEZE_BREAKOUT_VARIABLES = std::nullopt; + } + if (j.contains("NY_OPEN_RANGE_BREAKOUT_VARIABLES") && + !j.at("NY_OPEN_RANGE_BREAKOUT_VARIABLES").is_null()) { + s.NY_OPEN_RANGE_BREAKOUT_VARIABLES = + j.at("NY_OPEN_RANGE_BREAKOUT_VARIABLES").get(); + } else { + s.NY_OPEN_RANGE_BREAKOUT_VARIABLES = std::nullopt; + } + if (j.contains("LIQUIDITY_SWEEP_REVERSAL_VARIABLES") && + !j.at("LIQUIDITY_SWEEP_REVERSAL_VARIABLES").is_null()) { + s.LIQUIDITY_SWEEP_REVERSAL_VARIABLES = + j.at("LIQUIDITY_SWEEP_REVERSAL_VARIABLES").get(); + } else { + s.LIQUIDITY_SWEEP_REVERSAL_VARIABLES = std::nullopt; + } + if (j.contains("RANGE_VELOCITY_VARIABLES") && + !j.at("RANGE_VELOCITY_VARIABLES").is_null()) { + s.RANGE_VELOCITY_VARIABLES = + j.at("RANGE_VELOCITY_VARIABLES").get(); + } else { + s.RANGE_VELOCITY_VARIABLES = std::nullopt; + } } } diff --git a/source/shared/tradingDefinitions/variables/strategyVariables.hpp b/source/shared/tradingDefinitions/variables/strategyVariables.hpp index cc655ba..6aa5478 100644 --- a/source/shared/tradingDefinitions/variables/strategyVariables.hpp +++ b/source/shared/tradingDefinitions/variables/strategyVariables.hpp @@ -9,6 +9,13 @@ #include #include "shared/tradingDefinitions/variables/ohlcRsiVariables.hpp" #include "shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp" +#include "shared/tradingDefinitions/variables/fvgVariables.hpp" +#include "shared/tradingDefinitions/variables/keltnerFadeVariables.hpp" +#include "shared/tradingDefinitions/variables/sessionRangeBreakoutVariables.hpp" +#include "shared/tradingDefinitions/variables/squeezeBreakoutVariables.hpp" +#include "shared/tradingDefinitions/variables/nyOpenRangeBreakoutVariables.hpp" +#include "shared/tradingDefinitions/variables/liquiditySweepReversalVariables.hpp" +#include "shared/tradingDefinitions/variables/rangeVelocityVariables.hpp" namespace tradingDefinitions { @@ -16,6 +23,13 @@ namespace tradingDefinitions { struct StrategyVariables { std::optional OHLC_RSI_VARIABLES; std::optional OHLC_BREAKOUT_VARIABLES; + std::optional FVG_STRATEGY_VARIABLES; + std::optional KELTNER_FADE_VARIABLES; + std::optional SESSION_RANGE_BREAKOUT_VARIABLES; + std::optional SQUEEZE_BREAKOUT_VARIABLES; + std::optional NY_OPEN_RANGE_BREAKOUT_VARIABLES; + std::optional LIQUIDITY_SWEEP_REVERSAL_VARIABLES; + std::optional RANGE_VELOCITY_VARIABLES; }; void to_json(nlohmann::json& j, const StrategyVariables& s); diff --git a/source/shared/tradingDefinitions/variables/tradingVariables.hpp b/source/shared/tradingDefinitions/variables/tradingVariables.hpp index 7691dc5..1cdcd71 100644 --- a/source/shared/tradingDefinitions/variables/tradingVariables.hpp +++ b/source/shared/tradingDefinitions/variables/tradingVariables.hpp @@ -7,26 +7,40 @@ #pragma once #include #include +#include +#include #include #include namespace tradingDefinitions { struct TradingVariables { std::string STRATEGY; - // SL/TP distances in whole pips; TRADING_SIZE in whole lots. All integer — - // the engine converts pips->points and applies size inside its integer - // hot loop (see symbolScale.hpp / tradeManager). - int32_t STOP_DISTANCE_IN_PIPS = 0; - int32_t LIMIT_DISTANCE_IN_PIPS = 0; + // SL/TP distances as whole ATR multipliers (distance = ATR x multiplier, + // computed per entry by conditions::check, which also clamps the result + // to pip bounds); TRADING_SIZE in whole lots. All integer — the engine + // converts pips->points and applies size inside its integer hot loop + // (see symbolScale.hpp / tradeManager). + int32_t STOP_DISTANCE_IN_ATR = 0; + int32_t LIMIT_DISTANCE_IN_ATR = 0; int32_t TRADING_SIZE = 0; }; // Read an integer field that may arrive as a JSON number or, per this codebase's // string-encoded-numeric convention, as a string. lround matches // sweep::Combination::getInt and tolerates any legacy decimal value. +// Non-finite values ("nan"/"inf" — UB in lround) and values outside int32 are +// rejected: the old unchecked cast silently wrapped e.g. "99999999999" into a +// garbage (possibly negative) pip distance or size, producing wrong trades +// instead of a rejected config. inline int32_t readIntField(const nlohmann::json& j) { const double value = j.is_string() ? std::stod(j.get()) : j.get(); + if (!std::isfinite(value) || + value < static_cast(std::numeric_limits::min()) || + value > static_cast(std::numeric_limits::max())) { + throw std::invalid_argument( + "TradingVariables: integer field out of range: " + j.dump()); + } return static_cast(std::lround(value)); } @@ -36,16 +50,20 @@ inline int32_t readIntField(const nlohmann::json& j) { inline void to_json(nlohmann::json& j, const TradingVariables& v) { j = nlohmann::json{ {"STRATEGY", v.STRATEGY}, - {"STOP_DISTANCE_IN_PIPS", std::to_string(v.STOP_DISTANCE_IN_PIPS)}, - {"LIMIT_DISTANCE_IN_PIPS", std::to_string(v.LIMIT_DISTANCE_IN_PIPS)}, + {"STOP_DISTANCE_IN_ATR", std::to_string(v.STOP_DISTANCE_IN_ATR)}, + {"LIMIT_DISTANCE_IN_ATR", std::to_string(v.LIMIT_DISTANCE_IN_ATR)}, {"TRADING_SIZE", std::to_string(v.TRADING_SIZE)}, }; } inline void from_json(const nlohmann::json& j, TradingVariables& v) { j.at("STRATEGY").get_to(v.STRATEGY); - v.STOP_DISTANCE_IN_PIPS = readIntField(j.at("STOP_DISTANCE_IN_PIPS")); - v.LIMIT_DISTANCE_IN_PIPS = readIntField(j.at("LIMIT_DISTANCE_IN_PIPS")); + // Hard rename from *_IN_PIPS (no legacy-key fallback, deliberately): the + // semantics changed from pips to ATR multipliers, so an old payload + // parsing silently would trade a 100-pip value as a 100x multiplier. + // Pre-rename Redis payloads and Elasticsearch winners fail loudly here. + v.STOP_DISTANCE_IN_ATR = readIntField(j.at("STOP_DISTANCE_IN_ATR")); + v.LIMIT_DISTANCE_IN_ATR = readIntField(j.at("LIMIT_DISTANCE_IN_ATR")); v.TRADING_SIZE = readIntField(j.at("TRADING_SIZE")); } } diff --git a/source/shared/utilities/atr.cppm b/source/shared/utilities/atr.cppm new file mode 100644 index 0000000..0287c8d --- /dev/null +++ b/source/shared/utilities/atr.cppm @@ -0,0 +1,71 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// atr — integer Average True Range over OHLC bars of scaled INT32 points. +// +// Port of the C# AverageTrueRange.Calculate(List candles, int +// period) helper, kept entirely in integer arithmetic so it can run on the +// per-tick hot path (the engine's rule: no software-emulated decimal and no +// floating point there — see ema). +// +// True Range of a bar against its predecessor is the largest of: +// 1. high - low (the bar's own span) +// 2. |high - prevClose| (gap up through the previous close) +// 3. |low - prevClose| (gap down through the previous close) +// and the ATR here is the plain SMA of the last `period` true ranges — the +// C# original's form, not Wilder's recursive smoothing. +// +// Units: bar prices are the engine's scaled fixed-point points (see +// priceData / symbolScale), so the result is in POINTS. Divide by +// symbol_scale::get(symbol) for pips. +// +// Headroom: the largest scaled price (~4.5M for indices) is < 2^23, so a +// single TR is < 2^24 and the int64 sum cannot overflow for any real period. +export module atr; + +import std; // replaces , , +import ohlcObject; // OhlcObject bar record + +export namespace atr { + +// C# analogue: decimal AverageTrueRange.Calculate(List, int). +// +// `candles` MUST be chronological (index 0 = oldest, last = newest); the last +// element may be the in-progress bar — its TR moves until it closes, exactly +// like the C# reference which fed the live list straight in. Only the final +// period+1 candles are read (period TRs, each needing its predecessor's +// close); earlier candles are ignored. +// +// Fewer than period+1 candles returns 0 — the C# behaviour — so callers treat +// 0 as "not warm yet" (a genuinely zero ATR means a dead-flat market, which +// is equally untradeable). period < 1 throws std::invalid_argument. +[[nodiscard]] std::int32_t calculate(std::span candles, + int period = 14) { + if (period < 1) { + throw std::invalid_argument("atr::calculate: period must be >= 1"); + } + + const auto n = static_cast(period); + if (candles.size() < n + 1) { + return 0; + } + + std::int64_t sum = 0; + for (std::size_t i = candles.size() - n; i < candles.size(); ++i) { + const std::int64_t prevClose = candles[i - 1].close; + const std::int64_t highLow = + static_cast(candles[i].high) - candles[i].low; + const std::int64_t highGap = std::abs(candles[i].high - prevClose); + const std::int64_t lowGap = std::abs(candles[i].low - prevClose); + sum += std::max({highLow, highGap, lowGap}); + } + + // TRs are non-negative, so add-half-divisor is exact round-to-nearest + // (same idiom as ema's SMA seed). + return static_cast((sum + period / 2) / period); +} + +} // namespace atr diff --git a/source/shared/utilities/backtestLog.cppm b/source/shared/utilities/backtestLog.cppm new file mode 100644 index 0000000..3dd44ba --- /dev/null +++ b/source/shared/utilities/backtestLog.cppm @@ -0,0 +1,41 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// backtestLog (module) — the format-string half of the shared logging +// facility, layered on top of backtestLog.hpp. The header stays a plain +// GMF-safe include so non-module TUs can reach timestamp()/error()/ +// set_quiet(); this module adds the std::format-based logLine that module +// TUs (ingest, load, live/*) share, replacing the per-command copies they +// used to carry. + +module; + +#include // the `stdout` macro (import std exports std::fflush, not macros) + +#include "shared/utilities/backtestLog.hpp" // backtest_log::timestamp — GMF-safe + +export module backtestLog; + +import std; + +export namespace backtest_log { + +// Like std::println to stdout, but prefixed with the shared UTC millisecond +// timestamp (backtest_log::timestamp, so every command's lines match) and +// flushed immediately. stdout is fully buffered when redirected (pipe/file/ +// service log), so without the flush infrequent lines (the once-a-minute +// reports) would sit in the buffer and only appear in a burst when the +// process exits. +template +void logLine(std::format_string fmt, Args&&... args) { + const std::string message = std::format(fmt, std::forward(args)...); + std::println("{} {}", timestamp(), message); + std::fflush(stdout); + // No-op unless live mode registered the live-logs sink (backtestLog.hpp). + shipToSink(false, message); +} + +} // namespace backtest_log diff --git a/source/shared/utilities/backtestLog.hpp b/source/shared/utilities/backtestLog.hpp index f9313cb..36ae858 100644 --- a/source/shared/utilities/backtestLog.hpp +++ b/source/shared/utilities/backtestLog.hpp @@ -23,16 +23,81 @@ // std::cerr. RedisRunner calls set_quiet(true) so those call sites skip their output // entirely (no stream access -> no race), while error() serialises the rare // failure path behind a mutex so genuine problems still surface safely. +// +// Optional sink: live mode registers a shipping callback (liveTrace -> +// elastic::enqueueDocument, index "live-logs") so every error()/logLine() +// also lands in Elasticsearch. The sink is a plain function pointer behind +// an atomic — disarmed (nullptr) for backtests and tests, where sweep-scale +// log volume must never reach an index. The thread-local suppression exists +// for the publisher's OWN threads and delivery paths: a publisher failure +// line that re-entered the publisher's queue would feed every subsequent +// flush a fresh document for as long as an outage lasts (the same doctrine +// as DocumentBatcher::flush's no-putEngineException rule). namespace backtest_log { +// Registered shipping callback: isError distinguishes error() (stderr) from +// logLine() (stdout). Capture-free function pointer so the atomic stays +// lock-free and header-only. +using Sink = void (*)(bool isError, std::string_view message); + // Hide the actual state inside a private detail namespace. namespace detail { inline std::atomic& quiet_flag() { static std::atomic q{false}; return q; } + +inline std::atomic& sink_slot() { + static std::atomic s{nullptr}; + return s; +} + +// Per-thread opt-out — see the header comment. A plain function returning a +// thread_local reference keeps this header-only and ODR-safe. +inline bool& sink_suppressed() { + thread_local bool suppressed = false; + return suppressed; +} } // namespace detail +// nullptr disarms. Release/acquire pairing so the sink's referenced state +// (liveTrace's cached env/hostname) is visible to whichever thread ships. +inline void setSink(Sink sink) { + detail::sink_slot().store(sink, std::memory_order_release); +} + +// Whether a sink is registered — for tests pinning the arm/clear gates +// (whether shipping actually happens is unobservable from outside once the +// publisher drops the document). +[[nodiscard]] inline bool sinkArmed() { + return detail::sink_slot().load(std::memory_order_acquire) != nullptr; +} + +// RAII per-thread suppression for the publisher's delivery paths (and any +// other code whose log lines must not re-enter the publisher's queue). +struct SinkSuppression { + SinkSuppression() : previous_(detail::sink_suppressed()) { + detail::sink_suppressed() = true; + } + ~SinkSuppression() { detail::sink_suppressed() = previous_; } + SinkSuppression(const SinkSuppression&) = delete; + SinkSuppression& operator=(const SinkSuppression&) = delete; + +private: + bool previous_; +}; + +// Hand `message` to the registered sink, unless disarmed or this thread is +// suppressed. Shared by error() below and the module's logLine(). +inline void shipToSink(const bool isError, const std::string_view message) { + if (detail::sink_suppressed()) { + return; + } + if (const Sink sink = detail::sink_slot().load(std::memory_order_acquire)) { + sink(isError, message); + } +} + // Provide clean, explicit public getters and setters. inline bool is_quiet() { return detail::quiet_flag().load(std::memory_order_relaxed); @@ -68,8 +133,13 @@ inline std::string timestamp() { } inline void error(std::string_view message) { - std::scoped_lock lock(errorMutex()); - std::cerr << timestamp() << ' ' << message << std::endl; + { + std::scoped_lock lock(errorMutex()); + std::cerr << timestamp() << ' ' << message << std::endl; + } + // Outside the mutex: the sink enqueues into the elastic batcher (its own + // lock), and stderr ordering is already settled above. + shipToSink(true, message); } } // namespace backtest_log diff --git a/source/shared/utilities/barStore.cppm b/source/shared/utilities/barStore.cppm new file mode 100644 index 0000000..5ccfb3d --- /dev/null +++ b/source/shared/utilities/barStore.cppm @@ -0,0 +1,221 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// barStore — the shared per-symbol bar pipeline (OHLC and range bars). +// +// Historically each strategy built and owned its own bar histories inside +// during(), which ran AFTER decide() and left the run loop / live runner with +// no candles in scope for pre-decide checks (the ATR entry conditions). This +// store centralises that: the loop owner (backtest run or live worker) +// registers every timeframe anyone needs up front, feeds every tick in +// exactly once, and both the entry-condition gate and the strategies read +// from the same rolling windows. +// +// Design: +// - A series is keyed by its bar duration. Registering the same duration +// twice keeps the larger window — two consumers of one timeframe share +// one history and each reads its own tail (std::span(...).last(n)). +// - update() feeds the tick's ASK into every registered series, matching +// ohlcBuilder's one historical consumer (OhlcBreakoutStrategy). +// - The window `count` doubles as ohlcBuilder's prepopulateCount, so a +// live worker's first tick warms every series from QuestDB +// (OHLC_PREPOPULATE-gated) instead of starting cold. +// - The last element of a series is always the in-progress bar; earlier +// ones are complete (see ohlcBuilder). +// - Range-bar series ride the same store and the same update(): identity is +// a RangeBarSpec instead of a duration (registerRangeSeries/findRange), +// state lives in rangebar::RangeSeries, and the first update per symbol +// fires the raw-tick QuestDB warm-up exactly like the OHLC leg (see +// rangeBarBuilder). One nuance vs OHLC: a range series' last element may +// be just-closed, since completion is known at the breaching tick. +// +// NOT thread-safe: one instance per backtest run / per live worker, same +// ownership rule as TradeManager. +export module barStore; + +import std; // replaces , , , + // , , , , + // , +import ohlcBuilder; // ohlc::calculateOHLC +import ohlcObject; // OhlcObject bar record +import priceData; // PriceData +import rangeBarBuilder; // rangebar::RangeBarSpec / RangeSeries + +export namespace bars { + +// One bar series' shape: duration of a bar and the rolling window kept +// (also the QuestDB warm-up depth on a live cold start). +struct SeriesSpec { + std::chrono::minutes minutes; + int count; +}; + +class BarStore { +public: + // Register a timeframe before the first update(). Registering a duration + // twice keeps the larger count, so independent consumers (a strategy + // timeframe and the ATR gate) can each declare what they need without + // coordinating. Throws std::invalid_argument on a non-positive duration + // or count — a misconfigured series should die loudly at setup, not + // roll a bar per tick (see calculateOHLC's rationale). + void registerSeries(const std::chrono::minutes minutes, const int count) { + if (minutes < std::chrono::minutes{1} || count < 1) { + throw std::invalid_argument( + "BarStore::registerSeries: minutes and count must be >= 1"); + } + for (SeriesSpec& spec : specs_) { + if (spec.minutes == minutes) { + spec.count = std::max(spec.count, count); + return; + } + } + specs_.push_back({minutes, count}); + } + + // Register a range-bar series before the first update(). Identity is + // (atrTickWindow, atrPercent); re-registering an identity keeps the + // larger count, mirroring registerSeries. Throws on any non-positive + // field — same die-loudly-at-setup doctrine (RangeSeries would throw the + // same on first tick, but per-strategy setup is where it belongs). + void registerRangeSeries(const rangebar::RangeBarSpec& spec) { + if (spec.atrTickWindow < 1 || spec.atrPercent < 1 || spec.count < 1) { + throw std::invalid_argument( + "BarStore::registerRangeSeries: all spec fields must be >= 1"); + } + for (rangebar::RangeBarSpec& existing : rangeSpecs_) { + if (rangebar::sameIdentity(existing, spec)) { + existing.count = std::max(existing.count, spec.count); + return; + } + } + rangeSpecs_.push_back(spec); + } + + // Feed the tick into EVERY registered series for its symbol. Call + // exactly once per tick, unconditionally — bar history must never gap. + // The loop owners call this BEFORE the entry gates and decide(), so the + // ATR conditions and the strategies judge a tick against bar state that + // already includes it. + void update(const PriceData& tick) { + if (!specs_.empty()) { + // Heterogeneous find first: only a symbol's first tick pays for + // the std::string key construction (same pattern as TradeManager). + auto it = bySymbol_.find(std::string_view{tick.symbol}); + if (it == bySymbol_.end()) { + it = bySymbol_.emplace(tick.symbol, + std::vector>(specs_.size())) + .first; + } + // A series registered after this symbol's first tick still gets a + // (cold) history rather than an out-of-range index. + if (it->second.size() < specs_.size()) { + it->second.resize(specs_.size()); + } + for (std::size_t i = 0; i < specs_.size(); ++i) { + std::vector& series = it->second[i]; + ohlc::calculateOHLC(tick, tick.ask, specs_[i].minutes, series, + specs_[i].count); + // Rolling window: oldest bars dropped from the front. + const auto cap = static_cast(specs_[i].count); + if (series.size() > cap) { + series.erase(series.begin(), + series.end() - static_cast(cap)); + } + } + } + if (!rangeSpecs_.empty()) { + auto it = rangeBySymbol_.find(std::string_view{tick.symbol}); + if (it == rangeBySymbol_.end()) { + it = rangeBySymbol_.emplace(tick.symbol, + std::vector{}) + .first; + } + // Late-registered specs get a (cold) series appended, mirroring + // the OHLC resize above; RangeSeries has no default ctor, so the + // append is explicit rather than a resize. + while (it->second.size() < rangeSpecs_.size()) { + it->second.emplace_back(rangeSpecs_[it->second.size()]); + } + // A series' first update fires the QuestDB tick warm-up + // (rangeBarBuilder), exactly like calculateOHLC's first-tick seed. + for (rangebar::RangeSeries& series : it->second) { + series.update(tick); + } + } + } + + // The bars for (symbol, duration): chronological, last element + // in-progress. nullptr when the duration was never registered or the + // symbol has not ticked yet — callers treat both as "not warm". + [[nodiscard]] const std::vector* find( + const std::string_view symbol, const std::chrono::minutes minutes) const { + std::size_t index = specs_.size(); + for (std::size_t i = 0; i < specs_.size(); ++i) { + if (specs_[i].minutes == minutes) { + index = i; + break; + } + } + if (index == specs_.size()) { + return nullptr; + } + const auto it = bySymbol_.find(symbol); + if (it == bySymbol_.end() || index >= it->second.size()) { + return nullptr; + } + return &it->second[index]; + } + + // The range bars for (symbol, spec identity): chronological, last element + // in-progress or just-closed (see rangeBarBuilder). count is ignored in + // the lookup, like find() ignores it. nullptr when the identity was never + // registered or the symbol has not ticked yet — callers treat both as + // "not warm". + [[nodiscard]] const std::vector* findRange( + const std::string_view symbol, const rangebar::RangeBarSpec& spec) const { + std::size_t index = rangeSpecs_.size(); + for (std::size_t i = 0; i < rangeSpecs_.size(); ++i) { + if (rangebar::sameIdentity(rangeSpecs_[i], spec)) { + index = i; + break; + } + } + if (index == rangeSpecs_.size()) { + return nullptr; + } + const auto it = rangeBySymbol_.find(symbol); + if (it == rangeBySymbol_.end() || index >= it->second.size()) { + return nullptr; + } + return &it->second[index].bars(); + } + +private: + // Transparent hasher (same pattern as TradeManager::activeTrades) so the + // per-tick find() takes a string_view and never allocates a temporary key. + struct SymbolHash { + using is_transparent = void; + std::size_t operator()(const std::string_view symbol) const noexcept { + return std::hash{}(symbol); + } + }; + + std::vector specs_; + // bySymbol_[symbol][i] is the history for specs_[i]. A handful of specs + // per run, so the per-tick spec scans are trivially cheap. + std::unordered_map>, + SymbolHash, std::equal_to<>> + bySymbol_; + // The range-bar leg, parallel in shape: rangeBySymbol_[symbol][i] is the + // stateful series for rangeSpecs_[i] (bars plus the rolling tick window, + // which is why the element is a RangeSeries rather than a bare vector). + std::vector rangeSpecs_; + std::unordered_map, + SymbolHash, std::equal_to<>> + rangeBySymbol_; +}; + +} // namespace bars diff --git a/source/shared/utilities/jsonParser.cpp b/source/shared/utilities/jsonParser.cpp index 7322526..2b9daca 100644 --- a/source/shared/utilities/jsonParser.cpp +++ b/source/shared/utilities/jsonParser.cpp @@ -24,3 +24,11 @@ tradingDefinitions::RunConfiguration JsonParser::parseRunConfigurationFromBase64 tradingDefinitions::StrategyConfig JsonParser::parseStrategyFromBase64(const std::string& input) { return json::parse(Base64::b64decode(input)).get(); } + +experiments::ExperimentRunConfiguration JsonParser::parseExperimentRunFromBase64(const std::string& input) { + return json::parse(Base64::b64decode(input)).get(); +} + +experiments::ExperimentConfig JsonParser::parseExperimentFromBase64(const std::string& input) { + return json::parse(Base64::b64decode(input)).get(); +} diff --git a/source/shared/utilities/jsonParser.hpp b/source/shared/utilities/jsonParser.hpp index 7ada8db..c90fd36 100644 --- a/source/shared/utilities/jsonParser.hpp +++ b/source/shared/utilities/jsonParser.hpp @@ -9,10 +9,14 @@ #include #include #include "shared/tradingDefinitions.hpp" +#include "shared/experiments/experimentConfig.hpp" +#include "shared/experiments/experimentRunConfiguration.hpp" class JsonParser { public: static tradingDefinitions::Configuration parseConfigurationFromBase64(const std::string& input); static tradingDefinitions::RunConfiguration parseRunConfigurationFromBase64(const std::string& input); static tradingDefinitions::StrategyConfig parseStrategyFromBase64(const std::string& input); + static experiments::ExperimentRunConfiguration parseExperimentRunFromBase64(const std::string& input); + static experiments::ExperimentConfig parseExperimentFromBase64(const std::string& input); }; diff --git a/source/shared/utilities/marketHours.cppm b/source/shared/utilities/marketHours.cppm new file mode 100644 index 0000000..b31f5f4 --- /dev/null +++ b/source/shared/utilities/marketHours.cppm @@ -0,0 +1,182 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// marketHours — the peak-market-hours ENTRY filter (a port of the C# engine's +// MarketHours.TradePermitted). Every symbol maps to the session whose open +// drives its liquidity; tradePermitted answers whether a NEW ENTRY is allowed +// at a UTC instant. It gates entries only — exits (SL/TP, liquidation, the +// live book sync and during()) must never consult it. +// +// Rules, all in UTC (tick timestamps are UTC end-to-end, see tickPacket): +// - Weekend / low-liquidity block for every session: Friday from 16:00, +// all of Sunday, Monday before 02:00. +// - Asia: 00:00-06:00, fixed — no DST anchor. +// - Europe: three hours from the London open — 07:00 under BST, else 08:00. +// - NewYork: three hours from the US open — 13:30 under EDT, else 14:30. +// - Unknown symbol: blocked. Fail-closed on purpose (the C# behaviour, and +// the trade-lock doctrine): a missed entry is recoverable, an entry traded +// in the wrong session is not. The table below must partition the same +// 29-symbol universe as symbol_scale::kTable (pinned by test). +// +// DST is hand-rolled rather than tzdb-based: libc++'s chrono time-zone +// database support differs between the CI clang and the local Homebrew clang, +// while the calendar types used here are plain C++20 chrono, portable and +// constexpr. Date-level granularity ("is this DAY in summer time?") is exact +// for this filter because both markets transition in the small hours of a +// Sunday — and every Sunday is already weekend-blocked outright. + +export module marketHours; + +import std; // replaces , , , + +export namespace market_hours { + +// Which market's open anchors the symbol's peak trading window. +enum class Session { + Asia, // 00:00-06:00 UTC, fixed + Europe, // three hours from the London open (BST-aware) + NewYork, // three hours from the US open (EDT-aware) + Unknown, // not in kTable — tradePermitted fails closed +}; + +struct Entry { + std::string_view symbol; + Session session; +}; + +// MUST stay sorted ascending by symbol (static_assert below) — sessionFor +// binary-searches it. Same universe as symbol_scale::kTable / live::kMarkets; +// tests/marketHours.cpp pins the cross-table equivalence both ways, so adding +// a symbol to one table without the others fails at test time. +inline constexpr std::array kTable{{ + {"AUDNZD", Session::Asia}, + {"AUDUSD", Session::Asia}, + {"AUSIDXAUD", Session::Asia}, + {"BRENTCMDUSD", Session::NewYork}, + {"COPPERCMDUSD", Session::NewYork}, + {"DEUIDXEUR", Session::Europe}, + {"EURAUD", Session::Asia}, + {"EURCHF", Session::Europe}, + {"EURGBP", Session::Europe}, + {"EURJPY", Session::Asia}, + {"EURNOK", Session::Europe}, + {"EURUSD", Session::Europe}, + {"FRAIDXEUR", Session::Europe}, + {"GBPJPY", Session::Asia}, + {"GBPUSD", Session::Europe}, + {"GBRIDXGBP", Session::Europe}, + {"HKGIDXHKD", Session::Asia}, + {"JPNIDXJPY", Session::Asia}, + {"LIGHTCMDUSD", Session::NewYork}, + {"NZDUSD", Session::Asia}, + {"USA30IDXUSD", Session::NewYork}, + {"USA500IDXUSD", Session::NewYork}, + {"USATECHIDXUSD", Session::NewYork}, + {"USDCAD", Session::NewYork}, + {"USDCHF", Session::Europe}, + {"USDJPY", Session::Asia}, + {"USDSEK", Session::Europe}, + {"XAGUSD", Session::NewYork}, + {"XAUUSD", Session::NewYork}, +}}; + +static_assert( + [] { + for (std::size_t i = 1; i < kTable.size(); ++i) { + if (!(kTable[i - 1].symbol < kTable[i].symbol)) { + return false; + } + } + return true; + }(), + "market_hours::kTable must stay sorted ascending by symbol — " + "sessionFor binary-searches it"); + +// Binary search over kTable (same shape as symbol_scale::findEntry, with the +// overflow-safe midpoint). Unknown when the symbol is absent. +[[nodiscard]] constexpr Session sessionFor(std::string_view symbol) noexcept { + std::size_t lo = 0; + std::size_t hi = kTable.size(); + while (lo < hi) { + const std::size_t mid = lo + ((hi - lo) >> 1); + const Entry& entry = kTable[mid]; + if (entry.symbol < symbol) { + lo = mid + 1; + } else if (symbol < entry.symbol) { + hi = mid; + } else { + return entry.session; + } + } + return Session::Unknown; +} + +// London summer time (BST): [last Sunday of March, last Sunday of October). +// Exported so tests can pin the transition dates directly. +[[nodiscard]] constexpr bool isLondonSummer( + const std::chrono::sys_days day) noexcept { + using namespace std::chrono; + const year y = year_month_day{day}.year(); + const sys_days start{year_month_weekday_last{y, March, Sunday[last]}}; + const sys_days end{year_month_weekday_last{y, October, Sunday[last]}}; + return day >= start && day < end; +} + +// New York summer time (EDT): [second Sunday of March, first Sunday of +// November). +[[nodiscard]] constexpr bool isNewYorkSummer( + const std::chrono::sys_days day) noexcept { + using namespace std::chrono; + const year y = year_month_day{day}.year(); + const sys_days start{year_month_weekday{y, March, Sunday[2]}}; + const sys_days end{year_month_weekday{y, November, Sunday[1]}}; + return day >= start && day < end; +} + +// True when a NEW ENTRY is allowed for `symbol` at `utcTimestamp` (see the +// header comment for the rules). Callers gate decide()/entries on this and +// nothing else. +[[nodiscard]] constexpr bool tradePermitted( + const std::string_view symbol, + const std::chrono::system_clock::time_point utcTimestamp) noexcept { + using namespace std::chrono; + const sys_days day = floor(utcTimestamp); + const weekday dow{day}; + // Time of day as a raw duration in [0h, 24h): compared against hours/ + // minutes constants via common_type, so the 13:30 US open needs no + // whole-hour rounding and the window boundaries stay exact. + const auto sinceMidnight = utcTimestamp - day; + + if (dow == Sunday) { + return false; + } + if (dow == Friday && sinceMidnight >= hours{16}) { + return false; + } + if (dow == Monday && sinceMidnight < hours{2}) { + return false; + } + + switch (sessionFor(symbol)) { + case Session::Asia: + return sinceMidnight < hours{6}; + case Session::Europe: { + const minutes open{isLondonSummer(day) ? hours{7} : hours{8}}; + return sinceMidnight >= open && sinceMidnight < open + hours{3}; + } + case Session::NewYork: { + const minutes open = + minutes{isNewYorkSummer(day) ? hours{13} : hours{14}} + + minutes{30}; + return sinceMidnight >= open && sinceMidnight < open + hours{3}; + } + case Session::Unknown: + return false; + } + return false; // unreachable, but keeps -Wreturn-type quiet +} + +} // namespace market_hours diff --git a/source/shared/utilities/ohlcBuilder.cppm b/source/shared/utilities/ohlcBuilder.cppm index 5910ecb..2e33e57 100644 --- a/source/shared/utilities/ohlcBuilder.cppm +++ b/source/shared/utilities/ohlcBuilder.cppm @@ -20,34 +20,146 @@ // (std::chrono::minutes etc. convert implicitly) // - List -> std::vector&, mutated in place // (the C# returned the same list it mutated) +// +// Pre-population (port of the C# GetOHLCData): on the very first tick the bar +// list can be seeded with historical bars from QuestDB (SAMPLE BY over the +// symbol's tick table), so the strategy hits the ground running instead of +// waiting OHLC_COUNT * OHLC_MINUTES of ticks — backtests trade from the first +// replayed tick, live warms up on launch. ON by default; OHLC_PREPOPULATE=0 +// is the kill switch for sweeps, which fire two queries per run per symbol — +// memoize per (symbol, minutes, count) if that ever hurts. Unit tests force +// the gate off at startup (tests/ohlc.cpp) to stay hermetic. +// Connection comes from QUESTDB_HOST/QUESTDB_PORT; +// the run command takes its QuestDB host via argv, so set the env var too when +// the host isn't local (live has no host argument — the env vars are its only +// source, and liveCommand logs the warm-up mode at startup). Bars are +// aggregated from ask only, matching the one +// consumer (the breakout strategy feeds tick.ask). Both paths now use true +// last-tick closes; the remaining accepted drift vs incrementally built bars: +// buckets are calendar-aligned rather than first-tick-anchored, and buckets +// with < 10 ticks are dropped. + +module; + +#include "shared/utilities/env.hpp" export module ohlcBuilder; -import std; // replaces , , -import priceData; // PriceData tick (timestamp source) -import ohlcObject; // the bar record being built +import std; // replaces , , +import connectionFactory; // questdb::connectionFromEnv +import databaseConnection; // DatabaseConnection::queryOhlc +import ohlcObject; // the bar record being built +import priceData; // PriceData tick (timestamp source) +import symbolScale; // symbol whitelist guard before SQL interpolation export namespace ohlc { +// SQL for the warm-up bars, ending strictly before `before` (the first tick's +// timestamp — for live that is "now", for a backtest it keeps every replay +// tick out of the seed, so nothing is double-counted and there is no +// lookahead). The bucket containing `before` comes back partial: exactly the +// in-progress bar a continuously running builder would hold. +// +// Unlike the C# original there is no LIMIT over-fetch multiplier — the +// ticks >= 10 filter runs server-side (QuestDB's no-HAVING idiom: aggregate in +// a subquery, filter outside) so LIMIT count is exact — and no per-symbol +// lookback table: one generous formula covers the worst case in +// symbol_scale::kTable (a ~6h/day index across a weekend, (24/6)*(7/5) = 5.6x +// calendar/trading -> 6x), plus 10 days for holiday clusters. Over-scanning is +// cheap; LIMIT caps the rows returned. +std::string prepopulateQuery(std::string_view symbol, + std::chrono::system_clock::time_point before, + std::chrono::minutes barMinutes, int count) { + const std::int64_t beforeMicros = + std::chrono::duration_cast(before.time_since_epoch()) + .count(); + const std::int64_t totalMinutes = static_cast(barMinutes.count()) * count; + const std::int64_t days = (totalMinutes + 1439) / 1440 * 6 + 10; + const std::int64_t fromMicros = beforeMicros - days * 86'400'000'000; + + return std::format( + "SELECT timestamp, open, high, low, close FROM (" + "SELECT timestamp, first(ask) AS open, max(ask) AS high, min(ask) AS low, " + "last(ask) AS close, count() AS ticks FROM '{}' " + "WHERE timestamp >= cast({}L AS timestamp) AND timestamp < cast({}L AS timestamp) " + "SAMPLE BY {}m ALIGN TO CALENDAR" + ") WHERE ticks >= 10 ORDER BY timestamp DESC LIMIT {}", + symbol, fromMicros, beforeMicros, barMinutes.count(), count); +} + +// Fetches up to `count` warm-up bars ending just before `before`, oldest +// first, restoring the builder invariant (last element = in-progress bar). +// Returns empty — a plain cold start — when the gate is off, the symbol is +// unknown (also the SQL-injection guard: symbols are interpolated, not bound, +// same as sqlManager), or anything DB-side fails. One attempt, no retries. +std::vector prepopulateOHLC(std::string_view symbol, + std::chrono::system_clock::time_point before, + std::chrono::minutes barMinutes, int count) { + if (count <= 0 || barMinutes < std::chrono::minutes{1}) { + return {}; + } + if (env::getOr("OHLC_PREPOPULATE", "1") != "1") { + return {}; + } + if (symbol_scale::get(symbol) == symbol_scale::kUnknown) { + return {}; + } + try { + const DatabaseConnection db = questdb::connectionFromEnv(); + std::vector bars = + db.queryOhlc(prepopulateQuery(symbol, before, barMinutes, count)); + std::ranges::reverse(bars); + if (!bars.empty()) { + bars.back().complete = false; + } + return bars; + } catch (const std::exception& e) { + std::println(std::cerr, "prepopulateOHLC({}): {} — cold start", symbol, e.what()); + return {}; + } +} + void calculateOHLC(const PriceData& tick, std::int32_t price, std::chrono::system_clock::duration duration, - std::vector& bars) { - // First tick ever: seed the initial bar from it. + std::vector& bars, int prepopulateCount = 0) { + // A non-positive duration would make every tick roll a new bar: one bar + // per tick over a months-long tick stream grows the vector without bound + // (OOM), and every "bar" is a single tick. Nothing upstream hard-validates + // OHLC_MINUTES ({0,0} is a legal "no bars" sentinel for strategies that + // never call this), so the strategy that DOES build bars must be stopped + // here — the throw surfaces as a contained per-strategy failure. + if (duration <= std::chrono::system_clock::duration::zero()) { + throw std::invalid_argument( + "calculateOHLC: bar duration must be positive (check OHLC_MINUTES)"); + } + + // First tick ever: seed from QuestDB history when enabled, else from the + // tick. Either way bars is non-empty afterwards, so the query fires at + // most once per bar list. if (bars.empty()) { - bars.push_back({.date = tick.timestamp, - .open = price, - .close = price, - .high = price, - .low = price}); + if (prepopulateCount > 0) { + bars = prepopulateOHLC(tick.symbol, tick.timestamp, + std::chrono::duration_cast(duration), + prepopulateCount); + } + if (bars.empty()) { + bars.push_back({.date = tick.timestamp, + .open = price, + .close = price, + .high = price, + .low = price}); + } } // Strictly greater-than, matching the C# `diff > duration.TotalMinutes`: // a tick landing exactly on the boundary still belongs to the open bar. if (tick.timestamp - bars.back().date > duration) { - // C# behaviour, ported as-is: the completed bar's close is overwritten - // with the first price of the NEXT bucket, so consecutive bars join up. + // The finished bar keeps the close it accumulated tick by tick — its + // own bucket's true last price. (The C# original overwrote it with the + // NEXT bucket's first price to make bars "join up", which could push + // close outside [low, high] on a gap and disagreed with the + // prepopulate query's last(ask) convention.) bars.back().complete = true; - bars.back().close = price; bars.push_back({.date = tick.timestamp, .open = price, diff --git a/source/shared/utilities/queueKeys.hpp b/source/shared/utilities/queueKeys.hpp index 686ce9b..a434681 100644 --- a/source/shared/utilities/queueKeys.hpp +++ b/source/shared/utilities/queueKeys.hpp @@ -6,6 +6,7 @@ #pragma once +#include #include // Redis keys for the work queue. A sweep produces, linked by RUN_ID: @@ -19,10 +20,32 @@ namespace queue_keys { inline constexpr const char* RUN = "BACKTESTING_QUEUE_RUN"; + +// Every run queue, in the strict priority order workers drain them: a chain +// queue is only reached once every queue before it is empty. Sweeps and +// hand-queued one-offs land on RUN; a run chained by the rolling-window +// ladder lands on the chain queue matching its rung (rung N -> index N, see +// rolling::queueKeyFor). The keys are depth-indexed rather than named after +// the window ("wave 1", not "OFFSET_3") because the descriptor already +// carries LAST_MONTHS/OFFSET_MONTHS — retuning the ladder's windows must +// never rename a queue. Net effect: a fresh grid sweep always preempts the +// chained single-strategy backlog, and LLEN per key reads wave progress. +inline constexpr std::array RUN_QUEUES = std::to_array({ + RUN, + "BACKTESTING_QUEUE_RUN_CHAIN:1", + "BACKTESTING_QUEUE_RUN_CHAIN:2", + "BACKTESTING_QUEUE_RUN_CHAIN:3", +}); inline constexpr const char* STRATEGY_PREFIX = "BACKTESTING_QUEUE_STRATEGY:"; inline constexpr const char* STRATEGY_PAYLOAD_PREFIX = "BACKTESTING_QUEUE_STRATEGY_PAYLOAD:"; +// Safety-net expiry on strategy payload keys so a crashed or abandoned run +// cannot leak them forever; the worker's GETDEL is the normal cleanup. Part +// of the queue contract (shared by the sweep loader and the rolling-window +// re-queue in Operations) so every producer reaps on the same schedule. +inline constexpr long PAYLOAD_TTL_SECONDS = 7L * 24 * 60 * 60; + inline std::string strategyKey(const std::string& runId) { return std::string(STRATEGY_PREFIX) + runId; } @@ -34,4 +57,24 @@ inline std::string strategyPayloadKey(const std::string& runId, return std::string(STRATEGY_PAYLOAD_PREFIX) + runId + ":" + strategyUuid; } +// The experiment queue family: the same run/name-list/payload-key shape as +// the strategy queue above, produced by `experiments ` and drained by +// the `analysis` worker. ONE queue, no priority array — RUN_QUEUES exists +// only for the rolling-window ladder, which experiments never join. Payload +// keys reuse PAYLOAD_TTL_SECONDS, so crashed experiment runs are reaped on +// the same schedule as strategy runs. +inline constexpr const char* EXPERIMENT_RUN = "BACKTESTING_QUEUE_EXPERIMENT_RUN"; +inline constexpr const char* EXPERIMENT_PREFIX = "BACKTESTING_QUEUE_EXPERIMENT:"; +inline constexpr const char* EXPERIMENT_PAYLOAD_PREFIX = + "BACKTESTING_QUEUE_EXPERIMENT_PAYLOAD:"; + +inline std::string experimentKey(const std::string& runId) { + return std::string(EXPERIMENT_PREFIX) + runId; +} + +inline std::string experimentPayloadKey(const std::string& runId, + const std::string& experimentUuid) { + return std::string(EXPERIMENT_PAYLOAD_PREFIX) + runId + ":" + experimentUuid; +} + } // namespace queue_keys diff --git a/source/shared/utilities/rangeBarBuilder.cppm b/source/shared/utilities/rangeBarBuilder.cppm new file mode 100644 index 0000000..80626f7 --- /dev/null +++ b/source/shared/utilities/rangeBarBuilder.cppm @@ -0,0 +1,295 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// rangeBarBuilder — incremental tick -> range bar aggregation. +// +// A range bar completes when price travels a threshold distance within the +// bar (high - low >= threshold, ask prices). The breaching tick is included +// in the bar and closes it at its REAL price — overshoot past the threshold +// is kept, and the NEXT tick opens the new bar at its real price. No +// synthetic boundary closes and no phantom gap bars: same real-prices-only +// doctrine as ohlcBuilder (which deliberately dropped the C# "join up" +// close overwrite). One tick therefore closes at most one bar. +// +// The threshold is dynamic and PURELY event-driven: a rolling high-low range +// over the last RANGE_ATR_TICK_WINDOW ticks, tracked by two monotonic deques +// (O(1) amortised per tick). There is deliberately no clock anywhere in the +// construct — an earlier design that grouped ticks into fixed-duration +// true-range slices was rejected because a volatility spike early in a slice +// would not move the measure until the slice rolled; the tick-count window +// widens on the very tick a spike happens and forgets it exactly +// RANGE_ATR_TICK_WINDOW ticks later. Session/weekend gaps need no special +// handling: a gap inside the window is captured by max - min automatically. +// +// The threshold locks when a bar opens (a bar's target never moves under it) +// and no bar exists until the window has seen a full RANGE_ATR_TICK_WINDOW of +// ticks — so every bar's threshold is locked from a fully-warm measure, and +// consumers' existing "short series = not warm" gates cover the start-up. +// +// Pre-population mirrors ohlcBuilder's: on a series' very first update the +// builder fetches raw ticks ending strictly before that tick from QuestDB and +// replays them through the same state machine — one replay warms both the +// window and the bar history, in backtest (seeded from before the replay +// window, so nothing is double-counted and there is no lookahead) and live +// (first tick on the worker thread) alike. Range bars cannot be aggregated +// server-side (SAMPLE BY is time-bucketed), but the pure tick-count measure +// makes the fetch exact instead of heuristic: RANGE_ATR_TICK_WINDOW rows warm +// the window perfectly by definition, plus a bounded margin for bar +// formation. Gated by the same OHLC_PREPOPULATE switch (default ON) so sweeps +// that silence warm-up DB traffic silence this too. + +module; + +#include "shared/utilities/env.hpp" + +export module rangeBarBuilder; + +import std; // replaces , , , +import connectionFactory; // questdb::connectionFromEnv +import databaseConnection; // DatabaseConnection::executeQuery +import ohlcObject; // the bar record being built (shared with OHLC) +import priceData; // PriceData tick +import symbolScale; // symbol whitelist guard before SQL interpolation + +export namespace rangebar { + +// One range-bar series' shape. Identity is (atrTickWindow, atrPercent) — +// count is only the window depth kept (and the prepopulate depth), so +// registering the same identity twice merges to the larger count, mirroring +// BarStore::registerSeries. +struct RangeBarSpec { + int atrTickWindow; // rolling tick window for the range measure + int atrPercent; // threshold = windowRange * atrPercent / 100 + int count; // bars kept AND the prepopulate margin driver +}; + +[[nodiscard]] constexpr bool sameIdentity(const RangeBarSpec& a, + const RangeBarSpec& b) { + return a.atrTickWindow == b.atrTickWindow && a.atrPercent == b.atrPercent; +} + +// Rows fetched by the warm-up query. The window term is exact — that many +// ticks warm the measure perfectly, by definition of a tick-count window. +// The margin covers bar formation, which IS an estimate: over any window of +// atrTickWindow ticks price spans windowRange, and a bar needs atrPercent% of +// that, so ~atrTickWindow * pct / 100 ticks form one bar on average; x2 for +// safety. Partial BAR warm-up is accepted by design (strategies gate on +// series size) — the most recent rows come back first, so the window (the +// part that must be right) always warms. The overall clamp bounds the +// transient PriceData vector (~1M rows ≈ one peak FX day ≈ 50 MB, freed +// after replay). +inline constexpr std::int64_t kMaxPrepopulateTicks = 1'000'000; + +[[nodiscard]] constexpr std::int64_t prepopulateTickCap(const RangeBarSpec& spec) { + const std::int64_t ticksPerBar = + (static_cast(spec.atrTickWindow) * spec.atrPercent + 99) / 100; + const std::int64_t margin = 2 * static_cast(spec.count) * ticksPerBar; + return std::min(spec.atrTickWindow + margin, kMaxPrepopulateTicks); +} + +// SQL for the warm-up ticks, ending strictly before `before` (the first +// tick's timestamp — live: "now"; backtest: keeps every replay tick out of +// the seed, so nothing is double-counted and there is no lookahead). The +// SELECT shape matches sqlManager's tick queries so +// DatabaseConnection::executeQuery parses the rows as-is. No time lower +// bound: the row count is what matters for a tick-count measure, and LIMIT +// bounds the scan (QuestDB walks the designated timestamp index backwards). +std::string prepopulateTicksQuery(std::string_view symbol, + std::chrono::system_clock::time_point before, + const RangeBarSpec& spec) { + const std::int64_t beforeMicros = + std::chrono::duration_cast(before.time_since_epoch()) + .count(); + return std::format( + "SELECT '{}' as symbol, ask, bid, timestamp FROM '{}' " + "WHERE timestamp < cast({}L AS timestamp) " + "ORDER BY timestamp DESC LIMIT {}", + symbol, symbol, beforeMicros, prepopulateTickCap(spec)); +} + +// Fetches the warm-up ticks ending just before `before`, oldest first, ready +// to replay through the series. Returns empty — a plain cold start — when the +// gate is off, the symbol is unknown (also the SQL-injection guard: symbols +// are interpolated, not bound, same as sqlManager), or anything DB-side +// fails. One attempt, no retries. +std::vector prepopulateTicks(std::string_view symbol, + std::chrono::system_clock::time_point before, + const RangeBarSpec& spec) { + if (spec.atrTickWindow < 1 || spec.atrPercent < 1 || spec.count < 1) { + return {}; + } + if (env::getOr("OHLC_PREPOPULATE", "1") != "1") { + return {}; + } + if (symbol_scale::get(symbol) == symbol_scale::kUnknown) { + return {}; + } + try { + const DatabaseConnection db = questdb::connectionFromEnv(); + std::vector ticks = + db.executeQuery(prepopulateTicksQuery(symbol, before, spec)); + std::ranges::reverse(ticks); + return ticks; + } catch (const std::exception& e) { + std::println(std::cerr, "prepopulateTicks({}): {} — cold start", symbol, + e.what()); + return {}; + } +} + +// One symbol's range-bar series: the rolling tick window plus the bar +// history. Bars reuse OhlcObject so atr/ema/swingPivots and strategy code +// consume them exactly like time bars. The vector keeps ohlcBuilder's +// "last element in-progress" invariant with one nuance: completion is known +// AT the breaching tick, so the last element may be just-closed +// (complete == true) until the next tick pushes its successor — marking it +// lazily would delay truthful information for no benefit. +// +// NOT thread-safe: owned via BarStore, one instance per backtest run / live +// worker. +class RangeSeries { +public: + explicit RangeSeries(const RangeBarSpec& spec) : spec_(spec) { + // A zero window can never warm, a zero percent would floor every + // threshold, a zero count can hold no bars — die loudly at setup, + // not per tick (same rationale as calculateOHLC's duration throw). + if (spec.atrTickWindow < 1 || spec.atrPercent < 1 || spec.count < 1) { + throw std::invalid_argument( + "RangeSeries: all RangeBarSpec fields must be >= 1"); + } + } + + // Feed one tick, exactly once, in stream order. The very first call + // attempts the QuestDB warm-up and replays it through the same state + // machine before applying the live tick; the flag is set before the + // query so a DB failure is a cold start, never a retry storm. + void update(const PriceData& tick) { + if (!prepopulateAttempted_) { + prepopulateAttempted_ = true; + for (const PriceData& seed : + prepopulateTicks(tick.symbol, tick.timestamp, spec_)) { + applyTick(seed); + } + } + applyTick(tick); + } + + // Chronological; see the class comment for the completion nuance. + [[nodiscard]] const std::vector& bars() const { return bars_; } + + [[nodiscard]] const RangeBarSpec& spec() const { return spec_; } + + // The window has seen a full atrTickWindow of ticks; bars only form from + // here on. Exposed (with the two below) for tests and diagnostics. + [[nodiscard]] bool warm() const { + return tickIndex_ >= static_cast(spec_.atrTickWindow); + } + + // Rolling high - low over the last atrTickWindow ticks (fewer while + // warming). 0 before any tick. + [[nodiscard]] std::int32_t windowRange() const { + if (maxDeque_.empty()) { + return 0; + } + return maxDeque_.front().price - minDeque_.front().price; + } + + // The in-progress bar's locked threshold; 0 before the first bar opens. + [[nodiscard]] std::int32_t lockedThreshold() const { return lockedThreshold_; } + +private: + struct Entry { + std::uint64_t idx; + std::int32_t price; + }; + + // The pure state machine — replayed warm-up ticks and live ticks take + // exactly this path, so the two cannot diverge. Order matters: the + // window advances FIRST, so a bar opening on tick T locks a threshold + // that already includes T's own contribution (the same "the tick's own + // price counts" doctrine as the run loop's bars-before-decide ordering). + void applyTick(const PriceData& tick) { + const std::int32_t price = tick.ask; + ++tickIndex_; + + // (1) Monotonic deques: max non-increasing, min non-decreasing. + // Popping equals keeps the newer index, which survives expiry longer. + while (!maxDeque_.empty() && maxDeque_.back().price <= price) { + maxDeque_.pop_back(); + } + maxDeque_.push_back({tickIndex_, price}); + while (!minDeque_.empty() && minDeque_.back().price >= price) { + minDeque_.pop_back(); + } + minDeque_.push_back({tickIndex_, price}); + // Expire entries that fell out of the last-atrTickWindow window: + // valid indices are (tickIndex - window, tickIndex]. + const auto window = static_cast(spec_.atrTickWindow); + while (maxDeque_.front().idx + window <= tickIndex_) { + maxDeque_.pop_front(); + } + while (minDeque_.front().idx + window <= tickIndex_) { + minDeque_.pop_front(); + } + + // (2) Bar logic. Pre-warm ticks feed only the window — no bar exists + // yet, so every bar's threshold locks from a fully-warm measure. + if (!warm()) { + return; + } + if (bars_.empty() || bars_.back().complete) { + // Open at this tick and lock its threshold NOW. A fresh bar can + // never be born complete: its range is 0 and the floor keeps the + // threshold >= 1 point even when windowRange * pct rounds to 0 + // (the flat-market degenerate case) — so a dead-flat stream holds + // one in-progress bar instead of rolling a bar per tick. + const std::int64_t scaled = + (static_cast(windowRange()) * spec_.atrPercent + 50) / + 100; + lockedThreshold_ = + static_cast(std::max(1, scaled)); + bars_.push_back({.date = tick.timestamp, + .open = price, + .close = price, + .high = price, + .low = price}); + } else { + OhlcObject& bar = bars_.back(); + if (price > bar.high) { + bar.high = price; + } + if (price < bar.low) { + bar.low = price; + } + bar.close = price; + if (bar.high - bar.low >= lockedThreshold_) { + // Breaching tick included: the close is this real price and + // any overshoot past the threshold is kept. The next tick + // opens the successor, so a gap crossing ten thresholds + // still closes exactly one bar. + bar.complete = true; + } + } + + // (3) Rolling window of bars: oldest dropped from the front. Never + // touches the deques — the measure and the history age independently. + const auto cap = static_cast(spec_.count); + if (bars_.size() > cap) { + bars_.erase(bars_.begin(), + bars_.end() - static_cast(cap)); + } + } + + RangeBarSpec spec_; + std::vector bars_; + std::deque maxDeque_; + std::deque minDeque_; + std::uint64_t tickIndex_ = 0; // 1-based after the first tick + std::int32_t lockedThreshold_ = 0; + bool prepopulateAttempted_ = false; +}; + +} // namespace rangebar diff --git a/source/shared/utilities/swingPivots.cppm b/source/shared/utilities/swingPivots.cppm new file mode 100644 index 0000000..9c5f372 --- /dev/null +++ b/source/shared/utilities/swingPivots.cppm @@ -0,0 +1,74 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// swingPivots — fractal swing high/low detection over OHLC bars of scaled +// INT32 points. +// +// A swing high at index i is a bar whose high STRICTLY exceeds the highs of +// the pivotBars bars on each side (the classic fractal definition; a swing +// low mirrors it on the lows). Ties disqualify — the squeeze breakout's +// "ties lose" doctrine: an equalled extreme is no fresh extreme, and the +// strict inequality also guarantees no bar inside the right wing has reached +// the level, which sweep-detection callers rely on (the first later touch of +// the level must sit BEYOND the confirmation wing). +// +// The predicates are deliberately position-based and allocation-free (the +// SqueezeBreakoutStrategy::isPatternAt shape) so they can run on the per-tick +// hot path: callers scan their own closed-bars window and ask about one index +// at a time. Feed CLOSED bars only — a pivot does not exist until the +// pivotBars bars of its right wing have all closed; passing a span whose last +// element is still in progress would confirm pivots against a moving bar +// (lookahead). Pure integer comparisons throughout. +export module swingPivots; + +import std; // replaces , , +import ohlcObject; // OhlcObject bar record + +export namespace swing_pivots { + +// True when the bar at `index` is a swing high: its high strictly exceeds +// the highs of the pivotBars bars on each side. +// +// Precondition (the caller's window-minimum check proves it, the +// SqueezeBreakoutStrategy convention): index >= pivotBars and +// index + pivotBars < bars.size(). Out-of-range reads are the caller's bug; +// only pivotBars itself is validated — < 1 throws std::invalid_argument +// (atr::calculate's convention), since a wingless "pivot" would match every +// bar. +[[nodiscard]] bool isSwingHighAt(std::span bars, + std::size_t index, int pivotBars) { + if (pivotBars < 1) { + throw std::invalid_argument( + "swing_pivots::isSwingHighAt: pivotBars must be >= 1"); + } + const std::int32_t high = bars[index].high; + for (std::size_t wing = 1; wing <= static_cast(pivotBars); + ++wing) { + if (bars[index - wing].high >= high || bars[index + wing].high >= high) { + return false; + } + } + return true; +} + +// Mirror of isSwingHighAt on the lows: strictly below both wings. +[[nodiscard]] bool isSwingLowAt(std::span bars, + std::size_t index, int pivotBars) { + if (pivotBars < 1) { + throw std::invalid_argument( + "swing_pivots::isSwingLowAt: pivotBars must be >= 1"); + } + const std::int32_t low = bars[index].low; + for (std::size_t wing = 1; wing <= static_cast(pivotBars); + ++wing) { + if (bars[index - wing].low <= low || bars[index + wing].low <= low) { + return false; + } + } + return true; +} + +} // namespace swing_pivots diff --git a/source/strategies/conditions/entryConditions.cppm b/source/strategies/conditions/entryConditions.cppm new file mode 100644 index 0000000..f8be77e --- /dev/null +++ b/source/strategies/conditions/entryConditions.cppm @@ -0,0 +1,160 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// entryConditions — the shared pre-decide entry gate, run identically by the +// backtest loop (runLoop) and the live runner (liveStrategyRunner) so the two +// paths can never diverge on when an entry is allowed or how far its exits +// sit. Port of the C# ATR gate that replaced fixed pip distances. +// +// The trading variables STOP_DISTANCE_IN_ATR / LIMIT_DISTANCE_IN_ATR are +// small integer ATR multipliers, not pips. Per entry attempt, check(): +// +// 1. computes ATR(kAtrPeriod) over the gate timeframe's bars (BarStore), +// 2. rejects the entry when the current spread exceeds 30% of the ATR — +// a spread that wide eats the edge before the trade starts, +// 3. turns the multipliers into pip distances (ATR x multiplier), +// 4. rejects when volatility is too low for the floors (a sub-10-pip stop +// is inside broker noise), and clamps the rest to sane bounds. +// +// nullopt = skip this entry (never deferred, same doctrine as the caps). The +// floors and clamps live HERE, not at the call sites, so backtest and live +// stay in lockstep by construction. +// +// All integer arithmetic: bar prices and ticks are scaled INT32 points, the +// spread test cross-multiplies instead of dividing, and the floors compare in +// points (pips x points-per-pip) so truncation in the final pip division can +// never disagree with the floor that admitted the entry. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module entryConditions; + +import std; // replaces , , , , + // +import atr; // atr::calculate — integer ATR in points +import barStore; // bars::BarStore / bars::SeriesSpec +import ohlcObject; // OhlcObject bar record +import priceData; // PriceData +import symbolScale; // symbol_scale::get — points-per-pip + +namespace { + +// TODO customise: ATR period and the gate's fallback timeframe are fixed for +// now; revisit once results justify sweeping them (see the sweep configs). +constexpr int kAtrPeriod = 10; + +// Fallback gate timeframe for strategies that declare no OHLC bars +// (RandomStrategy): 15-minute bars put ATR(10) on FX majors around 8-15 pips +// — the band where multipliers {1,2} clear the stop floor below. +constexpr std::chrono::minutes kFallbackBarMinutes{15}; + +// TODO customise: max spread as a fraction of ATR, currently 3/10 = 30%. +// Cross-multiplied in check() so the test stays integer. +constexpr std::int64_t kMaxSpreadNum = 3; +constexpr std::int64_t kMaxSpreadDen = 10; + +// Safety bounds on the dynamic distances, in whole pips. Below the floor the +// volatility is too low to trade (the +/-10-pip broker-noise issue); above +// the cap the ATR blew out and the exposure would be silly. +constexpr std::int32_t kMinStopPips = 10; +constexpr std::int32_t kMaxStopPips = 80; +constexpr std::int32_t kMinLimitPips = 3; +constexpr std::int32_t kMaxLimitPips = 300; + +} // namespace + +export namespace conditions { + +// Dynamic exit distances for one entry attempt, in whole pips — the same +// unit TradeManager::openTrade and OrderIntent already carry, so everything +// downstream of the gate is unchanged. +struct Distances { + std::int32_t stopPips; + std::int32_t limitPips; +}; + +// The bar series the ATR gate reads for a given strategy config: the +// PRIMARY (first) OHLC timeframe — for OhlcBreakoutStrategy that is the +// breakout timeframe — widened to at least kAtrPeriod+1 bars so ATR can +// always warm; or the 15-minute fallback when the strategy builds no bars. +// Register the result on the run's BarStore alongside the strategy's own +// series. +[[nodiscard]] bars::SeriesSpec gateSeriesFor( + const tradingDefinitions::StrategyConfig& config) { + if (!config.OHLC_VARIABLES.empty()) { + const tradingDefinitions::OHLCVariables& primary = + config.OHLC_VARIABLES.front(); + // {0,0} is the documented "builds no bars" sentinel (RandomStrategy + // configs carry one — see ohlcVariables.hpp), not a usable timeframe. + if (primary.OHLC_MINUTES >= 1) { + return {std::chrono::minutes{primary.OHLC_MINUTES}, + std::max(primary.OHLC_COUNT, kAtrPeriod + 1)}; + } + } + return {kFallbackBarMinutes, kAtrPeriod + 1}; +} + +// The gate. nullopt = skip this entry: unknown symbol, gate series not warm +// (or dead flat), spread wider than 30% of ATR, or volatility below the pip +// floors. Otherwise the clamped dynamic distances. A multiplier <= 0 yields +// a distance below its floor, so such a config never trades — loud in the +// results rather than silently trading without a stop. +[[nodiscard]] std::optional check(const bars::BarStore& store, + const bars::SeriesSpec& gateSeries, + const PriceData& tick, + const std::int32_t stopMultiplier, + const std::int32_t limitMultiplier) { + const int pointsPerPip = symbol_scale::get(tick.symbol); + if (pointsPerPip == symbol_scale::kUnknown) { + return std::nullopt; // cannot convert points to pips — never trade + } + + const std::vector* candles = + store.find(tick.symbol, gateSeries.minutes); + if (candles == nullptr) { + return std::nullopt; // series unregistered / symbol not ticked yet + } + const std::int64_t atrPoints = atr::calculate(*candles, kAtrPeriod); + if (atrPoints <= 0) { + return std::nullopt; // not warm, or a dead-flat market + } + + // Spread gate, in points: spread > 30% of ATR <=> spread*10 > ATR*3. + // Converting both sides to pips first would just divide both by + // pointsPerPip (and truncate), so the test stays in points. + const std::int64_t spreadPoints = + static_cast(tick.ask) - tick.bid; + if (spreadPoints * kMaxSpreadDen > atrPoints * kMaxSpreadNum) { + return std::nullopt; + } + + // Dynamic distances in points. Overflow headroom: ATR < 2^24 points and + // the multipliers are single digits, so the products sit far inside + // int64 (and inside int32 after the pip division below). + const std::int64_t stopPointsRaw = atrPoints * stopMultiplier; + const std::int64_t limitPointsRaw = atrPoints * limitMultiplier; + + // Volatility floors, compared in points (pips x pointsPerPip): passing + // "raw >= floor * ppp" guarantees the truncated pip division below still + // yields at least the floor. + if (stopPointsRaw < std::int64_t{kMinStopPips} * pointsPerPip || + limitPointsRaw < std::int64_t{kMinLimitPips} * pointsPerPip) { + return std::nullopt; + } + + return Distances{ + .stopPips = std::clamp( + static_cast(stopPointsRaw / pointsPerPip), + kMinStopPips, kMaxStopPips), + .limitPips = std::clamp( + static_cast(limitPointsRaw / pointsPerPip), + kMinLimitPips, kMaxLimitPips), + }; +} + +} // namespace conditions diff --git a/source/strategies/fvg/fvgStrategy.cppm b/source/strategies/fvg/fvgStrategy.cppm new file mode 100644 index 0000000..173d908 --- /dev/null +++ b/source/strategies/fvg/fvgStrategy.cppm @@ -0,0 +1,295 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// FvgStrategy — Fair Value Gap retracement entries with a higher-timeframe +// SMA trend filter. Port of the C# FVG strategy; semantics preserved exactly +// (both codebases keep the in-progress bar as the LAST list element, so the +// index conventions map 1:1). +// +// Two OHLC timeframes are read per symbol from the loop owner's shared +// BarStore (built from the ask — see barStore): +// +// OHLC_VARIABLES[0] — the FVG timeframe. Closed 3-bar patterns (c1,c2,c3) +// are scanned newest-first for an unmitigated gap: +// bullish when c1.high sits >= the minimum gap below +// c3.low (bearish mirrored). A gap is mitigated when +// any LATER closed bar traded through it entirely +// (bullish: low <= c1.high) — partial fills don't count. +// OHLC_VARIABLES[1] — the HTF trend timeframe. The SMA of its last +// HTF_SMA_PERIOD CLOSED closes vs the last closed close +// gates direction: only longs in an uptrend, only +// shorts in a downtrend. +// +// Entry: in an HTF uptrend, when the ask retraces INTO a bullish gap +// [c1.high, c3.low] (bounds inclusive) whose last closed bar still closed +// above it -> LONG; mirror with the bid for SHORT. Exits stay fully central +// (ATR-derived SL/TP enforced by Operations) — during() is a no-op. +// +// MIN_GAP_PIPS is in PIPS, converted to points per symbol at decide() time +// via symbol_scale::get (the BUFFER_PIPS pattern from OhlcBreakoutStrategy), +// so one value means the same real-world size on every symbol scale — the +// property that lets the sweep run beyond EURUSD. +// +// MIN_GAP_AGE_BARS behaves as a MAXIMUM age despite its name (preserved from +// the C#): 0 examines only the newest closed pattern, larger values widen the +// scan backward, clamped by LOOKBACK_BARS. Two more quirks carried over +// deliberately: LOOKBACK_BARS = 1 is legal but dead (the scan window is +// empty), and the last-closed-outside-the-gap check is nearly vacuous for the +// newest pattern (the last closed bar IS c3, so it only fails when c3 closes +// exactly on the gap edge) — it bites for aged gaps, where it stops re-entry +// into stagnant ones. +// +// The same structural notes as OhlcBreakoutStrategy apply: the store updates +// BEFORE decide() (the in-progress last bar's close IS the current tick, and +// a bar-rolling tick promotes the previous in-progress bar into the closed +// window one tick sooner than the C#), the store may hold a deeper window +// than OHLC_COUNT when the ATR gate registered one on the same timeframe +// (decide() reads only its own tail), and one instance sees every symbol +// interleaved (per-symbol state lives in the BarStore). The signal re-fires +// every tick while price sits in the gap; the run loop's one-trade-per-symbol +// gate prevents stacking. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module fvgStrategy; + +import std; // replaces , , , , + // , , , +import strategy; // IStrategy base class +import barStore; // bars::BarStore — shared per-symbol bar histories +import priceData; // PriceData +import trade; // Direction +import tradeManager; // TradeManager +import timeCapExit; // strategy_exits::closeIfPastCap — the shared time cap +import ohlcObject; // OhlcObject bar record +import symbolScale; // symbol_scale::get — points-per-pip for the gap floor + +export class FvgStrategy : public IStrategy { +public: + // Validates the config up front and throws std::invalid_argument on a + // malformed one (missing timeframes / variables, windows too small for + // the scan) — a misconfigured run should die loudly at construction, not + // trade silently wrong. + explicit FvgStrategy(const tradingDefinitions::StrategyConfig& strategyConfig); + + std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) override; + + // Strategy-driven exit hook: the optional time cap (strategy_exits:: + // closeIfPastCap). SL/TP exits remain central (ATR-derived, enforced by + // Operations / exit_rules). + void during(const PriceData& price, const bars::BarStore& barStore, + TradeManager& tradeManager) override; + +private: + // `{}` value-initializes: OHLCVariables is an aggregate of plain ints with + // no defaults of its own, so without this the fields would hold + // indeterminate values until the constructor body assigns them (reading + // one before that is undefined behaviour). The constructor can't use a + // member-init list here because it must validate the config first. + tradingDefinitions::OHLCVariables fvgCfg{}; + tradingDefinitions::OHLCVariables htfCfg{}; + int lookbackBars{}; + std::int32_t minGapPips{}; // pips; decide() converts per symbol + int htfSmaPeriod{}; + int minGapAgeBars{}; + std::chrono::minutes maxTradeDuration{}; // <= 0 disables the time cap +}; + +// Config fields are assigned in the body, not a member-init list: the size +// check must run first to throw a descriptive error (an init list would have +// to use ohlcVars.at(0), dying with an unhelpful out_of_range instead). Their +// in-class {} initializers keep them defined in the meantime. +FvgStrategy::FvgStrategy(const tradingDefinitions::StrategyConfig& strategyConfig) { + + const auto& ohlcVars = strategyConfig.OHLC_VARIABLES; + + if (ohlcVars.size() < 2) { + throw std::invalid_argument( + "FvgStrategy: OHLC_VARIABLES needs two entries " + "(FVG timeframe, HTF trend timeframe)"); + } + + fvgCfg = ohlcVars[0]; + htfCfg = ohlcVars[1]; + + for (const auto& cfg : {fvgCfg, htfCfg}) { + if (cfg.OHLC_MINUTES < 1) { + throw std::invalid_argument("FvgStrategy: OHLC_MINUTES must be >= 1"); + } + } + + const auto& fvgVars = strategyConfig.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES; + if (!fvgVars) { + throw std::invalid_argument( + "FvgStrategy: STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES is required " + "(LOOKBACK_BARS, MIN_GAP_PIPS, HTF_SMA_PERIOD)"); + } + lookbackBars = fvgVars->LOOKBACK_BARS; + minGapPips = fvgVars->MIN_GAP_PIPS; + htfSmaPeriod = fvgVars->HTF_SMA_PERIOD; + minGapAgeBars = fvgVars->MIN_GAP_AGE_BARS; + + if (lookbackBars < 1 || minGapPips < 1 || htfSmaPeriod < 1) { + throw std::invalid_argument( + "FvgStrategy: LOOKBACK_BARS, MIN_GAP_PIPS and HTF_SMA_PERIOD " + "must be >= 1"); + } + if (minGapAgeBars < 0) { + throw std::invalid_argument("FvgStrategy: MIN_GAP_AGE_BARS must be >= 0"); + } + if (fvgVars->MAX_TRADE_DURATION_MINUTES < 0) { + throw std::invalid_argument( + "FvgStrategy: MAX_TRADE_DURATION_MINUTES must be >= 0"); + } + maxTradeDuration = + std::chrono::minutes{fvgVars->MAX_TRADE_DURATION_MINUTES}; + // Window minimums: once decide()'s warm-up gate passes, both spans are + // exactly OHLC_COUNT long, so these make the C# per-call size checks + // ("Count < lookbackBars + 3", "Count < htfSmaPeriod + 2") structurally + // impossible and prove every index in the scan in range. + if (fvgCfg.OHLC_COUNT < lookbackBars + 3) { + throw std::invalid_argument( + "FvgStrategy: OHLC_VARIABLES[0].OHLC_COUNT must be >= LOOKBACK_BARS + 3"); + } + if (htfCfg.OHLC_COUNT < htfSmaPeriod + 2) { + throw std::invalid_argument( + "FvgStrategy: OHLC_VARIABLES[1].OHLC_COUNT must be >= HTF_SMA_PERIOD + 2"); + } +} + +void FvgStrategy::during(const PriceData& price, const bars::BarStore&, + TradeManager& tradeManager) { + // Time cap (shared closeIfPastCap mechanics): close this symbol's trade + // once open STRICTLY longer than maxTradeDuration; <= 0 disables and + // preserves the original exits-are-central no-op behaviour. + strategy_exits::closeIfPastCap(price, tradeManager, maxTradeDuration); +} + +std::optional FvgStrategy::decide(const PriceData& tick, + const bars::BarStore& barStore) { + // This symbol's bar histories; nullptr means the timeframe was never + // registered on the store or the symbol has not ticked yet — either way + // there is nothing to decide on. + const std::vector* fvgSeries = + barStore.find(tick.symbol, std::chrono::minutes{fvgCfg.OHLC_MINUTES}); + const std::vector* htfSeries = + barStore.find(tick.symbol, std::chrono::minutes{htfCfg.OHLC_MINUTES}); + if (fvgSeries == nullptr || htfSeries == nullptr) { + return std::nullopt; + } + + // Warm-up gate: no signals until both timeframes have a full window. The + // ctor guarantees the windows cover the scan (OHLC_COUNT >= + // LOOKBACK_BARS + 3 / HTF_SMA_PERIOD + 2), so the C# per-call size checks + // are dropped as redundant. + const auto fvgCount = static_cast(fvgCfg.OHLC_COUNT); + const auto htfCount = static_cast(htfCfg.OHLC_COUNT); + if (fvgSeries->size() < fvgCount || htfSeries->size() < htfCount) { + return std::nullopt; + } + + // Read only this strategy's tail of each history: the store keeps the + // LARGEST window registered per timeframe, so another consumer (the ATR + // entry gate) may have deepened a series beyond OHLC_COUNT. + const std::span fvgBars = std::span(*fvgSeries).last(fvgCount); + const std::span htfBars = std::span(*htfSeries).last(htfCount); + + // Signed indices throughout: totalBars - 2 - minGapAgeBars can go + // negative before the max() clamp, which size_t would wrap into a + // scan-skipping garbage bound. + const auto totalBars = static_cast(fvgBars.size()); + const auto htfTotalBars = static_cast(htfBars.size()); + + // --- 1. HTF TREND FILTER --- + // SMA of the last htfSmaPeriod CLOSED closes (the last element is the + // in-progress bar, so the newest closed bar sits at htfTotalBars - 2 — + // the C# [htfTotalBars - 2] convention). Sum in int64: closes are int32 + // points and scaled index/metal closes run into the millions, so an int32 + // sum could overflow within a few hundred bars. + const std::ptrdiff_t htfStartIdx = htfTotalBars - 2; + std::int64_t htfCloseSum = 0; + for (std::ptrdiff_t i = htfStartIdx; i > htfStartIdx - htfSmaPeriod; --i) { + htfCloseSum += htfBars[static_cast(i)].close; + } + // Trend without floating point OR truncating division: close > sum/period + // <=> close * period > sum exactly (period > 0), reproducing the C# + // decimal comparison bit-for-bit; a truncated sum/period would misread + // closes sitting between the truncated and the true SMA. + const std::int64_t lastHtfCloseScaled = + std::int64_t{htfBars[static_cast(htfStartIdx)].close} * + htfSmaPeriod; + const bool isHtfUptrend = lastHtfCloseScaled > htfCloseSum; + const bool isHtfDowntrend = lastHtfCloseScaled < htfCloseSum; + + // --- 2. FVG SCAN --- + // Closed 3-bar patterns (c1,c2,c3 at i-2,i-1,i), newest-first. The window + // is bounded below by LOOKBACK_BARS and tightened to the newest + // MIN_GAP_AGE_BARS + 1 patterns (0 = newest only). The ctor's window + // minimum keeps every index >= 0: totalBars >= lookbackBars + 3 >= 4, so + // endSearchIdx >= 2 and i - 2 >= 0. + // + // The gap floor is configured in pips; bar prices are integer points, so + // convert per symbol here (same idiom as OhlcBreakoutStrategy's buffer). + // In production an unknown scale never reaches decide() — the ATR entry + // gate rejects those symbols first. + const std::int32_t minGapPoints = minGapPips * symbol_scale::get(tick.symbol); + const std::int32_t lastClosedClose = + fvgBars[static_cast(totalBars - 2)].close; + const std::ptrdiff_t startSearchIdx = totalBars - 2; + const std::ptrdiff_t endSearchIdx = + std::max(2, totalBars - lookbackBars); + const std::ptrdiff_t maxAgeIdx = + std::max(endSearchIdx, totalBars - 2 - minGapAgeBars); + + for (std::ptrdiff_t i = startSearchIdx; i >= maxAgeIdx; --i) { + const OhlcObject& c3 = fvgBars[static_cast(i)]; + // c2 (fvgBars[i - 1]) is the displacement bar between c1 and c3; the + // rule reads only c1/c3, so it is never touched (same as the C#). + const OhlcObject& c1 = fvgBars[static_cast(i - 2)]; + + // BULLISH gap: c1's high sits >= minGapPoints below c3's low. Both + // pattern conditions can't hold for one i (that would need c1.high < + // c1.low), so branch order is irrelevant. + if (isHtfUptrend && c1.high < c3.low && c3.low - c1.high >= minGapPoints) { + // Mitigated when ANY strictly-later CLOSED bar traded down + // through the whole gap (low <= c1.high) — a partial fill leaves + // the gap live. Empty range when i is the newest pattern. + bool mitigated = false; + for (std::ptrdiff_t j = i + 1; j <= totalBars - 2; ++j) { + if (fvgBars[static_cast(j)].low <= c1.high) { + mitigated = true; + break; + } + } + // Entry: the last closed bar still closed above the gap (stops + // re-entering stagnant aged gaps) and the ask has retraced INTO + // it, both bounds inclusive. + if (!mitigated && lastClosedClose > c3.low && + tick.ask <= c3.low && tick.ask >= c1.high) { + return Direction::LONG; + } + } + + // BEARISH mirror: c1's low sits >= minGapPoints above c3's high. + if (isHtfDowntrend && c1.low > c3.high && c1.low - c3.high >= minGapPoints) { + bool mitigated = false; + for (std::ptrdiff_t j = i + 1; j <= totalBars - 2; ++j) { + if (fvgBars[static_cast(j)].high >= c1.low) { + mitigated = true; + break; + } + } + if (!mitigated && lastClosedClose < c3.high && + tick.bid >= c3.high && tick.bid <= c1.low) { + return Direction::SHORT; + } + } + } + return std::nullopt; +} diff --git a/source/strategies/keltnerFade/keltnerFadeStrategy.cppm b/source/strategies/keltnerFade/keltnerFadeStrategy.cppm new file mode 100644 index 0000000..eb6b75f --- /dev/null +++ b/source/strategies/keltnerFade/keltnerFadeStrategy.cppm @@ -0,0 +1,216 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// KeltnerFadeStrategy — mean reversion: fade stretches beyond a volatility +// band back toward the mean. The first strategy in the book that PROFITS from +// chop rather than needing a trend, so it is deliberately anti-correlated +// with OhlcBreakoutStrategy / FvgStrategy (both trend-continuation). +// +// One OHLC timeframe is read per symbol from the loop owner's shared BarStore +// (built from the ask — see barStore): +// +// OHLC_VARIABLES[0] — the signal timeframe. The band centre is the SMA of +// its last BAND_SMA_PERIOD CLOSED closes; the half-width +// is BAND_ATR_MULT_TENTHS x ATR / 10, with the ATR taken +// over the same closed bars (Keltner-style bands: SMA +// +/- k x ATR — chosen over Bollinger because it needs +// no integer sqrt). +// +// Entry: the ask stretched STRICTLY below the lower band -> LONG (fade the +// down-move); the bid strictly above the upper band -> SHORT. No trend +// filter: the hypothesis under test is pure reversion to the mean. Exits stay +// central (ATR-derived SL/TP enforced by Operations); the one strategy-driven +// exit is the optional time cap copied from SessionRangeBreakoutStrategy: +// when MAX_TRADE_DURATION_MINUTES > 0, during() closes the symbol's trade +// once open strictly longer than that, at the exit-side price. For a fade +// the cap is the thesis clock — a stretch that has not snapped back within +// the window is a failed reversion, not a position to sit in. Note the sweep +// inverts the usual exit shape: reversion wants the limit NEARER than the +// stop (take the snap-back, survive the excursion). +// +// The band uses CLOSED bars only. The store updates BEFORE decide(), so the +// in-progress last bar's close IS the tick being judged — including it in the +// SMA would drag the band centre toward the very stretch being faded and +// systematically weaken the signal. Excluding the in-progress bar keeps the +// band fixed between bar rolls, mirroring how FvgStrategy reads only closed +// patterns. +// +// All arithmetic is integer: the band comparison is cross-multiplied +// (price x period x 10 vs closeSum x 10 -/+ mult x ATR x period) so no +// truncating division ever moves the band edge — same doctrine as the FVG +// trend filter. The signal re-fires every tick while price sits outside the +// band; the run loop's one-trade-per-symbol gate prevents stacking. +// +// The same structural notes as the other strategies apply: the strategy owns +// no bar state, one instance sees every symbol interleaved (per-symbol state +// lives in the BarStore), and the store may hold a deeper window than +// OHLC_COUNT when the ATR entry gate registered one on the same timeframe — +// decide() reads only its own tail. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module keltnerFadeStrategy; + +import std; // replaces , , , , + // , , +import strategy; // IStrategy base class +import barStore; // bars::BarStore — shared per-symbol bar histories +import priceData; // PriceData +import trade; // Direction +import tradeManager; // TradeManager +import timeCapExit; // strategy_exits::closeIfPastCap — the shared time cap +import ohlcObject; // OhlcObject bar record +import atr; // atr::calculate — integer ATR for the band width + +export class KeltnerFadeStrategy : public IStrategy { +public: + // Validates the config up front and throws std::invalid_argument on a + // malformed one (missing timeframe / variables, window too small for the + // band) — a misconfigured run should die loudly at construction, not + // trade silently wrong. + explicit KeltnerFadeStrategy(const tradingDefinitions::StrategyConfig& strategyConfig); + + std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) override; + + // Strategy-driven exit hook: the optional time cap (see header). SL/TP + // exits remain central. + void during(const PriceData& price, const bars::BarStore& barStore, + TradeManager& tradeManager) override; + +private: + // `{}` value-initializes: OHLCVariables is an aggregate of plain ints with + // no defaults of its own, so without this the fields would hold + // indeterminate values until the constructor body assigns them (reading + // one before that is undefined behaviour). The constructor can't use a + // member-init list here because it must validate the config first. + tradingDefinitions::OHLCVariables signalCfg{}; + int bandSmaPeriod{}; + int bandAtrMultTenths{}; + std::chrono::minutes maxTradeDuration{}; // <= 0 disables the time cap +}; + +// Config fields are assigned in the body, not a member-init list: the size +// check must run first to throw a descriptive error (an init list would have +// to use ohlcVars.at(0), dying with an unhelpful out_of_range instead). Their +// in-class {} initializers keep them defined in the meantime. +KeltnerFadeStrategy::KeltnerFadeStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig) { + + const auto& ohlcVars = strategyConfig.OHLC_VARIABLES; + + if (ohlcVars.empty()) { + throw std::invalid_argument( + "KeltnerFadeStrategy: OHLC_VARIABLES needs one entry " + "(the signal timeframe)"); + } + + signalCfg = ohlcVars[0]; + + if (signalCfg.OHLC_MINUTES < 1) { + throw std::invalid_argument("KeltnerFadeStrategy: OHLC_MINUTES must be >= 1"); + } + + const auto& fadeVars = strategyConfig.STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES; + if (!fadeVars) { + throw std::invalid_argument( + "KeltnerFadeStrategy: STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES is " + "required (BAND_SMA_PERIOD, BAND_ATR_MULT_TENTHS)"); + } + bandSmaPeriod = fadeVars->BAND_SMA_PERIOD; + bandAtrMultTenths = fadeVars->BAND_ATR_MULT_TENTHS; + maxTradeDuration = + std::chrono::minutes{fadeVars->MAX_TRADE_DURATION_MINUTES}; + + if (bandSmaPeriod < 1 || bandAtrMultTenths < 1) { + throw std::invalid_argument( + "KeltnerFadeStrategy: BAND_SMA_PERIOD and BAND_ATR_MULT_TENTHS " + "must be >= 1"); + } + // Window minimum: once decide()'s warm-up gate passes the span is exactly + // OHLC_COUNT long, whose last element is the in-progress bar. The closed + // remainder must cover both the SMA (period bars) and the ATR (period + 1 + // bars — each TR needs its predecessor's close), so OHLC_COUNT >= + // period + 2 proves every read in range and the ATR warm. + if (signalCfg.OHLC_COUNT < bandSmaPeriod + 2) { + throw std::invalid_argument( + "KeltnerFadeStrategy: OHLC_VARIABLES[0].OHLC_COUNT must be >= " + "BAND_SMA_PERIOD + 2"); + } +} + +void KeltnerFadeStrategy::during(const PriceData& price, + const bars::BarStore& /*barStore*/, + TradeManager& tradeManager) { + // Time cap (shared closeIfPastCap mechanics): close this symbol's trade + // once open STRICTLY longer than maxTradeDuration; <= 0 disables. For a + // fade the cap is the thesis clock — a stretch that has not snapped back + // within the window is a failed reversion, not a position to sit in. + strategy_exits::closeIfPastCap(price, tradeManager, maxTradeDuration); +} + +std::optional KeltnerFadeStrategy::decide(const PriceData& tick, + const bars::BarStore& barStore) { + // This symbol's bar history; nullptr means the timeframe was never + // registered on the store or the symbol has not ticked yet — either way + // there is nothing to decide on. + const std::vector* signalSeries = barStore.find( + tick.symbol, std::chrono::minutes{signalCfg.OHLC_MINUTES}); + if (signalSeries == nullptr) { + return std::nullopt; + } + + // Warm-up gate: no signals until the timeframe has a full window. + const auto signalCount = static_cast(signalCfg.OHLC_COUNT); + if (signalSeries->size() < signalCount) { + return std::nullopt; + } + + // Read only this strategy's tail of the history (the store keeps the + // LARGEST window registered per timeframe), then drop the in-progress + // last bar: the band is built from CLOSED bars only — see the header. + const std::span signalBars = + std::span(*signalSeries).last(signalCount); + const std::span closedBars = + signalBars.first(signalBars.size() - 1); + + // Band centre: SMA of the last bandSmaPeriod closed closes, kept as an + // int64 sum (closes are int32 points; scaled index/metal closes run into + // the millions, so an int32 sum could overflow within a few hundred bars). + std::int64_t closeSum = 0; + for (std::size_t i = closedBars.size() - static_cast(bandSmaPeriod); + i < closedBars.size(); ++i) { + closeSum += closedBars[i].close; + } + + // Band width: ATR over the same closed bars (only the final period + 1 + // are read). 0 means a dead-flat market — a zero-width band would fade + // every tick of noise, so treat it as untradeable, matching the ATR entry + // gate's convention. + const std::int32_t atrPoints = atr::calculate(closedBars, bandSmaPeriod); + if (atrPoints <= 0) { + return std::nullopt; + } + + // Fade a STRICT stretch beyond the band. Without floating point OR + // truncating division: price < SMA - mult/10 x ATR + // <=> price x period x 10 < closeSum x 10 - mult x ATR x period exactly + // (period > 0). Entries judge the trade's own fill side: a LONG buys the + // ask, a SHORT sells the bid — the spread must not flatter the stretch. + const std::int64_t bandOffset = + std::int64_t{bandAtrMultTenths} * atrPoints * bandSmaPeriod; + const std::int64_t centreScaled = closeSum * 10; + + if (std::int64_t{tick.ask} * bandSmaPeriod * 10 < centreScaled - bandOffset) { + return Direction::LONG; + } + if (std::int64_t{tick.bid} * bandSmaPeriod * 10 > centreScaled + bandOffset) { + return Direction::SHORT; + } + return std::nullopt; +} diff --git a/source/strategies/liquiditySweepReversal/liquiditySweepReversalStrategy.cppm b/source/strategies/liquiditySweepReversal/liquiditySweepReversalStrategy.cppm new file mode 100644 index 0000000..1b4a415 --- /dev/null +++ b/source/strategies/liquiditySweepReversal/liquiditySweepReversalStrategy.cppm @@ -0,0 +1,398 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// LiquiditySweepReversalStrategy — fade the failed breakout: old swing +// highs/lows are where resting stops pool, and a wick that runs through such +// a level but CLOSES back on the original side has taken that liquidity and +// found no follow-through. The rejection is the signal; the trade is the +// reversal. This is the book's first REVERSAL archetype — every winning +// family so far is trend-continuation / volatility-expansion, so this is the +// deliberate regime diversifier, and unlike the naive band fade +// (KeltnerFadeStrategy) the entry demands structural confirmation: a swept +// level AND a rejection, optionally a displacement-sized one. +// +// One OHLC timeframe is read per symbol from the loop owner's shared BarStore +// (built from the ask — see barStore): +// +// OHLC_VARIABLES[0] — the signal timeframe. Swing pivots (fractal extremes, +// PIVOT_BARS closed bars strictly beaten on each side — +// swing_pivots) are scanned over the last LOOKBACK_BARS +// closed bars. For each pivot level, the FIRST later +// closed bar to trade beyond it decides the level once +// and for all: +// - closed beyond it -> breakout, level dead +// - poked < MIN_SWEEP_PIPS -> shallow tap, level dead +// - poked >= MIN_SWEEP_PIPS +// and closed back inside -> the REJECTION bar +// A rejection is tradeable while it sits within the +// last VALID_BARS closed bars (the squeeze breakout's +// freshness idiom) and no later bar has closed beyond +// the level (breakout resumed). When +// DISPLACEMENT_ATR_TENTHS > 0 the rejection bar must +// also be a displacement candle against the sweep: body +// >= tenths x ATR(10) / 10, with the ATR frozen over +// the bars up to and including the rejection — a setup +// that qualified once cannot flicker as newer bars move +// a trailing ATR, and the judgment sees no post-event +// data. 0 disables the gate (and the body-direction +// check), so the sweep itself A/Bs displacement. +// +// Entry: the newest valid rejection wins; an older setup never outranks a +// fresher one, and a single outside bar rejecting BOTH a swept high and a +// swept low is ambiguous — refused. A swept high enters SHORT while the BID +// is back below the level; a swept low enters LONG while the ASK is back +// above it — entries judge the trade's own fill side (the KeltnerFade +// doctrine: the spread must not flatter the setup). No trend filter: the +// hypothesis under test is the failed breakout itself. +// +// First-touch-decides makes level consumption derivable from the bars alone — +// the strategy owns no per-symbol state (per-symbol state lives in the +// BarStore), so a shallow tap or a breakout permanently retires a level +// without anything to remember: re-scanning reaches the same verdict every +// tick. There is never a "second sweep" of the same level. +// +// Exits stay fully central (ATR-derived SL/TP enforced by Operations) — +// during() is a no-op, same doctrine as FvgStrategy / KeltnerFadeStrategy. +// Note the sweep inverts the usual exit shape, exactly like KeltnerFade: +// reversion wants the limit NEARER than the stop (take the snap-back, +// survive the excursion). +// +// All arithmetic is integer: prices are scaled int32 points (< 2^23), bodies +// and ATRs are < 2^24, and every product is computed in int64 (body x 10, +// tenths x ATR — comfortable headroom, the atr module's convention). The +// same structural notes as the other strategies apply: the store updates +// BEFORE decide(), one instance sees every symbol interleaved, the store may +// hold a deeper window than OHLC_COUNT (the ATR entry gate may deepen a +// shared timeframe) so decide() reads only its own tail, and the signal +// re-fires while a valid rejection stands; the run loop's +// one-trade-per-symbol gate prevents stacking. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module liquiditySweepReversalStrategy; + +import std; // replaces , , , , + // , , +import strategy; // IStrategy base class +import barStore; // bars::BarStore — shared per-symbol bar histories +import priceData; // PriceData +import trade; // Direction +import tradeManager; // TradeManager +import timeCapExit; // strategy_exits::closeIfPastCap — the shared time cap +import ohlcObject; // OhlcObject bar record +import atr; // atr::calculate — the displacement yardstick +import swingPivots; // swing_pivots — fractal swing high/low detection +import symbolScale; // symbol_scale::get — points-per-pip for the sweep depth + +export class LiquiditySweepReversalStrategy : public IStrategy { +public: + // Validates the config up front and throws std::invalid_argument on a + // malformed one (missing timeframe / variables, windows too small for the + // pivot scan or a warm displacement ATR) — a misconfigured run should die + // loudly at construction, not trade silently wrong. + explicit LiquiditySweepReversalStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig); + + std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) override; + + // Strategy-driven exit hook: the optional time cap (strategy_exits:: + // closeIfPastCap). SL/TP exits remain central (ATR-derived, enforced by + // Operations / exit_rules). For a reversal the cap is the thesis clock — + // a sweep that has not reverted within the window is a failed setup. + void during(const PriceData& price, const bars::BarStore& barStore, + TradeManager& tradeManager) override; + +private: + // `{}` value-initializes: OHLCVariables is an aggregate of plain ints with + // no defaults of its own, so without this the fields would hold + // indeterminate values until the constructor body assigns them (reading + // one before that is undefined behaviour). The constructor can't use a + // member-init list here because it must validate the config first. + tradingDefinitions::OHLCVariables signalCfg{}; + int pivotBars{}; + int lookbackBars{}; + int minSweepPips{}; + int displacementAtrTenths{}; + int validBars{}; + std::chrono::minutes maxTradeDuration{}; // <= 0 disables the time cap + + // A tradeable rejection: the level that was swept and the closed-bar + // index of the bar that rejected the sweep. Newest rejection wins. + struct Setup { + std::size_t rejection{}; + std::int32_t level{}; + bool found = false; + }; + + // True when the rejection bar at `rejection` clears the displacement + // gate against the sweep direction (bearish body for a swept high, + // bullish for a swept low). Always true when the gate is off. + [[nodiscard]] bool displacementOk(std::span closedBars, + std::size_t rejection, + bool sweptHigh) const; +}; + +namespace { +// The displacement yardstick's period — pinned to the ATR entry gate's so +// "one displacement" and "one stop unit" share a ruler; deliberately not +// swept. +inline constexpr int kAtrPeriod = 10; +} // namespace + +// Config fields are assigned in the body, not a member-init list: the size +// check must run first to throw a descriptive error (an init list would have +// to use ohlcVars.at(0), dying with an unhelpful out_of_range instead). Their +// in-class {} initializers keep them defined in the meantime. +LiquiditySweepReversalStrategy::LiquiditySweepReversalStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig) { + + const auto& ohlcVars = strategyConfig.OHLC_VARIABLES; + + if (ohlcVars.empty()) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: OHLC_VARIABLES needs one entry " + "(the signal timeframe)"); + } + + signalCfg = ohlcVars[0]; + + if (signalCfg.OHLC_MINUTES < 1) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: OHLC_MINUTES must be >= 1"); + } + + const auto& sweepVars = + strategyConfig.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES; + if (!sweepVars) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: " + "STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES is required " + "(PIVOT_BARS, LOOKBACK_BARS, MIN_SWEEP_PIPS, " + "DISPLACEMENT_ATR_TENTHS, VALID_BARS)"); + } + pivotBars = sweepVars->PIVOT_BARS; + lookbackBars = sweepVars->LOOKBACK_BARS; + minSweepPips = sweepVars->MIN_SWEEP_PIPS; + displacementAtrTenths = sweepVars->DISPLACEMENT_ATR_TENTHS; + validBars = sweepVars->VALID_BARS; + + if (pivotBars < 1) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: PIVOT_BARS must be >= 1"); + } + // The lookback must fit at least one pivot plus its right wing, or the + // scan range is empty and the config can NEVER trade. + if (lookbackBars < pivotBars + 1) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: LOOKBACK_BARS must be >= " + "PIVOT_BARS + 1"); + } + if (minSweepPips < 0) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: MIN_SWEEP_PIPS must be >= 0"); + } + if (displacementAtrTenths < 0) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: DISPLACEMENT_ATR_TENTHS must be " + ">= 0"); + } + if (validBars < 1 || validBars > lookbackBars) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: VALID_BARS must be >= 1 and <= " + "LOOKBACK_BARS"); + } + if (sweepVars->MAX_TRADE_DURATION_MINUTES < 0) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: MAX_TRADE_DURATION_MINUTES must " + "be >= 0"); + } + maxTradeDuration = + std::chrono::minutes{sweepVars->MAX_TRADE_DURATION_MINUTES}; + // Window minimum: once decide()'s warm-up gate passes, the signal span is + // exactly OHLC_COUNT long (closed = OHLC_COUNT - 1). The closed remainder + // must cover the full pivot scan (LOOKBACK_BARS of candidates, each + // needing PIVOT_BARS of left wing beyond the window's oldest candidate) + // AND keep the displacement ATR warm at the OLDEST valid rejection + // (kAtrPeriod TRs + the predecessor close, VALID_BARS from the end) — + // so every index the scan touches is proven in range and a rejection + // inside the freshness window can never be refused just for a cold ATR. + const int minCount = std::max(lookbackBars + pivotBars + 1, + validBars + kAtrPeriod + 1); + if (signalCfg.OHLC_COUNT < minCount) { + throw std::invalid_argument( + "LiquiditySweepReversalStrategy: OHLC_VARIABLES[0].OHLC_COUNT must " + "be >= max(LOOKBACK_BARS + PIVOT_BARS + 1, VALID_BARS + 11)"); + } +} + +void LiquiditySweepReversalStrategy::during(const PriceData& price, + const bars::BarStore&, + TradeManager& tradeManager) { + // Time cap (shared closeIfPastCap mechanics): close this symbol's trade + // once open STRICTLY longer than maxTradeDuration; <= 0 disables and + // preserves the original exits-are-central no-op behaviour. + strategy_exits::closeIfPastCap(price, tradeManager, maxTradeDuration); +} + +bool LiquiditySweepReversalStrategy::displacementOk( + std::span closedBars, std::size_t rejection, + bool sweptHigh) const { + if (displacementAtrTenths == 0) { + return true; + } + const OhlcObject& bar = closedBars[rejection]; + // The body must point AGAINST the sweep: bearish off a swept high, + // bullish off a swept low. A doji or wrong-way body is no displacement. + const std::int64_t body = sweptHigh + ? std::int64_t{bar.open} - bar.close + : std::int64_t{bar.close} - bar.open; + if (body <= 0) { + return false; + } + // ATR frozen at the rejection bar: only bars up to and including it are + // read, so the verdict never moves after the event. 0 means not warm / + // dead-flat — the ctor's window minimum keeps every in-window rejection + // warm, so 0 here is a genuinely untradeable market (the ATR entry + // gate's convention). + const std::int32_t atrPoints = + atr::calculate(closedBars.first(rejection + 1), kAtrPeriod); + if (atrPoints <= 0) { + return false; + } + // body >= tenths/10 x ATR without truncating division: + // body x 10 >= tenths x ATR exactly (both sides int64, operands < 2^24). + return body * 10 >= std::int64_t{displacementAtrTenths} * atrPoints; +} + +std::optional LiquiditySweepReversalStrategy::decide( + const PriceData& tick, const bars::BarStore& barStore) { + // This symbol's bar history; nullptr means the timeframe was never + // registered on the store or the symbol has not ticked yet — either way + // there is nothing to decide on. + const std::vector* signalSeries = barStore.find( + tick.symbol, std::chrono::minutes{signalCfg.OHLC_MINUTES}); + if (signalSeries == nullptr) { + return std::nullopt; + } + + // Warm-up gate: no signals until the timeframe has a full window. + const auto signalCount = static_cast(signalCfg.OHLC_COUNT); + if (signalSeries->size() < signalCount) { + return std::nullopt; + } + + // Read only this strategy's tail of the history (the store keeps the + // LARGEST window registered per timeframe), then drop the in-progress + // last bar: pivots, sweeps and rejections are all judged on CLOSED bars + // only — a pivot does not exist until its right wing has closed, and a + // "rejection" whose close is still moving is lookahead. + const std::span signalBars = + std::span(*signalSeries).last(signalCount); + const std::span closedBars = + signalBars.first(signalBars.size() - 1); + const std::size_t n = closedBars.size(); + + // Pips scale UP to points (pips x pointsPerPip), the OhlcBreakoutStrategy + // convention; an unknown symbol returns scale 0, which just relaxes the + // sweep-depth floor rather than corrupting the levels. + const std::int32_t sweepPoints = + minSweepPips * symbol_scale::get(tick.symbol); + + // --- THE PIVOT SCAN --- + // Candidates need PIVOT_BARS of closed wing on each side and must sit + // inside the lookback. The ctor's window minimum proves n >= + // LOOKBACK_BARS + PIVOT_BARS, so the subtraction cannot underflow. + const auto wing = static_cast(pivotBars); + const std::size_t scanStart = + std::max(wing, n - static_cast(lookbackBars)); + Setup bestShort; // newest valid rejection of a swept swing HIGH + Setup bestLong; // newest valid rejection of a swept swing LOW + + for (std::size_t i = scanStart; i + wing < n; ++i) { + // Swept swing high -> SHORT candidate. + if (swing_pivots::isSwingHighAt(closedBars, i, pivotBars)) { + const std::int32_t level = closedBars[i].high; + // The FIRST later bar to trade beyond the level decides it once: + // the strict pivot guarantees no right-wing bar can be that touch. + std::size_t j = i + 1; + while (j < n && closedBars[j].high <= level) { + ++j; + } + if (j < n && // touched at all + closedBars[j].close <= level && // not a breakout + closedBars[j].high - level >= sweepPoints // deep enough + && j + static_cast(validBars) >= n // fresh + && displacementOk(closedBars, j, /*sweptHigh=*/true)) { + // Breakout resumed after the rejection kills the setup. + bool invalidated = false; + for (std::size_t k = j + 1; k < n; ++k) { + if (closedBars[k].close > level) { + invalidated = true; + break; + } + } + if (!invalidated && + (!bestShort.found || j > bestShort.rejection)) { + bestShort = {.rejection = j, .level = level, .found = true}; + } + } + } + // Swept swing low -> LONG candidate (exact mirror). + if (swing_pivots::isSwingLowAt(closedBars, i, pivotBars)) { + const std::int32_t level = closedBars[i].low; + std::size_t j = i + 1; + while (j < n && closedBars[j].low >= level) { + ++j; + } + if (j < n && + closedBars[j].close >= level && + level - closedBars[j].low >= sweepPoints + && j + static_cast(validBars) >= n + && displacementOk(closedBars, j, /*sweptHigh=*/false)) { + bool invalidated = false; + for (std::size_t k = j + 1; k < n; ++k) { + if (closedBars[k].close < level) { + invalidated = true; + break; + } + } + if (!invalidated && + (!bestLong.found || j > bestLong.rejection)) { + bestLong = {.rejection = j, .level = level, .found = true}; + } + } + } + } + + // --- THE VERDICT --- + // Newest rejection wins outright; the loser is discarded, not queued (an + // older setup never trades while a fresher opposing one stands). The same + // bar rejecting both ways — an outside bar sweeping a high AND a low — is + // ambiguous: refused. + if (bestShort.found && bestLong.found && + bestShort.rejection == bestLong.rejection) { + return std::nullopt; + } + const bool shortWins = + bestShort.found && + (!bestLong.found || bestShort.rejection > bestLong.rejection); + if (shortWins) { + // Entries judge the trade's own fill side: a SHORT sells the bid, + // which must be back below the swept level — the spread must not + // flatter the rejection. + if (tick.bid < bestShort.level) { + return Direction::SHORT; + } + return std::nullopt; + } + if (bestLong.found && tick.ask > bestLong.level) { + return Direction::LONG; + } + return std::nullopt; +} diff --git a/source/strategies/nyOpenRangeBreakout/nyOpenRangeBreakoutStrategy.cppm b/source/strategies/nyOpenRangeBreakout/nyOpenRangeBreakoutStrategy.cppm new file mode 100644 index 0000000..965b596 --- /dev/null +++ b/source/strategies/nyOpenRangeBreakout/nyOpenRangeBreakoutStrategy.cppm @@ -0,0 +1,267 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// NyOpenRangeBreakoutStrategy — the New York open-range breakout: the hours +// before the US equities open define a coil; a break of that range shortly +// after the open trades the session's directional expansion. The direct +// sibling of SessionRangeBreakoutStrategy (the London variant), aimed at the +// instruments where the winning runs actually cluster — US index CFDs, gold, +// oil — whose liquidity event is the 09:30 New York open, not London's. +// Like the London strategy the range is anchored to the CLOCK, not a rolling +// bar count: the edge under test is specifically the open. +// +// One OHLC timeframe is read per symbol from the loop owner's shared BarStore +// (built from the ask — see barStore): +// +// OHLC_VARIABLES[0] — the signal timeframe. The pre-open range is the +// highest high / lowest low of its CLOSED bars whose +// date (first-tick timestamp) falls in +// [nyOpen - RANGE_HOURS, nyOpen), padded by +// BUFFER_PIPS. Unlike the London strategy's fixed +// Asian window the range depth is SWEPT: 13 hours +// approximates the whole overnight session, 4 a tight +// pre-open coil — the data decides which definition +// carries the edge. Bars are first-tick anchored, so a +// bar STARTING inside the window may spill slightly +// past the open — accepted; the range is "bars that +// started before the open". +// +// Entry: only within ENTRY_WINDOW_MINUTES of the NY open (13:30 UTC under +// EDT, else 14:30 — market_hours::isNewYorkSummer, the same DST rule the +// session gate uses). The bid breaking above the padded pre-open high -> +// LONG; the ask breaking below the padded low -> SHORT. No trend filter: the +// range itself is the setup. Two structural refusals keep a partial range +// from trading: the series must reach back to the range start (else coverage +// is incomplete — first day of a run, or a window too small), and at least +// one closed bar must sit inside the range window. +// +// The entry cutoff lives HERE, not in the run loop's peakHoursOnly gate: the +// strategy must behave identically however the risk limits are configured. +// (When peakHoursOnly IS on, NewYork-mapped symbols align neatly — their +// permitted entries are exactly the 3 hours from the NY open, so windows +// beyond 180 minutes buy nothing.) +// +// Exits stay central (ATR-derived SL/TP enforced by Operations); the one +// strategy-driven exit is the optional time cap copied from +// SessionRangeBreakoutStrategy: when MAX_TRADE_DURATION_MINUTES > 0, during() +// closes the symbol's trade once open strictly longer than that, at the +// exit-side price — here it stops an open-drive entry riding into the +// afternoon drift. +// +// The same structural notes as the other strategies apply: the store updates +// BEFORE decide(), the strategy owns no bar state, and one instance sees +// every symbol interleaved. decide() deliberately scans the WHOLE series +// rather than a fixed tail — bars are selected by date, so a deeper window +// (the ATR gate may deepen a shared timeframe) is harmless. The signal +// re-fires while price holds beyond the range inside the entry window; the +// run loop's one-trade-per-symbol gate prevents stacking. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module nyOpenRangeBreakoutStrategy; + +import std; // replaces , , , , + // , +import strategy; // IStrategy base class +import barStore; // bars::BarStore — shared per-symbol bar histories +import priceData; // PriceData +import trade; // Direction +import tradeManager; // TradeManager +import timeCapExit; // strategy_exits::closeIfPastCap — the shared time cap +import ohlcObject; // OhlcObject bar record +import marketHours; // market_hours::isNewYorkSummer — the NY open rule +import symbolScale; // symbol_scale::get — points-per-pip for the buffer + +export class NyOpenRangeBreakoutStrategy : public IStrategy { +public: + // Validates the config up front and throws std::invalid_argument on a + // malformed one (missing timeframe / variables, window that cannot span + // the range start -> entry cutoff) — a misconfigured run should die + // loudly at construction, not trade silently wrong. + explicit NyOpenRangeBreakoutStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig); + + std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) override; + + // Strategy-driven exit hook: the optional time cap (see header). SL/TP + // exits remain central. + void during(const PriceData& price, const bars::BarStore& barStore, + TradeManager& tradeManager) override; + +private: + // `{}` value-initializes: OHLCVariables is an aggregate of plain ints with + // no defaults of its own, so without this the fields would hold + // indeterminate values until the constructor body assigns them (reading + // one before that is undefined behaviour). The constructor can't use a + // member-init list here because it must validate the config first. + tradingDefinitions::OHLCVariables signalCfg{}; + std::chrono::hours rangeHours{}; + std::int32_t bufferPips{}; + std::chrono::minutes entryWindow{}; + std::chrono::minutes maxTradeDuration{}; // <= 0 disables the time cap +}; + +// Config fields are assigned in the body, not a member-init list: the size +// check must run first to throw a descriptive error (an init list would have +// to use ohlcVars.at(0), dying with an unhelpful out_of_range instead). Their +// in-class {} initializers keep them defined in the meantime. +NyOpenRangeBreakoutStrategy::NyOpenRangeBreakoutStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig) { + + const auto& ohlcVars = strategyConfig.OHLC_VARIABLES; + + if (ohlcVars.empty()) { + throw std::invalid_argument( + "NyOpenRangeBreakoutStrategy: OHLC_VARIABLES needs one entry " + "(the signal timeframe)"); + } + + signalCfg = ohlcVars[0]; + + if (signalCfg.OHLC_MINUTES < 1) { + throw std::invalid_argument( + "NyOpenRangeBreakoutStrategy: OHLC_MINUTES must be >= 1"); + } + + const auto& nyVars = + strategyConfig.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES; + if (!nyVars) { + throw std::invalid_argument( + "NyOpenRangeBreakoutStrategy: " + "STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES is required " + "(RANGE_HOURS, BUFFER_PIPS, ENTRY_WINDOW_MINUTES)"); + } + rangeHours = std::chrono::hours{nyVars->RANGE_HOURS}; + bufferPips = nyVars->BUFFER_PIPS; + entryWindow = std::chrono::minutes{nyVars->ENTRY_WINDOW_MINUTES}; + maxTradeDuration = std::chrono::minutes{nyVars->MAX_TRADE_DURATION_MINUTES}; + + if (rangeHours < std::chrono::hours{1}) { + throw std::invalid_argument( + "NyOpenRangeBreakoutStrategy: RANGE_HOURS must be >= 1"); + } + // A range reaching back past the previous UTC midnight would cross the + // weekend boundary on Mondays and mix Friday's US afternoon into "the + // overnight" — cap the depth at the summer open (13:30, the earlier one) + // so the range always starts on the same UTC day as the open it precedes. + if (std::chrono::minutes{rangeHours} > + std::chrono::hours{13} + std::chrono::minutes{30}) { + throw std::invalid_argument( + "NyOpenRangeBreakoutStrategy: RANGE_HOURS must not reach past the " + "previous UTC midnight (<= 13.5h, i.e. 13)"); + } + if (bufferPips < 0) { + throw std::invalid_argument( + "NyOpenRangeBreakoutStrategy: BUFFER_PIPS must be >= 0"); + } + if (entryWindow < std::chrono::minutes{1}) { + throw std::invalid_argument( + "NyOpenRangeBreakoutStrategy: ENTRY_WINDOW_MINUTES must be >= 1"); + } + // The rolling window must reach from the range start (the coverage gate + // decide() enforces) to the end of the entry window on a winter day + // (14:30 open), with a two-bar margin so the range-start bar survives + // eviction while the last decision tick is judged. A window that cannot + // span that is a config that would NEVER trade — fail at construction + // instead. Unlike the London strategy the requirement is anchored to the + // OPEN, not midnight: range depth + entry window + margin. + const std::int64_t spanMinutes = + std::int64_t{signalCfg.OHLC_COUNT} * signalCfg.OHLC_MINUTES; + const std::int64_t requiredMinutes = + std::chrono::minutes{rangeHours}.count() + entryWindow.count() + + std::int64_t{2} * signalCfg.OHLC_MINUTES; + if (spanMinutes < requiredMinutes) { + throw std::invalid_argument( + "NyOpenRangeBreakoutStrategy: OHLC_COUNT x OHLC_MINUTES must cover " + "the range through the entry window (>= RANGE_HOURS x 60 + " + "ENTRY_WINDOW_MINUTES + 2 bars)"); + } +} + +void NyOpenRangeBreakoutStrategy::during(const PriceData& price, + const bars::BarStore& /*barStore*/, + TradeManager& tradeManager) { + // Time cap (shared closeIfPastCap mechanics): close this symbol's trade + // once open STRICTLY longer than maxTradeDuration; <= 0 disables. Only + // the current tick's symbol is checked — each symbol's trade meets its + // own next tick, which also supplies the right close price. Here it stops + // an open-drive entry riding into the afternoon drift. + strategy_exits::closeIfPastCap(price, tradeManager, maxTradeDuration); +} + +std::optional NyOpenRangeBreakoutStrategy::decide( + const PriceData& tick, const bars::BarStore& barStore) { + using namespace std::chrono; + + // --- 1. THE CLOCK GATE --- + // Entries only within the window after today's NY open; everything else + // is refused before touching a bar. sys_days -> time_point is UTC + // midnight, the anchor for both the open and the range window below. + const sys_days day = floor(tick.timestamp); + const auto sinceMidnight = tick.timestamp - day; + const minutes nyOpen = + minutes{market_hours::isNewYorkSummer(day) ? hours{13} : hours{14}} + + minutes{30}; + if (sinceMidnight < nyOpen || sinceMidnight >= nyOpen + entryWindow) { + return std::nullopt; + } + + // This symbol's bar history; nullptr means the timeframe was never + // registered on the store or the symbol has not ticked yet. Fewer than + // two bars cannot hold a closed bar (the last element is in-progress). + const std::vector* signalSeries = barStore.find( + tick.symbol, minutes{signalCfg.OHLC_MINUTES}); + if (signalSeries == nullptr || signalSeries->size() < 2) { + return std::nullopt; + } + + // --- 2. COVERAGE GATE --- + // The series must reach back to the range start, or the pre-open window + // is only partially represented (a run's first day, or eviction) and the + // "range" would be a fragment — refuse to trade a partial coil. + const system_clock::time_point openTime = sys_days{day} + nyOpen; + const system_clock::time_point rangeStart = openTime - rangeHours; + if (signalSeries->front().date > rangeStart) { + return std::nullopt; + } + + // --- 3. THE PRE-OPEN RANGE --- + // Highest high / lowest low of the CLOSED bars that STARTED inside + // [rangeStart, open) today. Selected by date rather than position, so a + // deeper-than-OHLC_COUNT store window is harmless. The in-progress last + // bar is excluded — during the entry window it is a post-open bar anyway. + std::int32_t rangeHigh = std::numeric_limits::min(); + std::int32_t rangeLow = std::numeric_limits::max(); + bool rangeSeen = false; + for (std::size_t i = 0; i + 1 < signalSeries->size(); ++i) { + const OhlcObject& bar = (*signalSeries)[i]; + if (bar.date < rangeStart || bar.date >= openTime) { + continue; + } + rangeHigh = std::max(rangeHigh, bar.high); + rangeLow = std::min(rangeLow, bar.low); + rangeSeen = true; + } + if (!rangeSeen) { + return std::nullopt; + } + + // --- 4. EXECUTION LOGIC --- + // Pips scale UP to points (pips x pointsPerPip), the OhlcBreakoutStrategy + // convention; an unknown symbol returns scale 0, which just disables the + // buffer rather than corrupting the levels. + const std::int32_t bufferPoints = bufferPips * symbol_scale::get(tick.symbol); + if (tick.bid > rangeHigh + bufferPoints) { + return Direction::LONG; + } + if (tick.ask < rangeLow - bufferPoints) { + return Direction::SHORT; + } + return std::nullopt; +} diff --git a/source/strategies/ohlcBreakout/ohlcBreakoutStrategy.cppm b/source/strategies/ohlcBreakout/ohlcBreakoutStrategy.cppm index 7175d0b..8793da2 100644 --- a/source/strategies/ohlcBreakout/ohlcBreakoutStrategy.cppm +++ b/source/strategies/ohlcBreakout/ohlcBreakoutStrategy.cppm @@ -6,8 +6,8 @@ // // OhlcBreakoutStrategy — range breakout with an EMA trend filter. // -// Two OHLC timeframes are built per symbol from the tick stream (both from the -// ask, matching the C# original): +// Two OHLC timeframes are read per symbol from the loop owner's shared +// BarStore (built from the ask — see barStore): // // OHLC_VARIABLES[0] — the breakout timeframe. The highest high / lowest low // of its CLOSED candles (the in-progress bar is excluded) @@ -16,18 +16,32 @@ // half the candle count) is the macro trend filter. // // Entry: bid breaks above the range top in a macro uptrend -> LONG; ask breaks -// below the range bottom in a macro downtrend -> SHORT. Exits stay central -// (SL/TP pip distances enforced by Operations), so during() only builds bars. +// below the range bottom in a macro downtrend -> SHORT. SL/TP exits stay +// central (ATR-derived pip distances enforced by Operations); the one +// strategy-driven exit is the optional time cap: when +// MAX_TRADE_DURATION_MINUTES > 0, during() closes the symbol's trade once it +// has been open strictly longer than that, at the exit-side price (bid for +// LONG, ask for SHORT — the exit_rules convention). In live the runner's +// close-diff turns that closeTrade into a broker CloseIntent, so the strategy +// needs no environment awareness. // // Two structural notes that differ from the C# framework: -// - Bars are built in during(), not decide(): the run loop skips decide() for -// a symbol while it has an open trade, but during() runs on every tick, so -// the bar history never gaps. decide() therefore sees bars as of the -// previous tick — immaterial, because the range comes from closed candles -// only and is compared against the current tick's bid/ask. +// - The strategy owns NO bar state. The loop owner (runLoop / a live worker) +// registers this strategy's timeframes on its BarStore and updates it once +// per tick BEFORE decide(), so bar state here already includes the tick +// being judged: the in-progress trend bar's close IS the current tick, +// which makes the trend filter read "current price vs trailing EMA" (a +// tick's own move counts as trend evidence — deliberate; the old +// in-during() building lagged this by one tick). The breakout range is +// unaffected in spirit: it still uses CLOSED candles only, though a +// bar-rolling tick promotes the previous in-progress bar into the range +// one tick sooner. The same histories feed the pre-decide ATR entry +// conditions, and the store may hold a deeper window than OHLC_COUNT when +// the gate registered one on the same timeframe — decide() reads only its +// own tail. // - One strategy instance sees every symbol's ticks interleaved by timestamp -// (SYMBOLS = "EURUSD,AUDUSD"), so all bar state is per-symbol, keyed by -// tick.symbol — the C# "persistent list per instrument" requirement. +// (SYMBOLS = "EURUSD,AUDUSD"), and the BarStore keys its histories by +// symbol — the C# "persistent list per instrument" requirement. module; @@ -35,15 +49,15 @@ module; export module ohlcBreakoutStrategy; -import std; // replaces , , , , - // , , , , - // , +import std; // replaces , , , , + // , , import strategy; // IStrategy base class +import barStore; // bars::BarStore — shared per-symbol bar histories import priceData; // PriceData import trade; // Direction import tradeManager; // TradeManager +import timeCapExit; // strategy_exits::closeIfPastCap — the shared time cap import ohlcObject; // OhlcObject bar record -import ohlcBuilder; // ohlc::calculateOHLC import ema; // ema::calculate (integer EMA) import symbolScale; // symbol_scale::get — points-per-pip for the buffer @@ -54,32 +68,26 @@ public: // should die loudly at construction, not trade silently wrong. explicit OhlcBreakoutStrategy(const tradingDefinitions::StrategyConfig& strategyConfig); - std::optional decide(const PriceData& tick) override; + std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) override; - // Builds the per-symbol bars every tick. TradeManager is unused — exits - // are driven centrally by the configured SL/TP distances. - void during(const PriceData& price, TradeManager& tradeManager) override; + // Strategy-driven exit hook. Bars are built centrally (BarStore, updated + // by the loop owner every tick), so all that remains here is the optional + // time cap: close this symbol's trade once it has outlived + // MAX_TRADE_DURATION_MINUTES. SL/TP exits remain central. + void during(const PriceData& price, const bars::BarStore& barStore, + TradeManager& tradeManager) override; private: - struct SymbolState { - std::vector breakoutBars; - std::vector trendBars; - }; - - // Transparent hasher (same pattern as TradeManager::activeTrades) so the - // per-tick find() takes a string_view and never allocates a temporary key. - struct SymbolHash { - using is_transparent = void; - std::size_t operator()(std::string_view symbol) const noexcept { - return std::hash{}(symbol); - } - }; - - tradingDefinitions::OHLCVariables breakoutCfg; - tradingDefinitions::OHLCVariables trendCfg; - std::int32_t bufferPips; - - std::unordered_map> stateBySymbol; + // `{}` value-initializes: OHLCVariables is an aggregate of plain ints with + // no defaults of its own, so without this the fields would hold + // indeterminate values until the constructor body assigns them (reading + // one before that is undefined behaviour). The constructor can't use a + // member-init list here because it must validate the config first. + tradingDefinitions::OHLCVariables breakoutCfg{}; + tradingDefinitions::OHLCVariables trendCfg{}; + std::int32_t bufferPips{}; + std::chrono::minutes maxTradeDuration{}; // <= 0 disables the time cap // Scratch buffers reused every decide() — cleared, never shrunk, so the // per-tick path stops allocating once their capacity settles. @@ -87,32 +95,24 @@ private: std::vector emaScratch; }; -namespace { - -// Feed the tick into one timeframe's bars, then trim the history from the -// front to OHLC_COUNT — a rolling window, oldest bars dropped first. The last -// element is always the in-progress bar (see ohlcBuilder). -void updateBars(const PriceData& tick, const tradingDefinitions::OHLCVariables& cfg, - std::vector& bars) { - ohlc::calculateOHLC(tick, tick.ask, std::chrono::minutes{cfg.OHLC_MINUTES}, bars); - const auto cap = static_cast(cfg.OHLC_COUNT); - if (bars.size() > cap) { - bars.erase(bars.begin(), bars.end() - static_cast(cap)); - } -} - -} // namespace - +// Config fields are assigned in the body, not a member-init list: the size +// check must run first to throw a descriptive error (an init list would have +// to use ohlcVars.at(0), dying with an unhelpful out_of_range instead). Their +// in-class {} initializers keep them defined in the meantime. OhlcBreakoutStrategy::OhlcBreakoutStrategy( const tradingDefinitions::StrategyConfig& strategyConfig) { + const auto& ohlcVars = strategyConfig.OHLC_VARIABLES; + if (ohlcVars.size() < 2) { throw std::invalid_argument( "OhlcBreakoutStrategy: OHLC_VARIABLES needs two entries " "(breakout timeframe, trend timeframe)"); } + breakoutCfg = ohlcVars[0]; trendCfg = ohlcVars[1]; + for (const auto& cfg : {breakoutCfg, trendCfg}) { // COUNT >= 2 so the breakout list always has a closed candle besides // the in-progress one, and the trend EMA period (count/2) is >= 1. @@ -128,39 +128,51 @@ OhlcBreakoutStrategy::OhlcBreakoutStrategy( "(BUFFER_PIPS)"); } bufferPips = breakoutVars->BUFFER_PIPS; + maxTradeDuration = + std::chrono::minutes{breakoutVars->MAX_TRADE_DURATION_MINUTES}; } -void OhlcBreakoutStrategy::during(const PriceData& price, TradeManager& /*tradeManager*/) { - // This symbol's entry in stateBySymbol (its two bar histories), inserted - // empty on the symbol's first tick. Heterogeneous find first: only that - // first tick pays for the std::string key construction. - auto stateIt = stateBySymbol.find(std::string_view{price.symbol}); - if (stateIt == stateBySymbol.end()) { - stateIt = stateBySymbol.emplace(price.symbol, SymbolState{}).first; - } - updateBars(price, breakoutCfg, stateIt->second.breakoutBars); - updateBars(price, trendCfg, stateIt->second.trendBars); +void OhlcBreakoutStrategy::during(const PriceData& price, + const bars::BarStore& /*barStore*/, + TradeManager& tradeManager) { + // Time cap (shared closeIfPastCap mechanics): close this symbol's trade + // once open STRICTLY longer than maxTradeDuration; <= 0 disables. Only + // the current tick's symbol is checked — each symbol's trade meets its + // own next tick, which also supplies the right close price. decide() may + // re-enter on a later tick while the breakout condition still holds (the + // documented re-fire behaviour). + strategy_exits::closeIfPastCap(price, tradeManager, maxTradeDuration); } -std::optional OhlcBreakoutStrategy::decide(const PriceData& tick) { - // This symbol's bar histories; absent means during() has not seen a tick - // for the symbol yet, so there is nothing to decide on. - const auto stateIt = stateBySymbol.find(std::string_view{tick.symbol}); - if (stateIt == stateBySymbol.end()) { +std::optional OhlcBreakoutStrategy::decide(const PriceData& tick, + const bars::BarStore& barStore) { + // This symbol's bar histories; nullptr means the timeframe was never + // registered on the store or the symbol has not ticked yet — either way + // there is nothing to decide on. + const std::vector* breakoutSeries = barStore.find( + tick.symbol, std::chrono::minutes{breakoutCfg.OHLC_MINUTES}); + const std::vector* trendSeries = + barStore.find(tick.symbol, std::chrono::minutes{trendCfg.OHLC_MINUTES}); + if (breakoutSeries == nullptr || trendSeries == nullptr) { return std::nullopt; } - const SymbolState& state = stateIt->second; // Warm-up gate: no signals until both timeframes have a full window // (C#: `if (ohlcList.Count < totalOHLCCount) return`). - if (state.breakoutBars.size() < static_cast(breakoutCfg.OHLC_COUNT) || - state.trendBars.size() < static_cast(trendCfg.OHLC_COUNT)) { + const auto breakoutCount = static_cast(breakoutCfg.OHLC_COUNT); + const auto trendCount = static_cast(trendCfg.OHLC_COUNT); + if (breakoutSeries->size() < breakoutCount || + trendSeries->size() < trendCount) { return std::nullopt; } - // No upper-bound check to pair with the gate above: updateBars trims each - // history to OHLC_COUNT (oldest bars dropped from the front) on every - // tick, so past the gate both windows hold exactly OHLC_COUNT bars. + // Read only this strategy's tail of each history: the store keeps the + // LARGEST window registered per timeframe, so another consumer (the ATR + // entry gate) may have deepened a series beyond OHLC_COUNT. + const std::span breakoutBars = + std::span(*breakoutSeries).last(breakoutCount); + const std::span trendBars = + std::span(*trendSeries).last(trendCount); // --- 1. THE BREAKOUT LOGIC --- // Range from the CLOSED breakout candles (drop the in-progress last bar — @@ -170,19 +182,35 @@ std::optional OhlcBreakoutStrategy::decide(const PriceData& tick) { // just disables the buffer rather than corrupting the levels. std::int32_t highestHigh = std::numeric_limits::min(); std::int32_t lowestLow = std::numeric_limits::max(); - for (auto bar = state.breakoutBars.begin(); bar != state.breakoutBars.end() - 1; ++bar) { - highestHigh = std::max(highestHigh, bar->high); - lowestLow = std::min(lowestLow, bar->low); + for (const OhlcObject& bar : breakoutBars.first(breakoutBars.size() - 1)) { + highestHigh = std::max(highestHigh, bar.high); + lowestLow = std::min(lowestLow, bar.low); } const std::int32_t bufferPoints = bufferPips * symbol_scale::get(tick.symbol); // --- 2. THE TREND FILTER LOGIC --- // EMA over the trend timeframe's closes (chronological, in-progress bar // included, like the C#). Period = half the window (C#: count * 0.5m, - // truncated). Both the close and the EMA are read at the last index so - // today's price is compared against today's moving average. + // truncated). Both the close and the EMA are read at the last index — + // and since the store updated before decide(), that last close is the + // current tick itself: the comparison is "current price vs trailing + // EMA" (algebraically, close > EMA-including-it iff close > + // EMA-excluding-it). + // + // KNOWN RISK (accepted): the tick being judged supplies its own trend + // evidence. An EMA step can never drag the average past the new point, + // so a single-tick spike above the PRIOR EMA always reads as "uptrend" + // — there is no way for the spike itself to be on the wrong side of an + // EMA that includes it. With dense ticking this is indistinguishable + // from the old one-tick-lagged filter; it diverges exactly at price + // discontinuities (news, thin liquidity, session opens), where the + // strategy may buy the very tick of a spike out of a falling market. + // Two live mitigations: a steep prior fall keeps the trailing EMA far + // overhead (the spike must clear it, not just the local range), and the + // pre-decide spread-vs-ATR gate (entryConditions) rejects most news + // ticks because their spreads blow out before their prices do. closesScratch.clear(); - for (const auto& bar : state.trendBars) { + for (const OhlcObject& bar : trendBars) { closesScratch.push_back(bar.close); } const int emaPeriod = static_cast(closesScratch.size() / 2); diff --git a/source/strategies/randomStrategy/randomStrategy.cppm b/source/strategies/randomStrategy/randomStrategy.cppm index 2c90b90..3cefce6 100644 --- a/source/strategies/randomStrategy/randomStrategy.cppm +++ b/source/strategies/randomStrategy/randomStrategy.cppm @@ -12,6 +12,7 @@ export module randomStrategy; import std; // replaces , , import strategy; // IStrategy base class +import barStore; // bars::BarStore (unused here; interface contract) import priceData; // PriceData import trade; // Direction import tradeManager; // TradeManager @@ -48,9 +49,11 @@ public: // value type that may or may not hold a T, with no heap allocation. // // Not const because the RNG engine mutates its internal state on - // each call. The `tick` parameter is unused today but keeps the - // interface stable for strategies that will look at price. - std::optional decide(const PriceData& tick) override; + // each call. The `tick` and `barStore` parameters are unused today + // but keep the interface stable for strategies that look at price + // or bar history. + std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) override; // Per-tick management hook. The default RandomStrategy // implementation is a no-op — Operations closes trades when @@ -58,7 +61,7 @@ public: // is passed by mutable reference so future strategies (trailing // stops, partial closes, scale-ins) can act on open positions // here without changing the interface. - void during(const PriceData& price, + void during(const PriceData& price, const bars::BarStore& barStore, TradeManager& tradeManager) override; private: @@ -79,11 +82,13 @@ RandomStrategy::RandomStrategy(const tradingDefinitions::StrategyConfig& strateg coin(0.5), closeProb(0.0) {} // unused — exits are driven by SL/TP in Operations -std::optional RandomStrategy::decide(const PriceData& /*tick*/) { +std::optional RandomStrategy::decide(const PriceData& /*tick*/, + const bars::BarStore& /*barStore*/) { return coin(rng) ? Direction::LONG : Direction::SHORT; } void RandomStrategy::during(const PriceData& /*price*/, + const bars::BarStore& /*barStore*/, TradeManager& /*tradeManager*/) { // Exits are handled centrally by Operations using each trade's // stop-loss / take-profit pip distances. Strategies that want diff --git a/source/strategies/rangeVelocity/rangeVelocityStrategy.cppm b/source/strategies/rangeVelocity/rangeVelocityStrategy.cppm new file mode 100644 index 0000000..710fa5c --- /dev/null +++ b/source/strategies/rangeVelocity/rangeVelocityStrategy.cppm @@ -0,0 +1,283 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// RangeVelocityStrategy — velocity momentum on range bars. +// +// Range bars complete when price travels a volatility-scaled threshold (see +// rangeBarBuilder), so every bar covers ~equal distance and the time a bar +// took to form IS a momentum reading: fast bars mean urgent, one-sided flow. +// This is the clock as a SIGNAL — the bars themselves stay purely +// event-driven; formation time is measured over them, never used to build +// them. +// +// Entry: the last RUN_BARS closed range bars all point one way AND each +// formed faster than the recent norm — duration x 100 <= median x +// SPEED_RATIO_PERCENT, where the median is taken over the +// SPEED_LOOKBACK_BARS closed bars immediately preceding the run. Median, not +// mean: a weekend/session-gap bar inflates one baseline element and the +// median shrugs it off, while a gap bar inside the run itself fails the +// speed test — so the strategy naturally refuses to chase the first bars +// after a gap. +// +// Formation durations are open-to-open (OhlcObject carries only the bar's +// first tick time): bar i's duration = bars[i+1].date - bars[i].date, +// overstated by one inter-tick gap, uniformly. The NEWEST closed bar has no +// successor on the tick that closes it, and needs none — the breach is +// happening now, so tick.timestamp - back().date is its EXACT duration. +// +// Fire-once semantics: the whole signal is gated on back().complete, the +// just-closed state that exists exactly on the breach tick (the next tick +// pushes the successor, and a fresh bar can never be born complete — +// threshold >= 1 point). One evaluation per bar close, never on stale +// state; a stop-out mid-run may re-enter on the NEXT fast same-direction +// close — deliberate, and one-trade-per-symbol prevents stacking. In that +// gated state the series holds ONLY closed bars (the in-progress slot IS the +// just-closed back()), which is what makes the ctor's RANGE_COUNT bound +// provable. +// +// Exits: the central ATR stop/limit from the entry gate, plus two +// during() exits — an opposite-direction run of EXIT_RUN_BARS closed bars +// (no speed filter: momentum dying is reason enough to leave) and the +// OhlcBreakout-style max-duration time cap. during() receives the shared +// BarStore precisely for this: decide() is entry-gated and cannot watch an +// open position's bars. +// +// OHLC_VARIABLES is deliberately empty: the ATR entry gate falls back to its +// default 15m series (entryConditions::gateSeriesFor) for stop/limit sizing, +// the same path RandomStrategy takes. A gate-rejected breach tick loses the +// entry (skipped, never deferred) — a wide-spread breach shouldn't enter +// anyway. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module rangeVelocityStrategy; + +import std; // replaces , , , + // , , +import barStore; // bars::BarStore — findRange +import ohlcObject; // OhlcObject bar record (shared with OHLC bars) +import priceData; // PriceData +import rangeBarBuilder; // rangebar::RangeBarSpec — series identity +import strategy; // IStrategy +import timeCapExit; // strategy_exits::closeIfPastCap — the shared time cap +import trade; // Direction, Trade +import tradeManager; // TradeManager — during() exits + +namespace { + +// +1 up-close, -1 down-close, 0 doji. A CLOSED range bar can never be a doji +// (the breaching tick is always a strict new extreme of the bar, so close == +// high or close == low != open), but the guard is kept defensively — it +// costs nothing and protects against future builder changes. +int barDirection(const OhlcObject& bar) { + return (bar.close > bar.open) - (bar.close < bar.open); +} + +std::int64_t microsOf(const std::chrono::system_clock::time_point tp) { + return std::chrono::duration_cast( + tp.time_since_epoch()) + .count(); +} + +} // namespace + +export class RangeVelocityStrategy : public IStrategy { +public: + explicit RangeVelocityStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig) { + // Positional contract, like OHLC_VARIABLES: [0] is THE series this + // strategy trades. operations/liveStrategyCache silently skip + // zero-sentinel entries when registering, so a malformed entry here + // would mean a strategy that looks configured but can never see a + // bar — die loudly at construction instead. + if (strategyConfig.RANGE_VARIABLES.empty()) { + throw std::invalid_argument( + "RangeVelocityStrategy requires RANGE_VARIABLES[0] (the " + "range-bar series it trades)"); + } + const auto& rangeVars = strategyConfig.RANGE_VARIABLES[0]; + if (rangeVars.RANGE_ATR_TICK_WINDOW < 1 || + rangeVars.RANGE_ATR_PERCENT < 1 || rangeVars.RANGE_COUNT < 1) { + throw std::invalid_argument( + "RangeVelocityStrategy: RANGE_VARIABLES[0] fields must all " + "be >= 1 (a zero-sentinel entry would register no series)"); + } + spec_ = {.atrTickWindow = rangeVars.RANGE_ATR_TICK_WINDOW, + .atrPercent = rangeVars.RANGE_ATR_PERCENT, + .count = rangeVars.RANGE_COUNT}; + + const auto& vars = strategyConfig.STRATEGY_VARIABLES; + if (!vars.RANGE_VELOCITY_VARIABLES.has_value()) { + throw std::invalid_argument( + "RangeVelocityStrategy requires " + "STRATEGY_VARIABLES.RANGE_VELOCITY_VARIABLES"); + } + const auto& rv = *vars.RANGE_VELOCITY_VARIABLES; + if (rv.RUN_BARS < 1) { + throw std::invalid_argument( + "RangeVelocityStrategy: RUN_BARS must be >= 1"); + } + if (rv.SPEED_LOOKBACK_BARS < 1) { + throw std::invalid_argument( + "RangeVelocityStrategy: SPEED_LOOKBACK_BARS must be >= 1"); + } + if (rv.SPEED_RATIO_PERCENT < 1) { + throw std::invalid_argument( + "RangeVelocityStrategy: SPEED_RATIO_PERCENT must be >= 1 " + "(0 could never admit a bar)"); + } + if (rv.EXIT_RUN_BARS < 1) { + throw std::invalid_argument( + "RangeVelocityStrategy: EXIT_RUN_BARS must be >= 1"); + } + if (rv.MAX_TRADE_DURATION_MINUTES < 0) { + throw std::invalid_argument( + "RangeVelocityStrategy: MAX_TRADE_DURATION_MINUTES must be " + ">= 0 (0 disables the time cap)"); + } + runBars = rv.RUN_BARS; + speedLookbackBars = rv.SPEED_LOOKBACK_BARS; + speedRatioPercent = rv.SPEED_RATIO_PERCENT; + exitRunBars = rv.EXIT_RUN_BARS; + maxTradeDuration = std::chrono::minutes{rv.MAX_TRADE_DURATION_MINUTES}; + + // The provable window bound: at breach-tick evaluation every element + // is closed, so decide() touches indices down to n - K - M (baseline + // start) with successor lookups capped at n - 1 (the newest bar's + // duration comes from the tick itself), and during() scans the last + // E bars. Anything deeper than max(K + M, E) is unused margin. + const int required = std::max(rv.RUN_BARS + rv.SPEED_LOOKBACK_BARS, + rv.EXIT_RUN_BARS); + if (rangeVars.RANGE_COUNT < required) { + throw std::invalid_argument(std::format( + "RangeVelocityStrategy: RANGE_COUNT ({}) must be >= " + "max(RUN_BARS + SPEED_LOOKBACK_BARS, EXIT_RUN_BARS) ({})", + rangeVars.RANGE_COUNT, required)); + } + } + + std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) override { + const std::vector* series = + barStore.findRange(tick.symbol, spec_); + // Fire-once gate: only the breach tick sees back().complete — the + // successor's first tick resets it. Everything below may assume the + // series holds closed bars only. + if (series == nullptr || series->empty() || !series->back().complete) { + return std::nullopt; + } + const std::size_t n = series->size(); + const auto k = static_cast(runBars); + const auto m = static_cast(speedLookbackBars); + if (n < k + m) { + return std::nullopt; // still warming the run + baseline window + } + const std::span bars(*series); + + // 1. The run: last K closed bars, one strict direction throughout. + const int dir = barDirection(bars[n - 1]); + if (dir == 0) { + return std::nullopt; + } + for (std::size_t i = n - k; i < n - 1; ++i) { + if (barDirection(bars[i]) != dir) { + return std::nullopt; + } + } + + // 2. Baseline: median open-to-open duration of the M bars preceding + // the run. nth_element (upper median, deterministic) beats a full + // sort and only runs on breach ticks. + durationsScratch.clear(); + for (std::size_t i = n - k - m; i < n - k; ++i) { + durationsScratch.push_back(microsOf(bars[i + 1].date) - + microsOf(bars[i].date)); + } + const auto medianIt = durationsScratch.begin() + + static_cast(m / 2); + std::ranges::nth_element(durationsScratch, medianIt); + const std::int64_t median = *medianIt; + if (median <= 0) { + return std::nullopt; // degenerate stream — same "not warm" + // convention as ATR == 0 + } + + // 3. Speed test on every run bar: duration x 100 <= median x ratio, + // all int64 (a weekend gap is ~2.6e11 us; x100 is far from + // overflow). Earlier run bars measure open-to-open like the + // baseline (apples to apples); the newest bar's breach is THIS + // tick, so its duration is exact. + const std::int64_t allowance = median * speedRatioPercent; + for (std::size_t i = n - k; i + 1 < n; ++i) { + const std::int64_t duration = + microsOf(bars[i + 1].date) - microsOf(bars[i].date); + if (duration * 100 > allowance) { + return std::nullopt; + } + } + const std::int64_t newestDuration = + microsOf(tick.timestamp) - microsOf(bars[n - 1].date); + if (newestDuration * 100 > allowance) { + return std::nullopt; + } + + return dir > 0 ? Direction::LONG : Direction::SHORT; + } + + void during(const PriceData& price, const bars::BarStore& barStore, + TradeManager& tradeManager) override { + // (a) Time cap first (cheap, bar-independent): the shared + // closeIfPastCap — strictly greater, <= 0 disables. A true return + // means the position is gone; nothing left to manage. + if (strategy_exits::closeIfPastCap(price, tradeManager, + maxTradeDuration)) { + return; + } + const Trade* trade = tradeManager.findActiveTrade(price.symbol); + if (trade == nullptr) { + return; + } + // Direction read BEFORE closeTrade — the close erases the map node. + const std::int32_t exitPrice = + trade->direction == Direction::LONG ? price.bid : price.ask; + + // (b) Opposite run: EXIT_RUN_BARS closed bars all against the + // position, judged only on breach ticks (same fire-once gate as + // decide). No speed filter — momentum dying is reason enough. On the + // entry's own breach tick this can never trigger: the newest closed + // bar points WITH the entry, so an all-against run is impossible. + const std::vector* series = + barStore.findRange(price.symbol, spec_); + if (series == nullptr || series->empty() || !series->back().complete) { + return; + } + const std::size_t n = series->size(); + const auto e = static_cast(exitRunBars); + if (n < e) { + return; + } + const int against = trade->direction == Direction::LONG ? -1 : 1; + for (std::size_t i = n - e; i < n; ++i) { + if (barDirection((*series)[i]) != against) { + return; + } + } + tradeManager.closeTrade(price.symbol, exitPrice, price); + } + +private: + rangebar::RangeBarSpec spec_{}; // RANGE_VARIABLES[0]: identity + depth + int runBars = 0; // K + int speedLookbackBars = 0; // M + int speedRatioPercent = 0; + int exitRunBars = 0; // E + std::chrono::minutes maxTradeDuration{0}; // <= 0 disables + // Reused per breach tick so the median never allocates in the hot loop + // (the OhlcBreakout closesScratch idiom). + std::vector durationsScratch; +}; diff --git a/source/strategies/sessionRangeBreakout/sessionRangeBreakoutStrategy.cppm b/source/strategies/sessionRangeBreakout/sessionRangeBreakoutStrategy.cppm new file mode 100644 index 0000000..b4f21f4 --- /dev/null +++ b/source/strategies/sessionRangeBreakout/sessionRangeBreakoutStrategy.cppm @@ -0,0 +1,249 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// SessionRangeBreakoutStrategy — the classic London open-range breakout: the +// Asian session (00:00–06:00 UTC, matching market_hours' Asia window) defines +// a coil; a break of that range shortly after the London open trades the +// session's directional expansion. Unlike OhlcBreakoutStrategy the range is +// anchored to the CLOCK, not to a rolling bar count — the edge under test is +// specifically the open, so time is a first-class input here. +// +// One OHLC timeframe is read per symbol from the loop owner's shared BarStore +// (built from the ask — see barStore): +// +// OHLC_VARIABLES[0] — the signal timeframe. Today's Asian range is the +// highest high / lowest low of its CLOSED bars whose +// date (first-tick timestamp) falls in [00:00, 06:00) +// UTC, padded by BUFFER_PIPS. Bars are first-tick +// anchored, so a bar STARTING inside the window may +// spill slightly past 06:00 — accepted; the range is +// "bars that started in the session". +// +// Entry: only within ENTRY_WINDOW_MINUTES of the London open (07:00 UTC under +// BST, else 08:00 — market_hours::isLondonSummer, the same DST rule the +// session gate uses). The bid breaking above the padded Asian high -> LONG; +// the ask breaking below the padded low -> SHORT. No trend filter: the range +// itself is the setup. Two structural refusals keep a partial range from +// trading: the series must reach back to before midnight (else today's Asian +// coverage is incomplete — first day of a run, or a window too small), and at +// least one closed bar must sit inside the Asian window. +// +// The entry cutoff lives HERE, not in the run loop's peakHoursOnly gate: the +// strategy must behave identically however the risk limits are configured. +// (When peakHoursOnly IS on, Europe-mapped symbols align neatly — their +// permitted entries are exactly the 3 hours from the London open.) +// +// Exits stay central (ATR-derived SL/TP enforced by Operations); the one +// strategy-driven exit is the optional time cap copied from +// OhlcBreakoutStrategy: when MAX_TRADE_DURATION_MINUTES > 0, during() closes +// the symbol's trade once open strictly longer than that, at the exit-side +// price — here it stops a London entry riding into New York chop. +// +// The same structural notes as the other strategies apply: the store updates +// BEFORE decide(), the strategy owns no bar state, and one instance sees +// every symbol interleaved. decide() deliberately scans the WHOLE series +// rather than a fixed tail — bars are selected by date, so a deeper window +// (the ATR gate may deepen a shared timeframe) is harmless. The signal +// re-fires while price holds beyond the range inside the entry window; the +// run loop's one-trade-per-symbol gate prevents stacking. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module sessionRangeBreakoutStrategy; + +import std; // replaces , , , , + // , +import strategy; // IStrategy base class +import barStore; // bars::BarStore — shared per-symbol bar histories +import priceData; // PriceData +import trade; // Direction +import tradeManager; // TradeManager +import timeCapExit; // strategy_exits::closeIfPastCap — the shared time cap +import ohlcObject; // OhlcObject bar record +import marketHours; // market_hours::isLondonSummer — the London open rule +import symbolScale; // symbol_scale::get — points-per-pip for the buffer + +export class SessionRangeBreakoutStrategy : public IStrategy { +public: + // Validates the config up front and throws std::invalid_argument on a + // malformed one (missing timeframe / variables, window that cannot span + // midnight -> entry cutoff) — a misconfigured run should die loudly at + // construction, not trade silently wrong. + explicit SessionRangeBreakoutStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig); + + std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) override; + + // Strategy-driven exit hook: the optional time cap (see header). SL/TP + // exits remain central. + void during(const PriceData& price, const bars::BarStore& barStore, + TradeManager& tradeManager) override; + +private: + // `{}` value-initializes: OHLCVariables is an aggregate of plain ints with + // no defaults of its own, so without this the fields would hold + // indeterminate values until the constructor body assigns them (reading + // one before that is undefined behaviour). The constructor can't use a + // member-init list here because it must validate the config first. + tradingDefinitions::OHLCVariables signalCfg{}; + std::int32_t bufferPips{}; + std::chrono::minutes entryWindow{}; + std::chrono::minutes maxTradeDuration{}; // <= 0 disables the time cap +}; + +namespace { +// The Asian session in UTC — MUST match market_hours' Asia window (00:00 to +// 06:00), which is the definition being traded against. +inline constexpr std::chrono::hours kAsiaSessionEnd{6}; +// The latest possible London open (08:00 GMT); the ctor's window check uses +// the worst case so a winter run is as covered as a summer one. +inline constexpr std::chrono::minutes kLatestLondonOpen{std::chrono::hours{8}}; +} // namespace + +// Config fields are assigned in the body, not a member-init list: the size +// check must run first to throw a descriptive error (an init list would have +// to use ohlcVars.at(0), dying with an unhelpful out_of_range instead). Their +// in-class {} initializers keep them defined in the meantime. +SessionRangeBreakoutStrategy::SessionRangeBreakoutStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig) { + + const auto& ohlcVars = strategyConfig.OHLC_VARIABLES; + + if (ohlcVars.empty()) { + throw std::invalid_argument( + "SessionRangeBreakoutStrategy: OHLC_VARIABLES needs one entry " + "(the signal timeframe)"); + } + + signalCfg = ohlcVars[0]; + + if (signalCfg.OHLC_MINUTES < 1) { + throw std::invalid_argument( + "SessionRangeBreakoutStrategy: OHLC_MINUTES must be >= 1"); + } + + const auto& sessionVars = + strategyConfig.STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES; + if (!sessionVars) { + throw std::invalid_argument( + "SessionRangeBreakoutStrategy: " + "STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES is required " + "(BUFFER_PIPS, ENTRY_WINDOW_MINUTES)"); + } + bufferPips = sessionVars->BUFFER_PIPS; + entryWindow = std::chrono::minutes{sessionVars->ENTRY_WINDOW_MINUTES}; + maxTradeDuration = + std::chrono::minutes{sessionVars->MAX_TRADE_DURATION_MINUTES}; + + if (bufferPips < 0) { + throw std::invalid_argument( + "SessionRangeBreakoutStrategy: BUFFER_PIPS must be >= 0"); + } + if (entryWindow < std::chrono::minutes{1}) { + throw std::invalid_argument( + "SessionRangeBreakoutStrategy: ENTRY_WINDOW_MINUTES must be >= 1"); + } + // The rolling window must reach from before midnight (the coverage gate + // decide() enforces) to the end of the entry window on a winter day + // (08:00 open), with a two-bar margin so the pre-midnight bar survives + // eviction while the last decision tick is judged. A window that cannot + // span that is a config that would NEVER trade — fail at construction + // instead. + const std::int64_t spanMinutes = + std::int64_t{signalCfg.OHLC_COUNT} * signalCfg.OHLC_MINUTES; + const std::int64_t requiredMinutes = + kLatestLondonOpen.count() + entryWindow.count() + + std::int64_t{2} * signalCfg.OHLC_MINUTES; + if (spanMinutes < requiredMinutes) { + throw std::invalid_argument( + "SessionRangeBreakoutStrategy: OHLC_COUNT x OHLC_MINUTES must cover " + "midnight through the entry window (>= 480 + ENTRY_WINDOW_MINUTES " + "+ 2 bars)"); + } +} + +void SessionRangeBreakoutStrategy::during(const PriceData& price, + const bars::BarStore& /*barStore*/, + TradeManager& tradeManager) { + // Time cap (shared closeIfPastCap mechanics): close this symbol's trade + // once open STRICTLY longer than maxTradeDuration; <= 0 disables. Only + // the current tick's symbol is checked — each symbol's trade meets its + // own next tick, which also supplies the right close price. + strategy_exits::closeIfPastCap(price, tradeManager, maxTradeDuration); +} + +std::optional SessionRangeBreakoutStrategy::decide( + const PriceData& tick, const bars::BarStore& barStore) { + using namespace std::chrono; + + // --- 1. THE CLOCK GATE --- + // Entries only within the window after today's London open; everything + // else is refused before touching a bar. sys_days -> time_point is UTC + // midnight, the anchor for both the open and the Asian window below. + const sys_days day = floor(tick.timestamp); + const auto sinceMidnight = tick.timestamp - day; + const minutes londonOpen{market_hours::isLondonSummer(day) ? hours{7} + : hours{8}}; + if (sinceMidnight < londonOpen || sinceMidnight >= londonOpen + entryWindow) { + return std::nullopt; + } + + // This symbol's bar history; nullptr means the timeframe was never + // registered on the store or the symbol has not ticked yet. Fewer than + // two bars cannot hold a closed bar (the last element is in-progress). + const std::vector* signalSeries = barStore.find( + tick.symbol, minutes{signalCfg.OHLC_MINUTES}); + if (signalSeries == nullptr || signalSeries->size() < 2) { + return std::nullopt; + } + + // --- 2. COVERAGE GATE --- + // The series must reach back past midnight, or today's Asian session is + // only partially represented (a run's first day, or eviction) and the + // "range" would be a fragment — refuse to trade a partial coil. + const system_clock::time_point midnight = day; + if (signalSeries->front().date > midnight) { + return std::nullopt; + } + + // --- 3. THE ASIAN RANGE --- + // Highest high / lowest low of the CLOSED bars that STARTED inside + // [00:00, 06:00) UTC today. Selected by date rather than position, so a + // deeper-than-OHLC_COUNT store window is harmless. The in-progress last + // bar is excluded — during the entry window it is a London bar anyway. + const system_clock::time_point asiaEnd = midnight + kAsiaSessionEnd; + std::int32_t asianHigh = std::numeric_limits::min(); + std::int32_t asianLow = std::numeric_limits::max(); + bool sessionSeen = false; + for (std::size_t i = 0; i + 1 < signalSeries->size(); ++i) { + const OhlcObject& bar = (*signalSeries)[i]; + if (bar.date < midnight || bar.date >= asiaEnd) { + continue; + } + asianHigh = std::max(asianHigh, bar.high); + asianLow = std::min(asianLow, bar.low); + sessionSeen = true; + } + if (!sessionSeen) { + return std::nullopt; + } + + // --- 4. EXECUTION LOGIC --- + // Pips scale UP to points (pips x pointsPerPip), the OhlcBreakoutStrategy + // convention; an unknown symbol returns scale 0, which just disables the + // buffer rather than corrupting the levels. + const std::int32_t bufferPoints = bufferPips * symbol_scale::get(tick.symbol); + if (tick.bid > asianHigh + bufferPoints) { + return Direction::LONG; + } + if (tick.ask < asianLow - bufferPoints) { + return Direction::SHORT; + } + return std::nullopt; +} diff --git a/source/strategies/squeezeBreakout/squeezeBreakoutStrategy.cppm b/source/strategies/squeezeBreakout/squeezeBreakoutStrategy.cppm new file mode 100644 index 0000000..ba943b4 --- /dev/null +++ b/source/strategies/squeezeBreakout/squeezeBreakoutStrategy.cppm @@ -0,0 +1,290 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// SqueezeBreakoutStrategy — volatility contraction then expansion: a bar +// whose range has contracted (an inside bar, or the narrowest range of the +// last N) marks a coil, and a break of THAT BAR's high/low trades the +// expansion. Where OhlcBreakoutStrategy needs price to clear a whole +// multi-bar range, this fires off a single compressed bar — the tightest +// setup in the book, and the cheapest: pure OHLC patterns, no indicator on +// the signal timeframe. +// +// Two OHLC timeframes are read per symbol from the loop owner's shared +// BarStore (built from the ask — see barStore): +// +// OHLC_VARIABLES[0] — the signal timeframe. The last VALID_BARS closed +// bars are scanned newest-first for the pattern +// NR_LOOKBACK selects: 0 = inside bar (range within +// its predecessor's), N >= 2 = NR-N (high-low range +// STRICTLY the narrowest of the last N closed bars). +// A matched bar's own high/low, padded by BUFFER_PIPS, +// are the breakout levels. +// OHLC_VARIABLES[1] — the trend timeframe. An EMA over its closes (period +// = half the candle count) is the macro trend filter, +// the exact OhlcBreakoutStrategy idiom — so squeeze +// results read as a direct A/B against the range +// breakout. The same KNOWN RISK documented there +// applies: the judged tick supplies its own trend +// evidence. +// +// Entry: bid above a matched pattern bar's padded high in a macro uptrend -> +// LONG; ask below its padded low in a downtrend -> SHORT. Candidates are +// tried newest-first and the first breakout wins; a pattern that matched but +// was not broken does not stop older candidates inside VALID_BARS from +// firing. There is no mitigation concept — once a pattern ages past +// VALID_BARS it simply leaves the scan. +// +// Exits stay fully central (ATR-derived SL/TP enforced by Operations) — +// during() is a no-op, same doctrine as FvgStrategy / KeltnerFadeStrategy. +// +// The same structural notes as the other strategies apply: the store updates +// BEFORE decide() (the in-progress last bar's close IS the current tick), the +// strategy owns no bar state, one instance sees every symbol interleaved, and +// the store may hold a deeper window than OHLC_COUNT when the ATR gate +// registered one on the same timeframe — decide() reads only its own tail. +// The signal re-fires while price holds beyond a valid pattern's level; the +// run loop's one-trade-per-symbol gate prevents stacking. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module squeezeBreakoutStrategy; + +import std; // replaces , , , , + // , , , +import strategy; // IStrategy base class +import barStore; // bars::BarStore — shared per-symbol bar histories +import priceData; // PriceData +import trade; // Direction +import tradeManager; // TradeManager +import timeCapExit; // strategy_exits::closeIfPastCap — the shared time cap +import ohlcObject; // OhlcObject bar record +import ema; // ema::calculate (integer EMA) — the trend filter +import symbolScale; // symbol_scale::get — points-per-pip for the buffer + +export class SqueezeBreakoutStrategy : public IStrategy { +public: + // Validates the config up front and throws std::invalid_argument on a + // malformed one (missing timeframes / variables, windows too small for + // the scan, the degenerate NR-1) — a misconfigured run should die loudly + // at construction, not trade silently wrong. + explicit SqueezeBreakoutStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig); + + std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) override; + + // Strategy-driven exit hook: the optional time cap (strategy_exits:: + // closeIfPastCap). SL/TP exits remain central (ATR-derived, enforced by + // Operations / exit_rules). + void during(const PriceData& price, const bars::BarStore& barStore, + TradeManager& tradeManager) override; + +private: + // `{}` value-initializes: OHLCVariables is an aggregate of plain ints with + // no defaults of its own, so without this the fields would hold + // indeterminate values until the constructor body assigns them (reading + // one before that is undefined behaviour). The constructor can't use a + // member-init list here because it must validate the config first. + tradingDefinitions::OHLCVariables signalCfg{}; + tradingDefinitions::OHLCVariables trendCfg{}; + int nrLookback{}; // 0 = inside-bar mode, >= 2 = NR-N + int validBars{}; + std::int32_t bufferPips{}; + std::chrono::minutes maxTradeDuration{}; // <= 0 disables the time cap + + // Scratch buffers reused every decide() — cleared, never shrunk, so the + // per-tick path stops allocating once their capacity settles (the + // OhlcBreakoutStrategy pattern). + std::vector closesScratch; + std::vector emaScratch; + + // True when the closed bar at `index` (within `closedBars`) is the + // contraction pattern NR_LOOKBACK selects. The ctor's window minimum + // guarantees every predecessor the check reads is in range. + [[nodiscard]] bool isPatternAt(std::span closedBars, + std::size_t index) const; +}; + +// Config fields are assigned in the body, not a member-init list: the size +// check must run first to throw a descriptive error (an init list would have +// to use ohlcVars.at(0), dying with an unhelpful out_of_range instead). Their +// in-class {} initializers keep them defined in the meantime. +SqueezeBreakoutStrategy::SqueezeBreakoutStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig) { + + const auto& ohlcVars = strategyConfig.OHLC_VARIABLES; + + if (ohlcVars.size() < 2) { + throw std::invalid_argument( + "SqueezeBreakoutStrategy: OHLC_VARIABLES needs two entries " + "(signal timeframe, trend timeframe)"); + } + + signalCfg = ohlcVars[0]; + trendCfg = ohlcVars[1]; + + for (const auto& cfg : {signalCfg, trendCfg}) { + if (cfg.OHLC_MINUTES < 1) { + throw std::invalid_argument( + "SqueezeBreakoutStrategy: OHLC_MINUTES must be >= 1"); + } + } + // COUNT >= 2 keeps the trend EMA period (count / 2) at >= 1. + if (trendCfg.OHLC_COUNT < 2) { + throw std::invalid_argument( + "SqueezeBreakoutStrategy: OHLC_VARIABLES[1].OHLC_COUNT must be >= 2"); + } + + const auto& squeezeVars = + strategyConfig.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES; + if (!squeezeVars) { + throw std::invalid_argument( + "SqueezeBreakoutStrategy: STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES " + "is required (NR_LOOKBACK, VALID_BARS)"); + } + nrLookback = squeezeVars->NR_LOOKBACK; + validBars = squeezeVars->VALID_BARS; + bufferPips = squeezeVars->BUFFER_PIPS; + + // NR-1 is degenerate — "strictly the narrowest of the last one" matches + // every bar, turning the strategy into a permanent breakout scanner. + if (nrLookback < 0 || nrLookback == 1) { + throw std::invalid_argument( + "SqueezeBreakoutStrategy: NR_LOOKBACK must be 0 (inside bar) or >= 2"); + } + if (validBars < 1) { + throw std::invalid_argument( + "SqueezeBreakoutStrategy: VALID_BARS must be >= 1"); + } + if (bufferPips < 0) { + throw std::invalid_argument( + "SqueezeBreakoutStrategy: BUFFER_PIPS must be >= 0"); + } + if (squeezeVars->MAX_TRADE_DURATION_MINUTES < 0) { + throw std::invalid_argument( + "SqueezeBreakoutStrategy: MAX_TRADE_DURATION_MINUTES must be >= 0"); + } + maxTradeDuration = + std::chrono::minutes{squeezeVars->MAX_TRADE_DURATION_MINUTES}; + // Window minimum: once decide()'s warm-up gate passes, the signal span is + // exactly OHLC_COUNT long (closed = OHLC_COUNT - 1). The oldest pattern + // candidate sits VALID_BARS back and needs its own lookback — one + // predecessor for the inside bar, NR_LOOKBACK - 1 for NR-N — so this + // proves every index the scan touches in range. + const int patternLookback = std::max(1, nrLookback - 1); + if (signalCfg.OHLC_COUNT < validBars + patternLookback + 1) { + throw std::invalid_argument( + "SqueezeBreakoutStrategy: OHLC_VARIABLES[0].OHLC_COUNT must be >= " + "VALID_BARS + max(1, NR_LOOKBACK - 1) + 1"); + } +} + +void SqueezeBreakoutStrategy::during(const PriceData& price, + const bars::BarStore&, + TradeManager& tradeManager) { + // Time cap (shared closeIfPastCap mechanics): close this symbol's trade + // once open STRICTLY longer than maxTradeDuration; <= 0 disables and + // preserves the original exits-are-central no-op behaviour. + strategy_exits::closeIfPastCap(price, tradeManager, maxTradeDuration); +} + +bool SqueezeBreakoutStrategy::isPatternAt( + std::span closedBars, std::size_t index) const { + if (nrLookback == 0) { + // Inside bar: the whole range sits within the predecessor's (bounds + // inclusive — an equal-high/low bar is still "no expansion"). + const OhlcObject& bar = closedBars[index]; + const OhlcObject& mother = closedBars[index - 1]; + return bar.high <= mother.high && bar.low >= mother.low; + } + // NR-N: strictly the narrowest high-low range of the last N closed bars + // (ties lose — a repeat of an earlier width is no new contraction). + const std::int32_t range = + closedBars[index].high - closedBars[index].low; + for (std::size_t back = 1; back < static_cast(nrLookback); + ++back) { + const OhlcObject& other = closedBars[index - back]; + if (range >= other.high - other.low) { + return false; + } + } + return true; +} + +std::optional SqueezeBreakoutStrategy::decide( + const PriceData& tick, const bars::BarStore& barStore) { + // This symbol's bar histories; nullptr means the timeframe was never + // registered on the store or the symbol has not ticked yet — either way + // there is nothing to decide on. + const std::vector* signalSeries = barStore.find( + tick.symbol, std::chrono::minutes{signalCfg.OHLC_MINUTES}); + const std::vector* trendSeries = + barStore.find(tick.symbol, std::chrono::minutes{trendCfg.OHLC_MINUTES}); + if (signalSeries == nullptr || trendSeries == nullptr) { + return std::nullopt; + } + + // Warm-up gate: no signals until both timeframes have a full window. + const auto signalCount = static_cast(signalCfg.OHLC_COUNT); + const auto trendCount = static_cast(trendCfg.OHLC_COUNT); + if (signalSeries->size() < signalCount || trendSeries->size() < trendCount) { + return std::nullopt; + } + + // Read only this strategy's tail of each history: the store keeps the + // LARGEST window registered per timeframe, so another consumer (the ATR + // entry gate) may have deepened a series beyond OHLC_COUNT. + const std::span signalBars = + std::span(*signalSeries).last(signalCount); + const std::span trendBars = + std::span(*trendSeries).last(trendCount); + + // --- 1. THE TREND FILTER --- + // The exact OhlcBreakoutStrategy idiom: EMA over the trend timeframe's + // closes (chronological, in-progress bar included), period = half the + // window, both close and EMA read at the last index. See that strategy's + // header for the accepted single-tick-spike risk and its mitigations. + closesScratch.clear(); + for (const OhlcObject& bar : trendBars) { + closesScratch.push_back(bar.close); + } + const int emaPeriod = static_cast(closesScratch.size() / 2); + ema::calculate(closesScratch, emaPeriod, emaScratch); + const std::int32_t currentClose = closesScratch.back(); + const std::int32_t currentEma = emaScratch.back(); + const bool uptrend = currentClose > currentEma; + const bool downtrend = currentClose < currentEma; + if (!uptrend && !downtrend) { + return std::nullopt; + } + + // --- 2. THE PATTERN SCAN --- + // Newest-first over the last VALID_BARS closed bars (the last span + // element is the in-progress bar and never a pattern). Pips scale UP to + // points (pips x pointsPerPip), the OhlcBreakoutStrategy convention; an + // unknown symbol returns scale 0, which just disables the buffer rather + // than corrupting the levels. + const std::span closedBars = + signalBars.first(signalBars.size() - 1); + const std::int32_t bufferPoints = bufferPips * symbol_scale::get(tick.symbol); + for (std::size_t back = 0; back < static_cast(validBars); + ++back) { + const std::size_t index = closedBars.size() - 1 - back; + if (!isPatternAt(closedBars, index)) { + continue; + } + const OhlcObject& pattern = closedBars[index]; + if (uptrend && tick.bid > pattern.high + bufferPoints) { + return Direction::LONG; + } + if (downtrend && tick.ask < pattern.low - bufferPoints) { + return Direction::SHORT; + } + } + return std::nullopt; +} diff --git a/source/strategies/strategy.cppm b/source/strategies/strategy.cppm index f078780..2b7da02 100644 --- a/source/strategies/strategy.cppm +++ b/source/strategies/strategy.cppm @@ -7,6 +7,7 @@ export module strategy; import std; // replaces , +import barStore; // bars::BarStore — the shared per-symbol bar histories import priceData; // PriceData import trade; // Direction import tradeManager; // TradeManager @@ -43,7 +44,15 @@ public: // Entry signal. Returns `std::nullopt` to mean "no trade". // Non-const because some implementations (e.g. RandomStrategy) // mutate internal RNG state on each call. - virtual std::optional decide(const PriceData& tick) = 0; + // + // `barStore` is the loop owner's shared bar pipeline (runLoop / a live + // worker), updated once per tick BEFORE this call — so decide() sees + // bar state that already includes the tick it is judging (the + // in-progress bar's close IS this tick). Strategies own no bar state of + // their own — the store is the single pipeline shared with the + // pre-decide ATR entry conditions. + virtual std::optional decide(const PriceData& tick, + const bars::BarStore& barStore) = 0; // Called every tick. Receives the TradeManager by mutable // reference so strategies can both inspect open positions @@ -54,6 +63,12 @@ public: // analogue is just passing the manager as a parameter; C# has no // distinction between reference and pointer so the by-ref nature // is implicit there. + // `barStore` is the same shared pipeline decide() sees — passed here + // because bar-based exit logic cannot live in decide(): the run loop + // gates decide() behind "no open trade for this symbol", so a strategy + // managing an open position only ever observes bars from this hook (and + // a live position can be a broker-seeded trade decide() never saw). virtual void during(const PriceData& price, + const bars::BarStore& barStore, TradeManager& tradeManager) = 0; }; diff --git a/source/strategies/strategyFactory.cppm b/source/strategies/strategyFactory.cppm new file mode 100644 index 0000000..9f13cde --- /dev/null +++ b/source/strategies/strategyFactory.cppm @@ -0,0 +1,77 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module strategyFactory; + +import std; // replaces , , +import strategy; // IStrategy +import randomStrategy; // RandomStrategy +import ohlcBreakoutStrategy; // OhlcBreakoutStrategy +import fvgStrategy; // FvgStrategy +import keltnerFadeStrategy; // KeltnerFadeStrategy +import sessionRangeBreakoutStrategy; // SessionRangeBreakoutStrategy +import squeezeBreakoutStrategy; // SqueezeBreakoutStrategy +import nyOpenRangeBreakoutStrategy; // NyOpenRangeBreakoutStrategy +import liquiditySweepReversalStrategy; // LiquiditySweepReversalStrategy +import rangeVelocityStrategy; // RangeVelocityStrategy +import strategyErrors; // UnknownStrategyError + +export namespace strategies { + +// The strategy names live trading considers active: winners are only pulled +// from the weekly winners index for these strategies (liveWinners), and each +// name here +// must have a branch in makeStrategy below. +inline constexpr std::array kActiveStrategies{ + "RandomStrategy", "OhlcBreakoutStrategy", "FvgStrategy", + "KeltnerFadeStrategy", "SessionRangeBreakoutStrategy", + "SqueezeBreakoutStrategy", "NyOpenRangeBreakoutStrategy", + "LiquiditySweepReversalStrategy", "RangeVelocityStrategy"}; + +// Instantiate the strategy named by config.TRADING_VARIABLES.STRATEGY. Shared +// by the backtest path (Operations) and the live path (liveCommand), so adding +// a new strategy means adding one branch here; neither caller needs to know +// about the concrete type. Throws UnknownStrategyError on an unrecognised +// name; concrete constructors may throw std::invalid_argument on a malformed +// config (OhlcBreakoutStrategy validates its OHLC timeframes). +std::unique_ptr makeStrategy( + const tradingDefinitions::StrategyConfig& config) { + const auto& name = config.TRADING_VARIABLES.STRATEGY; + if (name == "RandomStrategy") { + return std::make_unique(config); + } + if (name == "OhlcBreakoutStrategy") { + return std::make_unique(config); + } + if (name == "FvgStrategy") { + return std::make_unique(config); + } + if (name == "KeltnerFadeStrategy") { + return std::make_unique(config); + } + if (name == "SessionRangeBreakoutStrategy") { + return std::make_unique(config); + } + if (name == "SqueezeBreakoutStrategy") { + return std::make_unique(config); + } + if (name == "NyOpenRangeBreakoutStrategy") { + return std::make_unique(config); + } + if (name == "LiquiditySweepReversalStrategy") { + return std::make_unique(config); + } + if (name == "RangeVelocityStrategy") { + return std::make_unique(config); + } + throw UnknownStrategyError(name); +} + +} // namespace strategies diff --git a/source/strategies/timeCapExit.cppm b/source/strategies/timeCapExit.cppm new file mode 100644 index 0000000..2241377 --- /dev/null +++ b/source/strategies/timeCapExit.cppm @@ -0,0 +1,40 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module timeCapExit; + +import std; // replaces , +import priceData; // PriceData +import trade; // Trade, Direction +import tradeManager; // TradeManager + +export namespace strategy_exits { + +// The MAX_TRADE_DURATION_MINUTES exit shared by every strategy's during(): +// close the tick symbol's open trade once it has been open STRICTLY longer +// than `cap` (a trade exactly at the cap stays open), at the exit-side +// price — a LONG closes at the bid, a SHORT at the ask, matching +// exit_rules::checkExit. cap <= 0 disables the exit. Returns true when it +// closed the trade so callers with further exit logic (e.g. RangeVelocity's +// opposite-run exit) can stop managing a position that no longer exists. +inline bool closeIfPastCap(const PriceData& price, TradeManager& tradeManager, + std::chrono::minutes cap) { + if (cap <= std::chrono::minutes::zero()) { + return false; + } + const Trade* trade = tradeManager.findActiveTrade(price.symbol); + if (trade == nullptr || price.timestamp - trade->openTime <= cap) { + return false; + } + // Read the direction BEFORE closeTrade — closing erases the map node the + // pointer aims into. + const std::int32_t closePrice = + trade->direction == Direction::LONG ? price.bid : price.ask; + tradeManager.closeTrade(price.symbol, closePrice, price); + return true; +} + +} // namespace strategy_exits diff --git a/source/tracking/dealPacket.cppm b/source/tracking/dealPacket.cppm new file mode 100644 index 0000000..de99341 --- /dev/null +++ b/source/tracking/dealPacket.cppm @@ -0,0 +1,175 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// dealPacket — the on-the-wire UDP deal/account-update format and its decoder. +// +// When the IG Lightstreamer feed pushes a TRADE:* account update (an OPU +// field), the C# producer (vortex igmarkets/SubListener.cs) parses it into a +// Deal and shared/DealSerializer.cs fires it at udp_ports::kTrade as a single +// datagram — the same hand-rolled, fixed-size, little-endian contract as the +// tick feed (tickPacket), on its own port. One datagram = one deal, exactly +// kPacketSize bytes; anything else is discarded. +// +// offset size type field +// 0 8 double level NaN when absent +// 8 8 double size NaN when absent +// 16 8 double stopLevel NaN when absent +// 24 8 double limitLevel NaN when absent +// 32 32 char[] dealReference +// 64 32 char[] dealId +// 96 32 char[] dealIdOrigin +// 128 32 char[] epic e.g. "IX.D.ASX.IFS.IP" +// 160 8 char[] direction "BUY" | "SELL" +// 168 12 char[] status "OPEN" | "UPDATED" | "DELETED" +// 180 12 char[] dealStatus "ACCEPTED" | "REJECTED" +// 192 4 char[] currency e.g. "GBP" +// 196 16 char[] channel e.g. "WTP", "PublicRestOTC" +// 212 8 char[] expiry e.g. "-", "DFB" +// 220 24 char[] timestamp IG's raw "yyyy-MM-ddTHH:mm:ss.fff" +// 244 8 char[] guaranteedStop "true" | "false" | "" +// 252 4 — reserved always zero +// +// String fields are ASCII, NUL-padded, and always NUL-terminated (the producer +// truncates content to field size - 1); an absent value is an empty field. +// decodeDeal surfaces the NaN-when-absent doubles as std::optional and the +// char fields as std::string, so consumers never touch the raw layout. + +module; + +#include // offsetof is a macro, so `import std` alone can't supply it + +export module dealPacket; + +import std; // , , , , , , + +export namespace deal_packet { + +inline constexpr std::size_t kPacketSize = 256; // == DealSerializer.PacketSize + +namespace detail { + +#pragma pack(push, 1) +// Byte-for-byte overlay of the wire packet (the byte map above). The deploy +// targets (x86-64, aarch64) are little-endian and every double sits at a +// naturally aligned offset, so a bit_cast of the datagram fills it directly — +// no per-field readLE walk like the 3-field tick packet needs. +struct DealMessage { + double level; + double size; + double stopLevel; + double limitLevel; + char dealReference[32]; + char dealId[32]; + char dealIdOrigin[32]; + char epic[32]; + char direction[8]; + char status[12]; + char dealStatus[12]; + char currency[4]; + char channel[16]; + char expiry[8]; + char timestamp[24]; + char guaranteedStop[8]; + char reserved[4]; +}; +#pragma pack(pop) + +// Pin the overlay to the documented contract at compile time — a reordered or +// resized member can't silently shift every field after it. +static_assert(sizeof(DealMessage) == kPacketSize, "wire format is 256 bytes"); +static_assert(offsetof(DealMessage, dealReference) == 32); +static_assert(offsetof(DealMessage, dealId) == 64); +static_assert(offsetof(DealMessage, dealIdOrigin) == 96); +static_assert(offsetof(DealMessage, epic) == 128); +static_assert(offsetof(DealMessage, direction) == 160); +static_assert(offsetof(DealMessage, status) == 168); +static_assert(offsetof(DealMessage, dealStatus) == 180); +static_assert(offsetof(DealMessage, currency) == 192); +static_assert(offsetof(DealMessage, channel) == 196); +static_assert(offsetof(DealMessage, expiry) == 212); +static_assert(offsetof(DealMessage, timestamp) == 220); +static_assert(offsetof(DealMessage, guaranteedStop) == 244); +static_assert(offsetof(DealMessage, reserved) == 252); + +// NaN is the producer's "absent" marker for the optional doubles. +[[nodiscard]] inline std::optional presentOrNullopt(const double value) noexcept { + if (std::isnan(value)) { + return std::nullopt; + } + return value; +} + +// Fixed-width, NUL-padded ASCII field -> owned string, trimmed at the first +// NUL. The producer always NUL-terminates, but a full-width field (no NUL) is +// still safe — the copy never reads past the array. +template +[[nodiscard]] std::string toString(const char (&field)[N]) { + const auto end = std::find(std::begin(field), std::end(field), '\0'); + return {std::begin(field), end}; +} + +} // namespace detail + +// One decoded deal/account update, with the wire's absence conventions +// (NaN doubles, empty strings) mapped to friendly types. +struct Deal { + std::optional level; // nullopt when IG sent no value + std::optional size; // nullopt when IG sent no value + std::optional stopLevel; // nullopt when IG sent no value + std::optional limitLevel; // nullopt when IG sent no value + std::string dealReference; + std::string dealId; + std::string dealIdOrigin; + std::string epic; // e.g. "IX.D.ASX.IFS.IP" + std::string direction; // "BUY" | "SELL" + std::string status; // "OPEN" | "UPDATED" | "DELETED" + std::string dealStatus; // "ACCEPTED" | "REJECTED" + std::string currency; // e.g. "GBP" + std::string channel; // e.g. "WTP", "PublicRestOTC" + std::string expiry; // e.g. "-", "DFB" + std::string timestamp; // IG's raw "yyyy-MM-ddTHH:mm:ss.fff" string + std::string guaranteedStop; // "true" | "false" | "" (absent) +}; + +// Decode one datagram. Returns nullopt — i.e. "drop it", since UDP is +// best-effort and a stray datagram must not take the receiver down — only for +// a wrong-sized packet, the single structural rule of the contract. Field +// content is passed through as-is: unlike ticks there is no price/timestamp +// plausibility gate here, because a deal is an account event whose meaning +// (including absent fields) is for the tracking consumer to judge. +[[nodiscard]] inline std::optional decodeDeal(std::span bytes) { + if (bytes.size() != kPacketSize) { + return std::nullopt; + } + + // bit_cast keeps the overlay well-defined (no reinterpret_cast aliasing UB); + // on the little-endian deploy targets the bytes map straight onto the + // native scalars with no swapping. + std::array raw{}; + std::ranges::copy(bytes, raw.begin()); + const auto msg = std::bit_cast(raw); + + Deal deal; + deal.level = detail::presentOrNullopt(msg.level); + deal.size = detail::presentOrNullopt(msg.size); + deal.stopLevel = detail::presentOrNullopt(msg.stopLevel); + deal.limitLevel = detail::presentOrNullopt(msg.limitLevel); + deal.dealReference = detail::toString(msg.dealReference); + deal.dealId = detail::toString(msg.dealId); + deal.dealIdOrigin = detail::toString(msg.dealIdOrigin); + deal.epic = detail::toString(msg.epic); + deal.direction = detail::toString(msg.direction); + deal.status = detail::toString(msg.status); + deal.dealStatus = detail::toString(msg.dealStatus); + deal.currency = detail::toString(msg.currency); + deal.channel = detail::toString(msg.channel); + deal.expiry = detail::toString(msg.expiry); + deal.timestamp = detail::toString(msg.timestamp); + deal.guaranteedStop = detail::toString(msg.guaranteedStop); + return deal; +} + +} // namespace deal_packet diff --git a/source/tracking/trackingCommand.cppm b/source/tracking/trackingCommand.cppm new file mode 100644 index 0000000..847506b --- /dev/null +++ b/source/tracking/trackingCommand.cppm @@ -0,0 +1,215 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// trackingCommand — the `tracking` subcommand. Receives the IG markets +// producer's trade-positioning updates (256-byte deal datagrams, see +// dealPacket) over UDP and, per deal, mirrors the C# trade-tracking service +// (vortex/trade_tracking/Program.cs): look up the originating position in +// Redis (PH# history first, PO# live fallback), compute the close pips for a +// DELETED deal, archive its book entry (PO# -> PH#, pruning the PL# list — +// the one step the C# left to expiry, which lost broker-stop closes), and +// queue the ElasticTradeLogs document to "live-trades". The pure pieces +// (lookup order, pip math, doc shape) live in trackingReport; this file is +// the wiring, structured like ingestCommand around the shared +// net::UdpReceiver. +// +// The GMF only #includes Asio-free headers (the receiver, Redis and Elastic +// machinery all hide their transports behind .cpp files), so it is safe to +// `import std` here. + +module; + +#include "run/reporting/elasticPublisher.hpp" +#include "shared/net/udpPorts.hpp" +#include "shared/net/udpReceiver.hpp" +#include "shared/redis/positionManager.hpp" +#include "shared/utilities/backtestLog.hpp" // backtest_log::error +#include "shared/utilities/env.hpp" + +export module trackingCommand; + +import std; +import backtestLog; // backtest_log::logLine — timestamped, flushed stdout +import dealPacket; // deal_packet::decodeDeal +import marketDefinitions; // live::findMarketByEpicMini — epic -> symbol fallback +import symbolScale; // symbol_scale::get / getPriceScale +import trackingReport; // the pure lookup/pip/document helpers + +export class TrackingCommand { +public: + static int run(int argc, const char* argv[]); +}; + +namespace { + +// Parse a port from a string, returning `fallback` on empty/garbage input so a +// stray env var can't crash startup. +std::uint16_t parsePort(std::string_view text, std::uint16_t fallback) { + unsigned value = 0; + const auto [ptr, ec] = std::from_chars(text.data(), text.data() + text.size(), value); + if (ec != std::errc{} || value == 0 || value > 65535) { + return fallback; + } + return static_cast(value); +} + +// The wire marks an absent double as NaN (surfaced here as nullopt); log it as +// "-" so an unset stop/limit reads as deliberately absent, not zero. +std::string fmtOptional(const std::optional& value) { + return value ? std::format("{}", *value) : std::string("-"); +} + +} // namespace + +int TrackingCommand::run(const int argc, const char* argv[]) { + using backtest_log::logLine; + + const std::string bindAddr = env::getOr("TRACKING_BIND_ADDR", "127.0.0.1"); + + // Bind port: CLI arg (argv[2]) overrides $TRACKING_UDP_PORT overrides kTrade. + std::uint16_t bindPort = parsePort(env::getOr("TRACKING_UDP_PORT", ""), udp_ports::kTrade); + if (argc >= 3) { + bindPort = parsePort(argv[2], bindPort); + } + + const std::string redisHost = env::getOr("REDIS_HOST", "127.0.0.1"); + constexpr int redisPort = 6379; // by convention, as positionsCommand + std::string tradingEnv = env::getOr("TRADING_ENVIRONMENT", "demo"); + std::ranges::transform(tradingEnv, tradingEnv.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + + // Not thread-safe, but the receiver invokes the handler on its run() + // thread only, so one instance in this scope is the whole story. Redis + // being unreachable degrades to fallback documents, never a crash: the + // getters return outer-nullopt (logged) and lookupPosition misses. + redis_positions::PositionManager positions(redisHost, redisPort); + + std::atomic received{0}; + std::atomic dropped{0}; + + // The enrichment pipeline for one decoded deal — the C# receive-loop + // body. Split from the receiver lambda so the handler reads as + // decode-log-report. + const auto report = [&](const deal_packet::Deal& deal) { + const auto record = tracking_report::lookupPosition( + deal.dealReference, + [&](const std::string& ref) { + return positions.getHistoryPositionPayload(ref); + }, + [&](const std::string& ref) { + return positions.getPositionPayload(ref); + }); + if (!record) { + logLine("TrackingCommand: no PH#/PO# record for ref={} — writing " + "doc with fallbacks", + deal.dealReference); + } + + // Last-resort symbol when the book misses: map the epic back through + // marketDefinitions, else ship the raw epic (the C# used deal.Epic). + std::string fallbackSymbol = deal.epic; + if (const auto* market = live::findMarketByEpicMini(deal.epic)) { + fallbackSymbol = std::string(market->symbol); + } + + // The C# pip calculator: only a DELETED (closed) deal with a known + // position and a real close level. Sign and size come from the + // position — the deal's own direction is the closing side. + std::optional pips; + if (record && deal.level && *deal.level != 0.0 + && deal.status == "DELETED") { + const std::string& symbol = + !record->symbol.empty() ? record->symbol : fallbackSymbol; + pips = tracking_report::computeClosePips( + *deal.level, record->level, record->direction, record->size, + symbol_scale::getPriceScale(symbol), symbol_scale::get(symbol)); + if (pips) { + logLine("TrackingCommand: {} TRADE UPDATE — close pips: " + "({} - {} points) for {} size={} => {} pips", + symbol, *deal.level, record->level, record->direction, + record->size, *pips); + } else { + logLine("TrackingCommand: unknown scale for symbol {} — " + "skipping pip calc", + symbol); + } + } + + // Archive the book entry the moment the broker says DELETED: PO# -> + // PH# (60 days) and out of the PL# list. Idempotent when the + // strategy-close path already moved it (a missing PO# skips the PH# + // write). An empty strategyId would address a malformed PL# key. + if (record && deal.status == "DELETED" && !record->strategyId.empty()) { + const bool archived = + positions.removePosition(record->strategyId, deal.dealReference); + logLine("TrackingCommand: archived ref={} PO#->PH# for strategy " + "{} (ok={})", + deal.dealReference, record->strategyId, archived); + } + + elastic::enqueueDocument( + "live-trades", + tracking_report::buildLiveTradeDocument(deal, record, fallbackSymbol, + tradingEnv, + elastic::nowIsoUtc(), pips)); + }; + + net::UdpReceiver receiver( + bindAddr, bindPort, [&](std::span bytes) { + const auto deal = deal_packet::decodeDeal(bytes); + if (!deal) { + dropped.fetch_add(1, std::memory_order_relaxed); + return; + } + received.fetch_add(1, std::memory_order_relaxed); + // Deals are account-level events (a handful per day, not a tick + // stream), so unlike ingest each one is logged in full — no + // throughput reporter needed. + logLine("TrackingCommand: {} {} {} {} size={} level={} stop={} " + "limit={} dealId={} ref={} origin={} channel={} expiry={} " + "currency={} guaranteedStop={} ts={}", + deal->status, deal->dealStatus, deal->epic, deal->direction, + fmtOptional(deal->size), fmtOptional(deal->level), + fmtOptional(deal->stopLevel), fmtOptional(deal->limitLevel), + deal->dealId, deal->dealReference, deal->dealIdOrigin, + deal->channel, deal->expiry, deal->currency, + deal->guaranteedStop, deal->timestamp); + // A surprise from the report path (Redis payload, serialization) + // must not take the receive loop down — log and move on, like the + // C# swallow-and-log. + try { + report(*deal); + } catch (const std::exception& e) { + backtest_log::error(std::format( + "TrackingCommand: reporting ref={} failed ({})", + deal->dealReference, e.what())); + } catch (...) { + backtest_log::error(std::format( + "TrackingCommand: reporting ref={} failed (non-std " + "exception)", + deal->dealReference)); + } + }); + + logLine("TrackingCommand: starting; binding udp://{}:{} for {}-byte deal " + "packets ([{}], redis {}:{})", + bindAddr, bindPort, deal_packet::kPacketSize, tradingEnv, redisHost, + redisPort); + + const bool ok = receiver.run(); // binds the socket, then blocks until SIGINT/SIGTERM + if (!ok) { + return 1; // bind failed (bad address / port in use) — logged + } + + // dropped: datagrams that weren't exactly 256 bytes (stray/garbled traffic). + logLine("TrackingCommand: shutting down (received={}, dropped={})", + received.load(), dropped.load()); + // Deliver the reporting tail now, deterministically, rather than leaving + // it to the publisher's exit-time flush. + elastic::flushQueuedDocuments(); + return 0; +} diff --git a/source/tracking/trackingReport.cppm b/source/tracking/trackingReport.cppm new file mode 100644 index 0000000..5c56014 --- /dev/null +++ b/source/tracking/trackingReport.cppm @@ -0,0 +1,173 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// trackingReport — the pure half of the tracking consumer, the C++ port of +// the C# trade-tracking service's enrichment (vortex trade_tracking/ +// Program.cs). trackingCommand feeds each decoded deal through these +// helpers to produce the "live-trades" Elastic document: +// +// lookupPosition — the C# FetchPosition: history (PH#) first, live +// (PO#) as the fallback, via injected getters +// computeClosePips — the C# pip calculator, scales injected the same +// way positionSync::makeFreshRecord takes its +// priceScale (tests exercise the arithmetic, not +// the current symbolScale table) +// serializeDeal — the C# JsonSerializer.Serialize(deal) analogue +// for the document's raw-deal `json` field +// buildLiveTradeDocument— the C# ElasticTradeLogs shape +// +// Everything here is I/O-free: Redis reads arrive as std::function getters, +// the env/date/symbol-fallback strings are passed in, and the return values +// are plain data — so the unit tests need no servers. +// +// The GMF only #includes Asio-free headers (the Redis connection machinery +// stays behind positionManager.cpp), so it is safe to `import std` here; +// nlohmann in a module GMF follows liveWinners. + +module; + +#include + +#include "shared/redis/positionManager.hpp" + +export module trackingReport; + +import std; +import dealPacket; // deal_packet::Deal — the decoded 256-byte datagram + +namespace { + +// Absent wire doubles (NaN -> nullopt) serialize as JSON null, matching the +// C# nullable doubles. +nlohmann::json numberOrNull(const std::optional& value) { + return value ? nlohmann::json(*value) : nlohmann::json(nullptr); +} + +// Wire strings are producer-truncated ASCII, but a garbled datagram must +// lose at most a character, never the whole document. +std::string dumpSafe(const nlohmann::json& doc) { + return doc.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); +} + +} // namespace + +export namespace tracking_report { + +// One raw payload read keyed by dealReference, with PositionManager's +// double-optional contract: outer nullopt = Redis failure (state UNKNOWN), +// inner nullopt = key missing. +using PayloadGetter = + std::function>(const std::string&)>; + +// The C# FetchPosition lookup order: history (PH#) first — a DELETED deal's +// position has usually already been archived — then live (PO#) as the +// fallback. A getter that fails, misses, or returns an undecodable payload +// falls through to the next; nullopt when neither yields a decodable record. +[[nodiscard]] std::optional lookupPosition( + const std::string& dealReference, const PayloadGetter& getHistory, + const PayloadGetter& getLive) { + for (const PayloadGetter* getter : {&getHistory, &getLive}) { + const auto payload = (*getter)(dealReference); + if (!payload || !payload->has_value()) { + continue; // Redis failure or missing key — try the next family + } + if (auto record = redis_positions::decodePositionRecord(**payload)) { + return record; + } + } + return std::nullopt; +} + +// Pips for a closed deal, the C# `(level - position.level) * scaling * dir * +// size`. closeLevel is IG's raw decimal price off the wire (deal.level); +// openLevelPoints is the PO#/PH# record's scaled INT32 points (decimal x +// priceScale, llround — see positionSync). The close is quantised with the +// same llround so both sides round identically: +// +// closePoints = llround(closeLevel * priceScale) +// pips = (closePoints - openLevelPoints) / pointsPerPip * dirSign * size +// +// dirSign follows the POSITION's direction, BUY = +1 and anything else = -1 +// (the C# ternary) — on a DELETED update the deal's own direction is the +// closing side, not the position's. nullopt when either scale is <= 0 +// (symbol_scale::kUnknown): an unknown symbol must skip the calc, not scale +// the P&L by zero. +[[nodiscard]] std::optional computeClosePips( + const double closeLevel, const std::int32_t openLevelPoints, + const std::string_view direction, const double size, const int priceScale, + const int pointsPerPip) { + if (priceScale <= 0 || pointsPerPip <= 0) { + return std::nullopt; + } + const auto closePoints = + static_cast(std::llround(closeLevel * priceScale)); + const double dirSign = direction == "BUY" ? 1.0 : -1.0; + return (closePoints - static_cast(openLevelPoints)) / pointsPerPip + * dirSign * size; +} + +// All 16 Deal fields, absent optionals as JSON null — the raw-deal audit +// string the C# stored as ElasticTradeLogs.json. +[[nodiscard]] std::string serializeDeal(const deal_packet::Deal& deal) { + const nlohmann::json doc{ + {"level", numberOrNull(deal.level)}, + {"size", numberOrNull(deal.size)}, + {"stopLevel", numberOrNull(deal.stopLevel)}, + {"limitLevel", numberOrNull(deal.limitLevel)}, + {"dealReference", deal.dealReference}, + {"dealId", deal.dealId}, + {"dealIdOrigin", deal.dealIdOrigin}, + {"epic", deal.epic}, + {"direction", deal.direction}, + {"status", deal.status}, + {"dealStatus", deal.dealStatus}, + {"currency", deal.currency}, + {"channel", deal.channel}, + {"expiry", deal.expiry}, + {"timestamp", deal.timestamp}, + {"guaranteedStop", deal.guaranteedStop}, + }; + return dumpSafe(doc); +} + +// The C# ElasticTradeLogs "live-trades" document (same shape as the order +// path's auditTrade, plus the tracking-only json/level/pips fields). +// fallbackSymbol is the epic-mapped internal symbol (or the raw epic) — +// injected so this module never touches the marketDefinitions table. pips is +// pre-computed by the caller (computeClosePips); nullopt omits the field. +// Key order is nlohmann-alphabetical — unlike the PO# payload this is not a +// shared wire contract, Elastic doesn't care. +[[nodiscard]] std::string buildLiveTradeDocument( + const deal_packet::Deal& deal, + const std::optional& position, + const std::string_view fallbackSymbol, const std::string_view env, + const std::string_view dateIso, const std::optional pips) { + nlohmann::json doc{ + {"date", dateIso}, + {"env", env}, + {"symbol", position && !position->symbol.empty() + ? position->symbol + : std::string(fallbackSymbol)}, + {"action", deal.status}, + {"strategy", position && !position->strategyId.empty() + ? position->strategyId + : std::string("Unknown")}, + {"dealReference", deal.dealReference}, + {"json", serializeDeal(deal)}, + }; + if (!deal.dealId.empty()) { + doc["dealId"] = deal.dealId; // the auditTrade omit-when-empty rule + } + if (deal.level && *deal.level != 0.0) { + doc["level"] = *deal.level; // IG sends 0 on some UPDATED events + } + if (pips) { + doc["pips"] = *pips; + } + return dumpSafe(doc); +} + +} // namespace tracking_report diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bba6641..25ebc8c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4,14 +4,50 @@ # they propagate here through the link below. add_executable(unit_tests symbolScale.cpp + marketHours.cpp jsonParser.cpp ohlc.cpp ema.cpp + atr.cpp + barStore.cpp + rangeBar.cpp + rangeVelocity.cpp + entryConditions.cpp ohlcBreakout.cpp + fvg.cpp + keltnerFade.cpp + sessionRangeBreakout.cpp + squeezeBreakout.cpp + nyOpenRangeBreakout.cpp + swingPivots.cpp + liquiditySweepReversal.cpp db.cpp + sqlManager.cpp sweep.cpp + chainMatcher.cpp + experimentConfig.cpp + experimentSweep.cpp + rollingWindow.cpp + outcomeIndices.cpp + tickCache.cpp tradeManager.cpp + tradeDocument.cpp tickPacket.cpp + dealPacket.cpp + liveWinners.cpp + liveRunner.cpp + liveStrategyCache.cpp + liveTrace.cpp + tradeLocks.cpp + positionManager.cpp + positionClustering.cpp + positionSync.cpp + trackingReport.cpp + orderRequest.cpp + orderChannel.cpp + igRequests.cpp + positionBook.cpp + positionCounterCache.cpp ) target_link_libraries(unit_tests PRIVATE diff --git a/tests/atr.cpp b/tests/atr.cpp new file mode 100644 index 0000000..a80e2fa --- /dev/null +++ b/tests/atr.cpp @@ -0,0 +1,98 @@ +#include + +#include +#include +#include + +import atr; +import ohlcObject; + +namespace { + +// ATR only reads high/low/close; open mirrors close and date is irrelevant. +OhlcObject candle(std::int32_t high, std::int32_t low, std::int32_t close) { + return OhlcObject{.open = close, .close = close, .high = high, .low = low}; +} + +} // namespace + +TEST_CASE("atr::calculate matches the C# reference shape", "[atr]") { + SECTION("fewer than period+1 candles returns 0 (not warm)") { + std::vector candles; + CHECK(atr::calculate(candles, 3) == 0); + for (int i = 0; i < 3; ++i) { + candles.push_back(candle(110010, 109990, 110000)); + } + CHECK(candles.size() == 3); // period 3 needs 4 + CHECK(atr::calculate(candles, 3) == 0); + } + + SECTION("contained ranges: high-low dominates, SMA of the TRs") { + const std::vector candles{ + candle(100, 100, 100), + candle(110, 95, 100), // prevClose 100 inside -> TR = 15 + candle(105, 98, 100), // TR = 7 + }; + CHECK(atr::calculate(candles, 2) == 11); // (15 + 7) / 2 + } + + SECTION("gap up: |high - prevClose| dominates") { + const std::vector candles{ + candle(100, 100, 100), + candle(130, 125, 128), // TR = max(5, 30, 25) = 30 + }; + CHECK(atr::calculate(candles, 1) == 30); + } + + SECTION("gap down: |low - prevClose| dominates") { + const std::vector candles{ + candle(100, 100, 100), + candle(80, 70, 75), // TR = max(10, 20, 30) = 30 + }; + CHECK(atr::calculate(candles, 1) == 30); + } + + SECTION("non-divisible sums round to the nearest point") { + const std::vector up{ + candle(100, 100, 100), + candle(110, 100, 100), // TR = 10 + candle(110, 95, 100), // TR = 15 + }; + CHECK(atr::calculate(up, 2) == 13); // 12.5 rounds up + + const std::vector down{ + candle(100, 100, 100), + candle(110, 100, 100), // TR = 10 + candle(110, 100, 100), // TR = 10 + candle(111, 100, 100), // TR = 11 + }; + CHECK(atr::calculate(down, 3) == 10); // 31/3 = 10.33 rounds down + } + + SECTION("only the last period+1 candles are read") { + const std::vector candles{ + candle(2000, 1, 500), // wild history that must not leak in + candle(3000, 1, 1000), + candle(100, 100, 100), // window starts here (prevClose source) + candle(110, 100, 105), // TR = 10 + candle(115, 105, 110), // TR = 10 + }; + CHECK(atr::calculate(candles, 2) == 10); + } + + SECTION("period below 1 throws") { + const std::vector candles{candle(100, 100, 100), + candle(110, 100, 105)}; + CHECK_THROWS_AS(atr::calculate(candles, 0), std::invalid_argument); + CHECK_THROWS_AS(atr::calculate(candles, -1), std::invalid_argument); + } + + SECTION("index-scale prices don't overflow") { + // ~4.5M points is the biggest value the engine stores (indices). + std::vector candles{candle(4'500'000, 4'500'000, 4'500'000)}; + for (int i = 0; i < 14; ++i) { + candles.push_back(candle(4'550'000, 4'450'000, 4'500'000)); + } + CHECK(atr::calculate(candles, 14) == 100'000); + } +} diff --git a/tests/barStore.cpp b/tests/barStore.cpp new file mode 100644 index 0000000..a4904b7 --- /dev/null +++ b/tests/barStore.cpp @@ -0,0 +1,240 @@ +#include + +#include +#include +#include // setenv — keep the store off QuestDB +#include +#include + +import barStore; +import ohlcObject; +import priceData; +import rangeBarBuilder; + +namespace { + +using std::chrono::minutes; +using std::chrono::seconds; + +const std::chrono::system_clock::time_point t0 = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; + +// Bars build from the ASK; the bid rides 2 points under and is ignored here. +PriceData tickAt(std::chrono::system_clock::duration offset, std::int32_t ask, + const std::string& symbol = "EURUSD") { + return PriceData(ask, ask - 2, t0 + offset, symbol); +} + +bars::BarStore makeStore() { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + return bars::BarStore{}; +} + +} // namespace + +TEST_CASE("BarStore rejects unusable series", "[barStore]") { + auto store = makeStore(); + CHECK_THROWS_AS(store.registerSeries(minutes{0}, 5), std::invalid_argument); + CHECK_THROWS_AS(store.registerSeries(minutes{15}, 0), std::invalid_argument); +} + +TEST_CASE("BarStore builds, rolls and trims a registered series", "[barStore]") { + auto store = makeStore(); + store.registerSeries(minutes{15}, 3); + + // Two ticks inside the first 15m window aggregate into ONE bar. + store.update(tickAt(minutes{0}, 110000)); + store.update(tickAt(minutes{10}, 110050)); + const auto* series = store.find("EURUSD", minutes{15}); + REQUIRE(series != nullptr); + REQUIRE(series->size() == 1); + CHECK(series->front().open == 110000); + CHECK(series->front().high == 110050); + CHECK(series->front().low == 110000); + CHECK(series->front().close == 110050); + CHECK_FALSE(series->front().complete); + + // 16 minutes on rolls a new bar; the finished one is marked complete. + store.update(tickAt(minutes{16}, 110100)); + REQUIRE(series->size() == 2); + CHECK(series->front().complete); + CHECK(series->back().open == 110100); + + // Two more rolls exceed the window of 3: the oldest bar drops off. + store.update(tickAt(minutes{32}, 110200)); + store.update(tickAt(minutes{48}, 110300)); + REQUIRE(series->size() == 3); + CHECK(series->front().open == 110100); // the 110000 bar was trimmed + CHECK(series->back().open == 110300); +} + +TEST_CASE("BarStore dedups a re-registered duration keeping the larger window", + "[barStore]") { + auto store = makeStore(); + store.registerSeries(minutes{15}, 3); + store.registerSeries(minutes{15}, 5); // same timeframe, deeper window + + for (int i = 0; i < 7; ++i) { + store.update(tickAt(minutes{16 * i}, 110000 + i * 10)); + } + const auto* series = store.find("EURUSD", minutes{15}); + REQUIRE(series != nullptr); + CHECK(series->size() == 5); // one shared history at the deeper window +} + +TEST_CASE("BarStore keeps registered timeframes independent", "[barStore]") { + auto store = makeStore(); + store.registerSeries(minutes{1}, 10); + store.registerSeries(minutes{60}, 10); + + // 2-minute spacing rolls a 1m bar per tick but stays inside one 60m bar. + for (int i = 0; i < 5; ++i) { + store.update(tickAt(minutes{2 * i}, 110000 + i)); + } + REQUIRE(store.find("EURUSD", minutes{1}) != nullptr); + REQUIRE(store.find("EURUSD", minutes{60}) != nullptr); + CHECK(store.find("EURUSD", minutes{1})->size() == 5); + CHECK(store.find("EURUSD", minutes{60})->size() == 1); +} + +TEST_CASE("BarStore keeps per-symbol histories isolated", "[barStore]") { + auto store = makeStore(); + store.registerSeries(minutes{15}, 4); + + store.update(tickAt(minutes{0}, 110000, "EURUSD")); + store.update(tickAt(seconds{30}, 65000, "AUDUSD")); + store.update(tickAt(minutes{16}, 110100, "EURUSD")); + + const auto* eur = store.find("EURUSD", minutes{15}); + const auto* aud = store.find("AUDUSD", minutes{15}); + REQUIRE(eur != nullptr); + REQUIRE(aud != nullptr); + CHECK(eur->size() == 2); + CHECK(aud->size() == 1); + CHECK(aud->front().high == 65000); // no EURUSD price leaked in +} + +TEST_CASE("BarStore find misses return nullptr", "[barStore]") { + auto store = makeStore(); + store.registerSeries(minutes{15}, 4); + store.update(tickAt(minutes{0}, 110000)); + + CHECK(store.find("EURUSD", minutes{5}) == nullptr); // never registered + CHECK(store.find("AUDUSD", minutes{15}) == nullptr); // symbol never ticked +} + +TEST_CASE("BarStore with no registered series ignores ticks", "[barStore]") { + auto store = makeStore(); + store.update(tickAt(minutes{0}, 110000)); // must not throw or allocate state + CHECK(store.find("EURUSD", minutes{15}) == nullptr); +} + +TEST_CASE("BarStore rejects unusable range series", "[barStore]") { + auto store = makeStore(); + CHECK_THROWS_AS(store.registerRangeSeries({0, 100, 8}), std::invalid_argument); + CHECK_THROWS_AS(store.registerRangeSeries({4, 0, 8}), std::invalid_argument); + CHECK_THROWS_AS(store.registerRangeSeries({4, 100, 0}), std::invalid_argument); +} + +TEST_CASE("BarStore feeds one update into both OHLC and range series", + "[barStore]") { + auto store = makeStore(); + store.registerSeries(minutes{15}, 3); + const rangebar::RangeBarSpec spec{.atrTickWindow = 4, + .atrPercent = 100, + .count = 8}; + store.registerRangeSeries(spec); + + // Four ticks: enough to warm the 4-tick range window; all inside the + // first 15m OHLC bucket. ONE update call per tick feeds both legs — the + // loop owners never change for a new bar type. + store.update(tickAt(minutes{0}, 110000)); + store.update(tickAt(minutes{1}, 110010)); + store.update(tickAt(minutes{2}, 110005)); + store.update(tickAt(minutes{3}, 110008)); + + const auto* ohlcSeries = store.find("EURUSD", minutes{15}); + REQUIRE(ohlcSeries != nullptr); + REQUIRE(ohlcSeries->size() == 1); + CHECK(ohlcSeries->front().high == 110010); + + const auto* rangeSeries = store.findRange("EURUSD", spec); + REQUIRE(rangeSeries != nullptr); + REQUIRE(rangeSeries->size() == 1); // bar #1 opened on the warm tick + CHECK(rangeSeries->front().open == 110008); + CHECK_FALSE(rangeSeries->front().complete); +} + +TEST_CASE("BarStore dedups a re-registered range identity keeping the larger " + "window", "[barStore]") { + auto store = makeStore(); + store.registerRangeSeries({.atrTickWindow = 1, .atrPercent = 100, .count = 2}); + store.registerRangeSeries({.atrTickWindow = 1, .atrPercent = 100, .count = 4}); + + // A 1-tick window is warm immediately with a floored threshold of 1, so + // alternating prices close a bar every second tick: 12 ticks -> 6 bars, + // trimmed to the MERGED window of 4 (not the first registration's 2). + for (int i = 0; i < 12; ++i) { + store.update(tickAt(seconds{i}, 110000 + (i % 2))); + } + const auto* series = + store.findRange("EURUSD", {.atrTickWindow = 1, .atrPercent = 100, .count = 2}); + REQUIRE(series != nullptr); + CHECK(series->size() == 4); // one shared history at the deeper window +} + +TEST_CASE("BarStore findRange misses return nullptr", "[barStore]") { + auto store = makeStore(); + const rangebar::RangeBarSpec spec{.atrTickWindow = 4, + .atrPercent = 100, + .count = 8}; + store.registerRangeSeries(spec); + store.update(tickAt(minutes{0}, 110000)); + + // Identity is (window, percent) — count is ignored, a different window or + // percent is a different series. + CHECK(store.findRange("EURUSD", {.atrTickWindow = 5, .atrPercent = 100, .count = 8}) == + nullptr); + CHECK(store.findRange("EURUSD", {.atrTickWindow = 4, .atrPercent = 50, .count = 8}) == + nullptr); + CHECK(store.findRange("AUDUSD", spec) == nullptr); // symbol never ticked +} + +TEST_CASE("BarStore with only range series registered still processes ticks", + "[barStore]") { + // Pins the early-return fix: no OHLC series must not short-circuit the + // range leg. + auto store = makeStore(); + const rangebar::RangeBarSpec spec{.atrTickWindow = 1, + .atrPercent = 100, + .count = 4}; + store.registerRangeSeries(spec); + + store.update(tickAt(minutes{0}, 110000)); + + REQUIRE(store.findRange("EURUSD", spec) != nullptr); + CHECK(store.findRange("EURUSD", spec)->size() == 1); + CHECK(store.find("EURUSD", minutes{15}) == nullptr); // no OHLC state grew +} + +TEST_CASE("BarStore keeps per-symbol range histories isolated", "[barStore]") { + auto store = makeStore(); + const rangebar::RangeBarSpec spec{.atrTickWindow = 1, + .atrPercent = 100, + .count = 4}; + store.registerRangeSeries(spec); + + store.update(tickAt(minutes{0}, 110000, "EURUSD")); + store.update(tickAt(seconds{30}, 65000, "AUDUSD")); + store.update(tickAt(minutes{1}, 110100, "EURUSD")); // closes EURUSD bar 1 + + const auto* eur = store.findRange("EURUSD", spec); + const auto* aud = store.findRange("AUDUSD", spec); + REQUIRE(eur != nullptr); + REQUIRE(aud != nullptr); + CHECK(eur->size() == 1); + CHECK(eur->front().complete); // 100-point move >= the floored threshold + CHECK(aud->size() == 1); + CHECK_FALSE(aud->front().complete); + CHECK(aud->front().open == 65000); // no EURUSD price leaked in +} diff --git a/tests/chainMatcher.cpp b/tests/chainMatcher.cpp new file mode 100644 index 0000000..440a398 --- /dev/null +++ b/tests/chainMatcher.cpp @@ -0,0 +1,569 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// Pins the matcher semantics stated in chainMatcher.cppm's module comment: +// touch-within-window completion, rolling (not anchored) leg 1, next-tick +// anchoring, whole-chain expiry with same-tick re-evaluation, non-overlap +// with full tracker clears, and the coverage rules for band/lookback legs. +// Synthetic mid/timestamp sequences only — no servers. + +#include + +#include +#include +#include +#include +#include + +#include "shared/experiments/experimentConfig.hpp" + +import chainMatcher; +import priceData; + +using chain_matcher::ChainMatcher; +using experiments::Activity; +using experiments::ActivityType; +using experiments::ExperimentConfig; + +namespace { + +// All timestamps are offsets (in seconds) from an arbitrary fixed epoch — +// the matcher only ever compares timestamps, never reads a clock. +std::chrono::system_clock::time_point at(const long seconds) { + const std::chrono::system_clock::time_point base{ + std::chrono::sys_days{std::chrono::year{2026} / std::chrono::January / 5}}; + return base + std::chrono::seconds(seconds); +} + +ExperimentConfig chainOf(std::vector chain) { + return ExperimentConfig{ + .UUID = "test-uuid", .NAME = "test", .CHAIN = std::move(chain)}; +} + +Activity move(const double signedPercent, const int windowSeconds) { + return Activity{.TYPE = ActivityType::DirectionalMove, + .MOVE_PERCENT = signedPercent, + .WINDOW_SECONDS = windowSeconds}; +} + +} // namespace + +TEST_CASE("DirectionalMove fires on a touch within the trailing window", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({move(-1.0, 600)})); + matcher.onTick(100000, 0, at(0)); + matcher.onTick(99600, 0, at(60)); + CHECK(matcher.stats().occurrences == 0); + // Exactly -1% from the rolling max, well inside the 10-minute window. + matcher.onTick(99000, 0, at(120)); + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("DirectionalMove does not fire when the drop is slower than the window", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({move(-1.0, 600)})); + matcher.onTick(100000, 0, at(0)); + matcher.onTick(99500, 0, at(550)); + // The 100000 print expired from the trailing 600s window (deque expiry), + // so the rolling max is 99500 and the remaining drop is only 0.5%. + matcher.onTick(99000, 0, at(1101)); + CHECK(matcher.stats().occurrences == 0); +} + +TEST_CASE("split drop still fires via rolling extremes (anti-re-anchor)", + "[chainMatcher]") { + // 0.6% then a further 0.5% inside one window: an anchored design that + // re-anchored at the first partial drop would miss this; the rolling max + // sees the cumulative 1.1%. + ChainMatcher matcher(chainOf({move(-1.0, 600)})); + matcher.onTick(100000, 0, at(0)); + matcher.onTick(99400, 0, at(100)); + CHECK(matcher.stats().occurrences == 0); + matcher.onTick(98900, 0, at(200)); + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("two-leg chain anchors at leg 1's completion tick", "[chainMatcher]") { + ChainMatcher matcher(chainOf({move(-1.0, 600), move(0.5, 600)})); + matcher.onTick(100000, 0, at(0)); + CHECK(matcher.activeLeg() == 0); + matcher.onTick(99000, 0, at(60)); // leg 1 completes: anchor (99000, 60) + CHECK(matcher.activeLeg() == 1); + CHECK(matcher.stats().occurrences == 0); + // +0.5% from the ANCHOR (99000 -> 99495), inside leg 2's own window. + matcher.onTick(99495, 0, at(120)); + CHECK(matcher.stats().occurrences == 1); + CHECK(matcher.activeLeg() == 0); +} + +TEST_CASE("a leg cannot complete on its own anchor tick", "[chainMatcher]") { + // Leg 2 is a trailing band whose condition ALREADY holds on the tick + // that completes leg 1 — but evaluation of leg N+1 starts on the NEXT + // tick, so the chain needs one more tick to finish. + ChainMatcher matcher(chainOf({ + move(-1.0, 600), + Activity{.TYPE = ActivityType::StaysInBand, + .MOVE_PERCENT = 5.0, + .WINDOW_SECONDS = 600, + .LOOKBACK_SECONDS = 300}, + })); + for (long t = 0; t <= 300; t += 30) { + matcher.onTick(100000, 0, at(t)); + } + CHECK(matcher.activeLeg() == 0); + matcher.onTick(99000, 0, at(330)); // -1%: leg 1 completes here + CHECK(matcher.activeLeg() == 1); + // The band (range 1000 <= 5% of mid, fully covered) held at the anchor + // tick too — but only the NEXT tick may complete the chain. + CHECK(matcher.stats().occurrences == 0); + matcher.onTick(99000, 0, at(360)); + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("an expired attempt counts a failure against the leg being sought", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({move(-1.0, 600), move(0.5, 60)})); + matcher.onTick(100000, 0, at(0)); + matcher.onTick(99000, 0, at(100)); // leg 1 completes: attempt 1 anchors + CHECK(matcher.stats().attempts == 1); + // 61s > leg 2's window, and this tick is only -0.6% off the rolling max — + // the chain dies (attributed to leg 2) WITHOUT re-anchoring. + matcher.onTick(99400, 0, at(161)); + CHECK(matcher.activeLeg() == 0); + CHECK(matcher.stats().attempts == 1); + CHECK(matcher.stats().occurrences == 0); + REQUIRE(matcher.stats().failuresByLeg.size() == 2); + CHECK(matcher.stats().failuresByLeg[0] == 0); // leg 1 never expires + CHECK(matcher.stats().failuresByLeg[1] == 1); +} + +TEST_CASE("mid-chain expiry fails the chain and the failing tick can start a new attempt", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({move(-1.0, 600), move(0.5, 60)})); + matcher.onTick(100000, 0, at(0)); + matcher.onTick(99000, 0, at(100)); // leg 1 completes: anchor (99000, 100) + CHECK(matcher.activeLeg() == 1); + // 61s > leg 2's 60s window: the attempt fails — and this same tick is a + // fresh -2% off the still-live rolling max, so leg 1 completes AGAIN on + // the failing tick (trackers were fed all along, no re-scan). + matcher.onTick(98000, 0, at(161)); + CHECK(matcher.stats().occurrences == 0); + CHECK(matcher.activeLeg() == 1); + // The failing tick counted BOTH the failure and the fresh attempt. + CHECK(matcher.stats().attempts == 2); + CHECK(matcher.stats().failuresByLeg[1] == 1); + // The new attempt is anchored at (98000, 161): +0.5% completes it. + matcher.onTick(98490, 0, at(200)); + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("non-overlap: back-to-back matches count, pre-match extremes cannot seed", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({move(-1.0, 600)})); + matcher.onTick(100000, 0, at(0)); + matcher.onTick(99000, 0, at(60)); + CHECK(matcher.stats().occurrences == 1); // first match clears ALL trackers + // 1.5% below the PRE-match max — but that extreme is gone; the fresh + // tracker's max is this tick itself, so nothing fires. + matcher.onTick(98500, 0, at(120)); + CHECK(matcher.stats().occurrences == 1); + // A full -1% against the POST-match max (98500 -> 98515 needed; 985 + // points is exactly 1%): back-to-back occurrence number two. + matcher.onTick(97515, 0, at(180)); + CHECK(matcher.stats().occurrences == 2); + // Single-leg chains: every leg-1 completion IS the whole chain, so + // attempts == occurrences by construction (and nothing can expire). + CHECK(matcher.stats().attempts == 2); + CHECK(matcher.stats().failuresByLeg == std::vector{0}); +} + +TEST_CASE("serial attempts: an interleaved second trigger yields one occurrence", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({move(-1.0, 600), move(0.5, 600)})); + matcher.onTick(100000, 0, at(0)); + matcher.onTick(99000, 0, at(60)); // attempt 1 anchors at 99000 + CHECK(matcher.activeLeg() == 1); + // A further drop that would qualify as a NEW leg-1 trigger — ignored, + // a single anchor is in flight (documented v1 undercount). + matcher.onTick(98000, 0, at(120)); + CHECK(matcher.activeLeg() == 1); + // Recovery to +0.5% off the FIRST anchor completes exactly one chain. + matcher.onTick(99495, 0, at(240)); + CHECK(matcher.stats().occurrences == 1); + CHECK(matcher.activeLeg() == 0); +} + +TEST_CASE("StaysInBand requires a full covered window before it can fire", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({Activity{.TYPE = ActivityType::StaysInBand, + .MOVE_PERCENT = 0.5, + .LOOKBACK_SECONDS = 300}})); + // Dead flat from the first tick — but the band cannot fire until a full + // 300s of real tick history has accumulated (warm-up is coverage too). + for (long t = 0; t <= 300; t += 30) { + matcher.onTick(100000, 0, at(t)); + CHECK(matcher.stats().occurrences == 0); + } + matcher.onTick(100000, 0, at(330)); + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("StaysInBand: a gap voids coverage until a fresh window accumulates", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({Activity{.TYPE = ActivityType::StaysInBand, + .MOVE_PERCENT = 5.0, + .LOOKBACK_SECONDS = 300}})); + // Oscillate outside the band (range ~5.8% > 5%) so nothing fires while + // the tracker is warm and covered... + for (long t = 0; t <= 600; t += 30) { + matcher.onTick(t % 60 == 0 ? 100000 : 106000, 0, at(t)); + } + CHECK(matcher.stats().occurrences == 0); + // ...then a 400s silent gap (> the 300s window). The post-gap window + // looks dead flat — the oscillating prints all expired — but a weekend + // gap must NOT satisfy "stays in band": coverage is void until 300s of + // dense post-gap ticks accumulate. + for (long t = 1000; t <= 1300; t += 30) { + matcher.onTick(100000, 0, at(t)); + CHECK(matcher.stats().occurrences == 0); + } + matcher.onTick(100000, 0, at(1330)); + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("NewExtreme fires on a strict new high, never before coverage", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({Activity{.TYPE = ActivityType::NewExtreme, + .LOOKBACK_SECONDS = 300, + .DIRECTION = 1}})); + // Steadily rising: every tick beats every prior print, but nothing may + // fire until the lookback is fully covered. + for (long t = 0; t <= 300; t += 30) { + matcher.onTick(static_cast(100000 + t / 3), 0, at(t)); + CHECK(matcher.stats().occurrences == 0); + } + matcher.onTick(100110, 0, at(330)); + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("NewExtreme is strict: equalling the lookback high does not fire", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({Activity{.TYPE = ActivityType::NewExtreme, + .LOOKBACK_SECONDS = 300, + .DIRECTION = 1}})); + // Dead flat, fully covered: the current mid always EQUALS the trailing + // high — a new high must strictly exceed it. + for (long t = 0; t <= 600; t += 30) { + matcher.onTick(100000, 0, at(t)); + } + CHECK(matcher.stats().occurrences == 0); + matcher.onTick(100001, 0, at(630)); // strictly above: fires + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("RangeRelativeMove clamps the threshold to >= 1 point", "[chainMatcher]") { + ChainMatcher matcher(chainOf({Activity{.TYPE = ActivityType::RangeRelativeMove, + .WINDOW_SECONDS = 600, + .LOOKBACK_SECONDS = 300, + .DIRECTION = -1, + .ATR_MULTIPLE = 0.3}})); + // Dead flat and covered: the basis range is 0, so an unclamped threshold + // would be 0 and a zero move would "fire" on every tick. The >= 1 clamp + // keeps a flat market silent. + for (long t = 0; t <= 600; t += 30) { + matcher.onTick(100000, 0, at(t)); + CHECK(matcher.stats().occurrences == 0); + } + // A 1-point drop: the basis range grows to 1, 1 x 0.3 rounds to 0, the + // clamp lifts it back to 1 point — met exactly by this move. + matcher.onTick(99999, 0, at(630)); + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("anchored RangeRelativeMove freezes its threshold at the anchor", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({ + move(-1.0, 600), + Activity{.TYPE = ActivityType::RangeRelativeMove, + .WINDOW_SECONDS = 600, + .LOOKBACK_SECONDS = 300, + .DIRECTION = -1, + .ATR_MULTIPLE = 2.0}, + })); + // Warm the basis flat at 101000, then a -1.02% drop completes leg 1. + // Basis range at the anchor = 101000 - 99980 = 1020, so leg 2's frozen + // threshold is 2040 points below the anchor (99980). + for (long t = 0; t <= 330; t += 30) { + matcher.onTick(101000, 0, at(t)); + } + matcher.onTick(99980, 0, at(360)); + CHECK(matcher.activeLeg() == 1); + // 2020 points below the anchor: 20 short of the frozen threshold. + matcher.onTick(97960, 0, at(420)); + CHECK(matcher.stats().occurrences == 0); + // 2040 points below the anchor fires. Had the threshold been recomputed + // live, the drop itself would have widened the basis range to ~3060 + // (threshold 6120) and this could never fire — freezing is the contract. + matcher.onTick(97940, 0, at(480)); + CHECK(matcher.stats().occurrences == 1); +} + +TEST_CASE("ChainMatcher validates the chain at construction", "[chainMatcher]") { + CHECK_THROWS_AS(ChainMatcher(chainOf({})), std::invalid_argument); + // DirectionalMove without a move. + CHECK_THROWS_AS(ChainMatcher(chainOf({move(0.0, 600)})), + std::invalid_argument); + // DirectionalMove without a window. + CHECK_THROWS_AS(ChainMatcher(chainOf({move(-1.0, 0)})), + std::invalid_argument); + // NewExtreme needs an explicit +1/-1 direction. + CHECK_THROWS_AS(ChainMatcher(chainOf({Activity{ + .TYPE = ActivityType::NewExtreme, + .LOOKBACK_SECONDS = 300}})), + std::invalid_argument); + // RangeRelativeMove needs a positive multiplier. + CHECK_THROWS_AS(ChainMatcher(chainOf({Activity{ + .TYPE = ActivityType::RangeRelativeMove, + .WINDOW_SECONDS = 600, + .LOOKBACK_SECONDS = 300, + .DIRECTION = -1, + .ATR_MULTIPLE = 0.0}})), + std::invalid_argument); +} + +TEST_CASE("evaluateExperiment demuxes symbols and reports the scan span", + "[chainMatcher]") { + const auto tick = [](const std::int32_t mid, const long seconds, + const char* symbol) { + return PriceData(mid, mid, at(seconds), symbol); + }; + // EURUSD dips a full 1% (one occurrence); GBPUSD only 0.1% (none). The + // interleaved UNION-ALL order must not leak one symbol's prints into the + // other's matcher, and the final tick stretches the span to 2 days. + const std::vector ticks{ + tick(100000, 0, "EURUSD"), + tick(100000, 30, "GBPUSD"), + tick(99000, 60, "EURUSD"), + tick(99900, 90, "GBPUSD"), + tick(99000, 2 * 86400, "EURUSD"), + }; + + const auto outcome = chain_matcher::evaluateExperiment( + ticks, chainOf({move(-1.0, 600)})); + + CHECK(outcome.occurrences == 1); + CHECK(outcome.ticksScanned == 5); + CHECK(outcome.daysSpanned == 2.0); + REQUIRE(outcome.occurrencesBySymbol.size() == 2); + CHECK(outcome.occurrencesBySymbol.at("EURUSD") == 1); + CHECK(outcome.occurrencesBySymbol.at("GBPUSD") == 0); +} + +TEST_CASE("evaluateExperiment validates the chain even for an empty stream", + "[chainMatcher]") { + CHECK_THROWS_AS( + chain_matcher::evaluateExperiment({}, chainOf({move(0.0, 600)})), + std::invalid_argument); + // A VALID chain over an empty stream still reports a leg-sized failure + // vector and zero attempts — the doc's completionRate then reads null, + // never a fake 0. + const auto empty = + chain_matcher::evaluateExperiment({}, chainOf({move(-1.0, 600)})); + CHECK(empty.attempts == 0); + CHECK(empty.failuresByLeg == std::vector{0}); +} + +TEST_CASE("evaluateExperiment buckets completions by UTC month and hour", + "[chainMatcher]") { + const auto tick = [](const std::int32_t mid, const long seconds) { + return PriceData(mid, mid, at(seconds), "EURUSD"); + }; + // Occurrence 1 completes at the epoch tick + 60s: 2026-01-05, hour 0. + // Occurrence 2 completes 27 days later at 13:00 UTC: 2026-02-01, hour 13 + // — straddling the month boundary splits the buckets. + const long feb = 27 * 86400 + 13 * 3600; + const std::vector ticks{ + tick(100000, 0), + tick(99000, 60), // completes: 2026-01, hour 0 + tick(99000, feb), // fresh post-match trackers + tick(98010, feb + 60), // completes: 2026-02, hour 13 + }; + + const auto outcome = chain_matcher::evaluateExperiment( + ticks, chainOf({move(-1.0, 600)})); + + CHECK(outcome.occurrences == 2); + REQUIRE(outcome.occurrencesByMonth.size() == 2); // absent months absent + CHECK(outcome.occurrencesByMonth.at("2026-01") == 1); + CHECK(outcome.occurrencesByMonth.at("2026-02") == 1); + CHECK(outcome.occurrencesByHourUtc[0] == 1); + CHECK(outcome.occurrencesByHourUtc[13] == 1); + std::uint64_t total = 0; + for (const std::uint64_t count : outcome.occurrencesByHourUtc) { + total += count; + } + CHECK(total == outcome.occurrences); +} + +TEST_CASE("the attempt recorder tracks excursions from anchor to resolution only", + "[chainMatcher]") { + ChainMatcher matcher(chainOf({move(-1.0, 600), move(0.5, 600)})); + // COMPLETED attempt: anchor at (99000, 60); dips 200 below, completes + // 495 above at t=180 (duration 120s). The pre-anchor 100000 print must + // not register as excursion. + matcher.onTick(100000, 0, at(0)); + matcher.onTick(99000, 0, at(60)); + matcher.onTick(98800, 0, at(120)); + matcher.onTick(99495, 0, at(180)); + CHECK(matcher.stats().occurrences == 1); + REQUIRE(matcher.stats().completedExcursions.size() == 1); + CHECK(matcher.stats().completedExcursions[0].abovePoints == 495); + CHECK(matcher.stats().completedExcursions[0].belowPoints == 200); + CHECK(matcher.stats().completedExcursions[0].anchorMid == 99000); + REQUIRE(matcher.stats().completionSeconds.size() == 1); + CHECK(matcher.stats().completionSeconds[0] == 120); + + // Post-resolution tick: trackers were cleared, nothing may extend the + // recorded sample or start an attempt on its own. + matcher.onTick(97000, 0, at(240)); + CHECK(matcher.stats().attempts == 1); + + // FAILED attempt: anchor at (96030, 300); rises 70, dips 230, then the + // window expires at t=902 — the sample lands in the FAILED population. + matcher.onTick(96030, 0, at(300)); + CHECK(matcher.stats().attempts == 2); + matcher.onTick(96100, 0, at(400)); + matcher.onTick(95800, 0, at(460)); + matcher.onTick(96000, 0, at(902)); + CHECK(matcher.activeLeg() == 0); + REQUIRE(matcher.stats().failedExcursions.size() == 1); + CHECK(matcher.stats().failedExcursions[0].abovePoints == 70); + CHECK(matcher.stats().failedExcursions[0].belowPoints == 230); + // Completed population untouched by the failure. + CHECK(matcher.stats().completedExcursions.size() == 1); + CHECK(matcher.stats().completionSeconds.size() == 1); +} + +TEST_CASE("spread is sampled at the anchor tick only", "[chainMatcher]") { + ChainMatcher matcher(chainOf({move(-1.0, 600), move(0.5, 600)})); + matcher.onTick(100000, 50, at(0)); // pre-anchor spread: ignored + matcher.onTick(99000, 30, at(60)); // ANCHOR: sampled + matcher.onTick(99495, 70, at(120)); // post-anchor spread: ignored + CHECK(matcher.stats().occurrences == 1); + CHECK(matcher.stats().spreadAtTriggerCount == 1); + CHECK(matcher.stats().spreadAtTriggerSum == 30); +} + +TEST_CASE("the sample cap stops sampling while counts stay exact", + "[chainMatcher]") { + // Test-only cap of 2; three single-leg occurrences. Counts stay exact, + // samples stop at the cap, and the truncation is flagged. + ChainMatcher matcher(chainOf({move(-1.0, 600)}), /*maxAttemptSamples=*/2); + std::int32_t price = 100000; + for (int i = 0; i < 3; ++i) { + const long t = i * 200; + matcher.onTick(price, 0, at(t)); + price -= price / 100 + 1; // just over -1% off the fresh tracker + // (+1 outruns the integer-division floor) + matcher.onTick(price, 0, at(t + 60)); + } + CHECK(matcher.stats().occurrences == 3); + CHECK(matcher.stats().attempts == 3); + CHECK(matcher.stats().completedExcursions.size() == 2); + CHECK(matcher.stats().completionSeconds.size() == 2); + CHECK(matcher.stats().samplesTruncated); +} + +TEST_CASE("evaluateExperiment summarises excursions oriented by the final leg", + "[chainMatcher]") { + const auto tick = [](const std::int32_t mid, const long seconds) { + // ask = mid+10 / bid = mid-10: same mid, spread 20 on every tick. + return PriceData(mid + 10, mid - 10, at(seconds), "EURUSD"); + }; + // The completed-attempt sequence from the recorder test, through the + // full aggregation: one sample, so p50 == p90 == the sample value. + const std::vector ticks{ + tick(100000, 0), + tick(99000, 60), + tick(98800, 120), + tick(99495, 180), + }; + + const auto outcome = chain_matcher::evaluateExperiment( + ticks, chainOf({move(-1.0, 600), move(0.5, 600)})); + + // Final leg rises, so above-anchor is favorable. + CHECK(outcome.excursionOrientation == "up"); + REQUIRE(outcome.completedAttempts.has_value()); + CHECK(outcome.completedAttempts->samples == 1); + CHECK(outcome.completedAttempts->mfePoints.p50 == 495.0); + CHECK(outcome.completedAttempts->mfePoints.p90 == 495.0); + CHECK(outcome.completedAttempts->maePoints.p50 == 200.0); + // Percent variants are relative to the attempt's own anchor (99000). + CHECK(outcome.completedAttempts->mfePercent.p50 == 100.0 * 495 / 99000); + CHECK(outcome.completedAttempts->maePercent.p50 == 100.0 * 200 / 99000); + // No failed attempts: null population, not fake zeros. + CHECK_FALSE(outcome.failedAttempts.has_value()); + REQUIRE(outcome.completionSeconds.has_value()); + CHECK(outcome.completionSeconds->p50 == 120.0); + REQUIRE(outcome.meanSpreadAtTriggerPoints.has_value()); + CHECK(*outcome.meanSpreadAtTriggerPoints == 20.0); + CHECK_FALSE(outcome.samplesTruncated); +} + +TEST_CASE("a falling final leg flips the excursion orientation", "[chainMatcher]") { + // Two-leg "drop then keeps dropping": the final leg falls, so + // BELOW-anchor excursion is the favorable side. + const auto tick = [](const std::int32_t mid, const long seconds) { + return PriceData(mid, mid, at(seconds), "EURUSD"); + }; + const std::vector ticks{ + tick(100000, 0), + tick(99000, 60), // leg 1: anchor + tick(98505, 120), // leg 2: a further -0.5% completes + }; + const auto outcome = chain_matcher::evaluateExperiment( + ticks, chainOf({move(-1.0, 600), move(-0.5, 600)})); + CHECK(outcome.excursionOrientation == "down"); + REQUIRE(outcome.completedAttempts.has_value()); + CHECK(outcome.completedAttempts->mfePoints.p50 == 495.0); // below anchor + CHECK(outcome.completedAttempts->maePoints.p50 == 0.0); // never above +} + +TEST_CASE("evaluateExperiment reports per-symbol coverage alongside the v1 map", + "[chainMatcher]") { + const auto tick = [](const std::int32_t mid, const long seconds, + const char* symbol) { + return PriceData(mid, mid, at(seconds), symbol); + }; + // EURUSD: 3 ticks spanning 2 days, one occurrence. GBPUSD: 2 ticks + // spanning 1 day, none. The per-symbol spans must reflect each symbol's + // OWN coverage, not the shared stream's. + const std::vector ticks{ + tick(100000, 0, "EURUSD"), + tick(100000, 30, "GBPUSD"), + tick(99000, 60, "EURUSD"), + tick(99900, 86430, "GBPUSD"), + tick(99000, 2 * 86400, "EURUSD"), + }; + + const auto outcome = chain_matcher::evaluateExperiment( + ticks, chainOf({move(-1.0, 600)})); + + REQUIRE(outcome.perSymbol.size() == 2); + CHECK(outcome.perSymbol.at("EURUSD").occurrences == 1); + CHECK(outcome.perSymbol.at("EURUSD").ticksScanned == 3); + CHECK(outcome.perSymbol.at("EURUSD").daysSpanned == 2.0); + CHECK(outcome.perSymbol.at("GBPUSD").occurrences == 0); + CHECK(outcome.perSymbol.at("GBPUSD").ticksScanned == 2); + CHECK(outcome.perSymbol.at("GBPUSD").daysSpanned == 1.0); + // The flat v1 field stays, and stays consistent with the breakdown. + CHECK(outcome.occurrencesBySymbol.at("EURUSD") == 1); + CHECK(outcome.occurrencesBySymbol.at("GBPUSD") == 0); + // Attempts aggregate across the demuxed matchers (single-leg chain: + // attempts == occurrences). + CHECK(outcome.attempts == 1); +} diff --git a/tests/db.cpp b/tests/db.cpp index 2cc1cc0..28e17e0 100644 --- a/tests/db.cpp +++ b/tests/db.cpp @@ -6,8 +6,13 @@ #include +#include +#include +#include +#include #include +import connectionFactory; import databaseConnection; TEST_CASE("DatabaseConnection builds the QuestDB connection string", "[db]") { @@ -22,3 +27,51 @@ TEST_CASE("DatabaseConnection builds the QuestDB connection string", "[db]") { "connect_timeout=3"; CHECK(connection_string == db.getConnectionString()); } + +// Postgres wire-format text trims trailing zeros in the fractional seconds, so +// the parser must scale by the digits actually present: ".5" is 500000 µs. The +// old "%d" parse read it as 5 µs — a 100000x error that corrupted tick ordering +// at bar boundaries whenever QuestDB trimmed a timestamp. +TEST_CASE("fastParseTimestamp scales fractional seconds by digit count", "[db]") { + DateCache cache; + const auto base = std::chrono::system_clock::time_point{ + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5}} + std::chrono::hours{9}; + const auto us = [](std::int64_t n) { return std::chrono::microseconds{n}; }; + + CHECK(fastParseTimestamp("2026-01-05 09:00:00", cache) == base); + CHECK(fastParseTimestamp("2026-01-05 09:00:00.5", cache) == base + us(500000)); + CHECK(fastParseTimestamp("2026-01-05 09:00:00.50", cache) == base + us(500000)); + CHECK(fastParseTimestamp("2026-01-05 09:00:00.500000", cache) == base + us(500000)); + CHECK(fastParseTimestamp("2026-01-05 09:00:00.000123", cache) == base + us(123)); + // Sub-microsecond digits are truncated, not misread. + CHECK(fastParseTimestamp("2026-01-05 09:00:00.123456789", cache) == base + us(123456)); + + CHECK_THROWS_AS(fastParseTimestamp("garbage", cache), InvalidTimestampFormatError); + CHECK_THROWS_AS(fastParseTimestamp("2026-01-05 09:00:00.", cache), + InvalidTimestampFormatError); +} + +TEST_CASE("connectionFromEnv resolves QuestDB settings from the environment", "[db]") { + setenv("QUESTDB_HOST", "envhost", 1); + setenv("QUESTDB_PORT", "9009", 1); + + CHECK(questdb::connectionFromEnv().getConnectionString() == + "host=envhost port=9009 dbname=qdb user=admin password=quest connect_timeout=3"); + + // An explicit host (the run command passes its argv host) wins over + // $QUESTDB_HOST; the port still comes from the environment. + CHECK(questdb::connectionFromEnv("argvhost").getConnectionString() == + "host=argvhost port=9009 dbname=qdb user=admin password=quest connect_timeout=3"); + + // Unset variables fall back to the local QuestDB defaults. + unsetenv("QUESTDB_HOST"); + unsetenv("QUESTDB_PORT"); + CHECK(questdb::connectionFromEnv().getConnectionString() == + "host=127.0.0.1 port=8812 dbname=qdb user=admin password=quest connect_timeout=3"); + + // A malformed port is rejected loudly, including trailing junk that + // std::stoi would have silently truncated. + setenv("QUESTDB_PORT", "8812x", 1); + CHECK_THROWS_AS(questdb::connectionFromEnv(), std::runtime_error); + unsetenv("QUESTDB_PORT"); +} diff --git a/tests/dealPacket.cpp b/tests/dealPacket.cpp new file mode 100644 index 0000000..825e71a --- /dev/null +++ b/tests/dealPacket.cpp @@ -0,0 +1,200 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include + +#include +#include +#include +#include +#include +#include + +import dealPacket; + +namespace { + +constexpr double kNaN = std::numeric_limits::quiet_NaN(); + +// The producer-side view of one deal (vortex/shared/DealSerializer.cs): +// doubles default to NaN (= absent on the wire), strings to empty. +struct Wire { + double level = kNaN; + double size = kNaN; + double stopLevel = kNaN; + double limitLevel = kNaN; + std::string dealReference; + std::string dealId; + std::string dealIdOrigin; + std::string epic; + std::string direction; + std::string status; + std::string dealStatus; + std::string currency; + std::string channel; + std::string expiry; + std::string timestamp; + std::string guaranteedStop; +}; + +// Build the 256-byte packet exactly as DealSerializer.Serialize does: +// little-endian doubles, ASCII strings NUL-padded and capped at field +// size - 1. The offsets are the documented wire positions written as literals +// — deliberately NOT the decoder's own layout — so every test through this +// helper also pins the decoder's field positions against the contract. +std::array makePacket(const Wire& wire) { + std::array packet{}; + + const auto putDouble = [&](std::size_t offset, double value) { + const auto raw = std::bit_cast>(value); + std::ranges::copy(raw, packet.begin() + offset); + }; + const auto putString = [&](std::size_t offset, std::size_t size, const std::string& s) { + const std::size_t n = std::min(s.size(), size - 1); // always keep a NUL + for (std::size_t i = 0; i < n; ++i) { + packet[offset + i] = static_cast(s[i]); + } + }; + + putDouble(0, wire.level); + putDouble(8, wire.size); + putDouble(16, wire.stopLevel); + putDouble(24, wire.limitLevel); + putString(32, 32, wire.dealReference); + putString(64, 32, wire.dealId); + putString(96, 32, wire.dealIdOrigin); + putString(128, 32, wire.epic); + putString(160, 8, wire.direction); + putString(168, 12, wire.status); + putString(180, 12, wire.dealStatus); + putString(192, 4, wire.currency); + putString(196, 16, wire.channel); + putString(212, 8, wire.expiry); + putString(220, 24, wire.timestamp); + putString(244, 8, wire.guaranteedStop); + return packet; +} + +} // namespace + +TEST_CASE("decodeDeal parses a fully populated packet", "[dealPacket]") { + const auto packet = makePacket({ + .level = 8123.5, + .size = 1.0, + .stopLevel = 8100.0, + .limitLevel = 8150.0, + .dealReference = "9FKSX2Y5S8NT2E4", + .dealId = "DIAAAAUD3HG2JA6", + .dealIdOrigin = "DIAAAAUD3HG2JA5", + .epic = "IX.D.ASX.IFS.IP", + .direction = "BUY", + .status = "OPEN", + .dealStatus = "ACCEPTED", + .currency = "GBP", + .channel = "PublicRestOTC", + .expiry = "DFB", + .timestamp = "2026-07-11T09:15:03.123", + .guaranteedStop = "false", + }); + + const auto deal = deal_packet::decodeDeal(packet); + REQUIRE(deal.has_value()); + REQUIRE(deal->level.has_value()); + CHECK(*deal->level == 8123.5); + REQUIRE(deal->size.has_value()); + CHECK(*deal->size == 1.0); + REQUIRE(deal->stopLevel.has_value()); + CHECK(*deal->stopLevel == 8100.0); + REQUIRE(deal->limitLevel.has_value()); + CHECK(*deal->limitLevel == 8150.0); + CHECK(deal->dealReference == "9FKSX2Y5S8NT2E4"); + CHECK(deal->dealId == "DIAAAAUD3HG2JA6"); + CHECK(deal->dealIdOrigin == "DIAAAAUD3HG2JA5"); + CHECK(deal->epic == "IX.D.ASX.IFS.IP"); + CHECK(deal->direction == "BUY"); + CHECK(deal->status == "OPEN"); + CHECK(deal->dealStatus == "ACCEPTED"); + CHECK(deal->currency == "GBP"); + CHECK(deal->channel == "PublicRestOTC"); + CHECK(deal->expiry == "DFB"); + CHECK(deal->timestamp == "2026-07-11T09:15:03.123"); + CHECK(deal->guaranteedStop == "false"); +} + +TEST_CASE("decodeDeal maps absent values to their empty forms", "[dealPacket]") { + // All doubles NaN, all strings empty — the producer's shape for a deal + // where IG omitted every optional field. + const auto deal = deal_packet::decodeDeal(makePacket({.status = "DELETED"})); + REQUIRE(deal.has_value()); + CHECK_FALSE(deal->level.has_value()); + CHECK_FALSE(deal->size.has_value()); + CHECK_FALSE(deal->stopLevel.has_value()); + CHECK_FALSE(deal->limitLevel.has_value()); + CHECK(deal->dealReference.empty()); + CHECK(deal->epic.empty()); + CHECK(deal->status == "DELETED"); + CHECK(deal->guaranteedStop.empty()); +} + +TEST_CASE("decodeDeal passes field content through unjudged", "[dealPacket]") { + // Unlike ticks there is no plausibility gate: a zero or negative level is + // a value IG sent, not corruption — present it, don't drop the deal. + const auto deal = deal_packet::decodeDeal(makePacket({.level = 0.0, .size = -2.5})); + REQUIRE(deal.has_value()); + REQUIRE(deal->level.has_value()); + CHECK(*deal->level == 0.0); + REQUIRE(deal->size.has_value()); + CHECK(*deal->size == -2.5); +} + +TEST_CASE("decodeDeal rejects wrong-sized datagrams", "[dealPacket]") { + SECTION("too small") { + std::array tooSmall{}; + CHECK_FALSE(deal_packet::decodeDeal(tooSmall).has_value()); + } + SECTION("too large") { + std::array tooLarge{}; + CHECK_FALSE(deal_packet::decodeDeal(tooLarge).has_value()); + } + SECTION("a 40-byte tick packet is not a deal") { + std::array tick{}; + CHECK_FALSE(deal_packet::decodeDeal(tick).has_value()); + } +} + +TEST_CASE("decodeDeal hardening against a non-conforming producer", "[dealPacket]") { + SECTION("an unterminated field yields the full width, no overread") { + // The producer guarantees a NUL by capping content at size - 1; if a + // future producer breaks that, the decoder must still stop at the + // field edge rather than run into the neighbouring field. + auto packet = makePacket({.direction = "BUY"}); + for (std::size_t i = 64; i < 96; ++i) { // fill dealId wall to wall + packet[i] = static_cast('A'); + } + const auto deal = deal_packet::decodeDeal(packet); + REQUIRE(deal.has_value()); + CHECK(deal->dealId == std::string(32, 'A')); + CHECK(deal->dealIdOrigin.empty()); // neighbour untouched + CHECK(deal->direction == "BUY"); + } + SECTION("non-zero reserved bytes do not reject the deal") { + auto packet = makePacket({.status = "OPEN"}); + packet[252] = static_cast(0xFF); + packet[255] = static_cast(0x01); + const auto deal = deal_packet::decodeDeal(packet); + REQUIRE(deal.has_value()); + CHECK(deal->status == "OPEN"); + } +} + +TEST_CASE("makePacket mirrors the producer's size-1 truncation", "[dealPacket]") { + // DealSerializer caps content at field size - 1 so a NUL always survives; + // a 40-char epic therefore arrives as its first 31 chars. + const std::string longEpic(40, 'E'); + const auto deal = deal_packet::decodeDeal(makePacket({.epic = longEpic})); + REQUIRE(deal.has_value()); + CHECK(deal->epic == std::string(31, 'E')); +} diff --git a/tests/entryConditions.cpp b/tests/entryConditions.cpp new file mode 100644 index 0000000..75acb04 --- /dev/null +++ b/tests/entryConditions.cpp @@ -0,0 +1,156 @@ +#include + +#include +#include +#include // setenv — keep the store off QuestDB +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import barStore; +import entryConditions; +import priceData; + +namespace { + +using std::chrono::minutes; + +const std::chrono::system_clock::time_point t0 = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; + +// The gate's series as production registers it: 15m bars, ATR(10)'s 11-bar +// window (conditions::gateSeriesFor's fallback shape). +const bars::SeriesSpec kGate{minutes{15}, 11}; + +// A store fed `ticks` zero-spread ticks 16 minutes apart, each rolling a +// fresh 15m bar and stepping the ask by `stepPoints` — so once 11 bars exist, +// every true range is `stepPoints` and ATR(10) reads exactly that. +bars::BarStore warmStore(int ticks, std::int32_t stepPoints, + const std::string& symbol = "EURUSD") { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + bars::BarStore store; + store.registerSeries(kGate.minutes, kGate.count); + for (int i = 0; i < ticks; ++i) { + const std::int32_t price = 110000 + i * stepPoints; + store.update(PriceData(price, price, t0 + minutes{16 * i}, symbol)); + } + return store; +} + +// The tick under judgement: only its bid/ask (the spread) and symbol matter +// to check() — the ATR comes from the store's bars. +PriceData tickWithSpread(std::int32_t spreadPoints, + const std::string& symbol = "EURUSD") { + return PriceData(110000 + spreadPoints, 110000, t0 + std::chrono::hours{5}, + symbol); +} + +} // namespace + +TEST_CASE("conditions::check is nullopt until the gate series warms", + "[entryConditions]") { + // 10 bars < the 11 ATR(10) needs -> not warm; the 11th bar arms it. + const auto cold = warmStore(10, 200); + CHECK_FALSE(conditions::check(cold, kGate, tickWithSpread(0), 1, 3)); + + const auto warm = warmStore(11, 200); + const auto distances = conditions::check(warm, kGate, tickWithSpread(0), 1, 3); + REQUIRE(distances.has_value()); + // ATR 200 points = 20 EURUSD pips: stop = 20 x 1, limit = 20 x 3. + CHECK(distances->stopPips == 20); + CHECK(distances->limitPips == 60); +} + +TEST_CASE("conditions::check rejects a spread above 30% of ATR", + "[entryConditions]") { + // ATR 200 points -> the spread ceiling is exactly 60 points. + const auto store = warmStore(12, 200); + CHECK(conditions::check(store, kGate, tickWithSpread(60), 1, 3).has_value()); + CHECK_FALSE(conditions::check(store, kGate, tickWithSpread(61), 1, 3)); +} + +TEST_CASE("conditions::check enforces the volatility floors in points", + "[entryConditions]") { + SECTION("stop floor: ATR x mult below 10 pips skips the entry") { + // ATR 99 points x 1 = 99 < 100 (10 pips x 10 points/pip) -> skip; + // ATR 100 x 1 = 100 passes and lands exactly on the 10-pip floor. + const auto below = warmStore(12, 99); + CHECK_FALSE(conditions::check(below, kGate, tickWithSpread(0), 1, 9)); + + const auto at = warmStore(12, 100); + const auto distances = conditions::check(at, kGate, tickWithSpread(0), 1, 9); + REQUIRE(distances.has_value()); + CHECK(distances->stopPips == 10); + CHECK(distances->limitPips == 90); + } + + SECTION("limit floor: 3 pips") { + // ATR 25 points x 4 = 100 clears the stop floor, but x 1 = 25 < 30 + // (3 pips x 10) fails the limit floor. + const auto store = warmStore(12, 25); + CHECK_FALSE(conditions::check(store, kGate, tickWithSpread(0), 4, 1)); + const auto distances = conditions::check(store, kGate, tickWithSpread(0), 4, 2); + REQUIRE(distances.has_value()); + CHECK(distances->stopPips == 10); // 100 points + CHECK(distances->limitPips == 5); // 50 points + } + + SECTION("a zero multiplier never trades") { + const auto store = warmStore(12, 200); + CHECK_FALSE(conditions::check(store, kGate, tickWithSpread(0), 0, 3)); + CHECK_FALSE(conditions::check(store, kGate, tickWithSpread(0), 1, 0)); + } +} + +TEST_CASE("conditions::check clamps blown-out volatility", "[entryConditions]") { + // ATR 2000 points = 200 pips: stop 400 pips -> 80, limit 1800 -> 300. + const auto store = warmStore(12, 2000); + const auto distances = conditions::check(store, kGate, tickWithSpread(0), 2, 9); + REQUIRE(distances.has_value()); + CHECK(distances->stopPips == 80); + CHECK(distances->limitPips == 300); +} + +TEST_CASE("conditions::check refuses symbols outside symbol_scale", + "[entryConditions]") { + const auto store = warmStore(12, 200, "NOPEUSD"); + CHECK_FALSE( + conditions::check(store, kGate, tickWithSpread(0, "NOPEUSD"), 1, 3)); +} + +TEST_CASE("conditions::gateSeriesFor picks the primary timeframe or falls back", + "[entryConditions]") { + tradingDefinitions::StrategyConfig config; + + SECTION("primary OHLC timeframe, window already deep enough") { + config.OHLC_VARIABLES = { + tradingDefinitions::OHLCVariables{.OHLC_COUNT = 24, .OHLC_MINUTES = 15}, + tradingDefinitions::OHLCVariables{.OHLC_COUNT = 50, .OHLC_MINUTES = 60}, + }; + const auto series = conditions::gateSeriesFor(config); + CHECK(series.minutes == minutes{15}); + CHECK(series.count == 24); + } + + SECTION("primary window shallower than ATR(10) is widened to 11") { + config.OHLC_VARIABLES = { + tradingDefinitions::OHLCVariables{.OHLC_COUNT = 5, .OHLC_MINUTES = 30}, + }; + const auto series = conditions::gateSeriesFor(config); + CHECK(series.minutes == minutes{30}); + CHECK(series.count == 11); + } + + SECTION("no OHLC timeframes: the 15m fallback") { + const auto series = conditions::gateSeriesFor(config); + CHECK(series.minutes == minutes{15}); + CHECK(series.count == 11); + } + + SECTION("the {0,0} 'builds no bars' sentinel also falls back") { + config.OHLC_VARIABLES = {tradingDefinitions::OHLCVariables{}}; + const auto series = conditions::gateSeriesFor(config); + CHECK(series.minutes == minutes{15}); + CHECK(series.count == 11); + } +} diff --git a/tests/experimentConfig.cpp b/tests/experimentConfig.cpp new file mode 100644 index 0000000..4051c67 --- /dev/null +++ b/tests/experimentConfig.cpp @@ -0,0 +1,262 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// Pins the experiment payload wire shapes: JSON round-trips per activity +// type, the tolerant-defaults contract (TYPE strictly required, every knob +// falling back to the struct default), the unknown-TYPE poison-pill throw, +// the run descriptor's RunConfiguration-style serializer, and the Base64 +// paths the queue actually uses (JsonParser::parseExperiment*FromBase64). + +#include + +#include +#include + +#include + +#include "shared/experiments/experimentConfig.hpp" +#include "shared/experiments/experimentRunConfiguration.hpp" +#include "shared/utilities/base64.hpp" +#include "shared/utilities/jsonParser.hpp" +#include "analysis/reporting/experimentResults.hpp" + +using experiments::Activity; +using experiments::ActivityType; +using experiments::ExperimentConfig; +using experiments::ExperimentRunConfiguration; + +namespace { + +void checkActivityEqual(const Activity& restored, const Activity& original) { + CHECK(restored.TYPE == original.TYPE); + CHECK(restored.MOVE_PERCENT == original.MOVE_PERCENT); + CHECK(restored.WINDOW_SECONDS == original.WINDOW_SECONDS); + CHECK(restored.LOOKBACK_SECONDS == original.LOOKBACK_SECONDS); + CHECK(restored.DIRECTION == original.DIRECTION); + CHECK(restored.ATR_MULTIPLE == original.ATR_MULTIPLE); +} + +} // namespace + +TEST_CASE("Activity round-trips through JSON for every type", "[experimentConfig]") { + const Activity samples[] = { + {.TYPE = ActivityType::DirectionalMove, + .MOVE_PERCENT = -1.5, + .WINDOW_SECONDS = 600}, + {.TYPE = ActivityType::StaysInBand, + .MOVE_PERCENT = 0.5, + .WINDOW_SECONDS = 900, + .LOOKBACK_SECONDS = 300}, + {.TYPE = ActivityType::NewExtreme, + .WINDOW_SECONDS = 600, + .LOOKBACK_SECONDS = 1800, + .DIRECTION = -1}, + {.TYPE = ActivityType::RangeRelativeMove, + .WINDOW_SECONDS = 600, + .LOOKBACK_SECONDS = 300, + .DIRECTION = 1, + .ATR_MULTIPLE = 2.5}, + }; + for (const Activity& original : samples) { + const nlohmann::json j = original; + INFO("TYPE = " << j.at("TYPE").get()); + checkActivityEqual(j.get(), original); + } +} + +TEST_CASE("Activity parses with tolerant defaults, TYPE strictly required", + "[experimentConfig]") { + // Only the type present: every knob falls back to the struct default. + const auto sparse = + nlohmann::json{{"TYPE", "StaysInBand"}}.get(); + checkActivityEqual(sparse, Activity{.TYPE = ActivityType::StaysInBand}); + + // No TYPE at all is not an activity. + CHECK_THROWS((nlohmann::json{{"MOVE_PERCENT", 1.0}}.get())); +} + +TEST_CASE("an unknown activity TYPE throws (poison-pill path)", + "[experimentConfig]") { + CHECK_THROWS_AS((nlohmann::json{{"TYPE", "TeleportsSideways"}}.get()), + std::invalid_argument); + CHECK_THROWS_AS(experiments::activityTypeFromString("nonsense"), + std::invalid_argument); +} + +TEST_CASE("ExperimentConfig round-trips its chain through JSON", + "[experimentConfig]") { + const ExperimentConfig original{ + .UUID = "uuid-1", + .NAME = "dipRecovery", + .CHAIN = { + {.TYPE = ActivityType::DirectionalMove, + .MOVE_PERCENT = -1.0, + .WINDOW_SECONDS = 600}, + {.TYPE = ActivityType::DirectionalMove, + .MOVE_PERCENT = 0.5, + .WINDOW_SECONDS = 300}, + }, + }; + const nlohmann::json j = original; + const auto restored = j.get(); + CHECK(restored.UUID == original.UUID); + CHECK(restored.NAME == original.NAME); + REQUIRE(restored.CHAIN.size() == original.CHAIN.size()); + for (std::size_t i = 0; i < restored.CHAIN.size(); ++i) { + checkActivityEqual(restored.CHAIN[i], original.CHAIN[i]); + } + + // UUID and CHAIN are strictly required; NAME is tolerant (empty default). + CHECK_THROWS((nlohmann::json{{"NAME", "x"}}.get())); + const auto unnamed = nlohmann::json{ + {"UUID", "u"}, + {"CHAIN", nlohmann::json::array()}}.get(); + CHECK(unnamed.NAME.empty()); +} + +TEST_CASE("ExperimentRunConfiguration round-trips and tolerates legacy payloads", + "[experimentConfig]") { + const ExperimentRunConfiguration original{ + .RUN_ID = "run-1", + .SYMBOLS = "EURUSD,GBPUSD", + .BATCH = "2099-01", + .EXECUTION_TS = "2099-01-01T00:00:00Z", + .LAST_MONTHS = 9, + .OFFSET_MONTHS = 3, + }; + const nlohmann::json j = original; + const auto restored = j.get(); + CHECK(restored.RUN_ID == original.RUN_ID); + CHECK(restored.SYMBOLS == original.SYMBOLS); + CHECK(restored.BATCH == original.BATCH); + CHECK(restored.EXECUTION_TS == original.EXECUTION_TS); + CHECK(restored.LAST_MONTHS == original.LAST_MONTHS); + CHECK(restored.OFFSET_MONTHS == original.OFFSET_MONTHS); + + // The optional fields fall back to the struct defaults (RunConfiguration's + // serializer doctrine); the core three are strictly required. + const auto minimal = nlohmann::json{ + {"RUN_ID", "r"}, {"SYMBOLS", "EURUSD"}, {"LAST_MONTHS", 6}} + .get(); + CHECK(minimal.BATCH.empty()); + CHECK(minimal.EXECUTION_TS.empty()); + CHECK(minimal.OFFSET_MONTHS == 0); + CHECK_THROWS((nlohmann::json{{"RUN_ID", "r"}, {"SYMBOLS", "EURUSD"}} + .get())); +} + +TEST_CASE("JsonParser decodes experiment payloads from Base64", + "[experimentConfig]") { + const ExperimentConfig experiment{ + .UUID = "uuid-64", + .NAME = "dipRecovery", + .CHAIN = {{.TYPE = ActivityType::NewExtreme, + .WINDOW_SECONDS = 600, + .LOOKBACK_SECONDS = 1800, + .DIRECTION = 1}}, + }; + const auto decoded = JsonParser::parseExperimentFromBase64( + Base64::b64encode(nlohmann::json(experiment).dump())); + CHECK(decoded.UUID == experiment.UUID); + REQUIRE(decoded.CHAIN.size() == 1); + checkActivityEqual(decoded.CHAIN[0], experiment.CHAIN[0]); + + const ExperimentRunConfiguration runConfig{ + .RUN_ID = "run-64", + .SYMBOLS = "EURUSD", + .BATCH = "2099-01", + .EXECUTION_TS = "2099-01-01T00:00:00Z", + .LAST_MONTHS = 9, + }; + const auto decodedRun = JsonParser::parseExperimentRunFromBase64( + Base64::b64encode(nlohmann::json(runConfig).dump())); + CHECK(decodedRun.RUN_ID == runConfig.RUN_ID); + CHECK(decodedRun.SYMBOLS == runConfig.SYMBOLS); + CHECK(decodedRun.BATCH == runConfig.BATCH); + CHECK(decodedRun.LAST_MONTHS == runConfig.LAST_MONTHS); + + // Garbage base64/JSON propagates a parse error — the drain loop's + // poison-pill path relies on the throw, not a silent default. + CHECK_THROWS(JsonParser::parseExperimentFromBase64("not base64 json")); +} + +TEST_CASE("ExperimentResults serialises the phase-1 conditionality fields", + "[experimentConfig]") { + ExperimentResults results{ + .RUN_ID = "run-1", + .timestamp = "2026-07-18T00:00:00Z", + .hostname = "host", + .occurrences = 3, + .attempts = 4, + .failuresByLeg = {0, 1}, + .occurrencesByMonth = {{"2026-01", 2}, {"2026-02", 1}}, + .perSymbol = {{"EURUSD", {.occurrences = 3, + .ticksScanned = 100, + .daysSpanned = 2.5}}}, + .occurrencesBySymbol = {{"EURUSD", 3}}, + }; + results.occurrencesByHourUtc[13] = 3; + + const nlohmann::json j = results; + + CHECK(j.at("attempts") == 4); + CHECK(j.at("completionRate") == 0.75); + REQUIRE(j.at("failuresByLeg").is_array()); + CHECK(j.at("failuresByLeg")[1] == 1); + // Absent months are absent keys, not zeros. + REQUIRE(j.at("occurrencesByMonth").size() == 2); + CHECK(j.at("occurrencesByMonth").at("2026-01") == 2); + REQUIRE(j.at("occurrencesByHourUtc").is_array()); + REQUIRE(j.at("occurrencesByHourUtc").size() == 24); + CHECK(j.at("occurrencesByHourUtc")[13] == 3); + // Per-symbol breakdown nests coverage; the flat v1 field is unchanged. + CHECK(j.at("perSymbol").at("EURUSD").at("ticksScanned") == 100); + CHECK(j.at("perSymbol").at("EURUSD").at("daysSpanned") == 2.5); + CHECK(j.at("occurrencesBySymbol").at("EURUSD") == 3); +} + +TEST_CASE("completionRate is null, never a fake zero, when nothing attempted", + "[experimentConfig]") { + const ExperimentResults results{.RUN_ID = "run-1"}; + const nlohmann::json j = results; + CHECK(j.at("completionRate").is_null()); + CHECK(j.at("attempts") == 0); + // Phase-2 empty populations follow the same doctrine. + CHECK(j.at("completedAttempts").is_null()); + CHECK(j.at("failedAttempts").is_null()); + CHECK(j.at("completionSeconds").is_null()); + CHECK(j.at("meanSpreadAtTriggerPoints").is_null()); + CHECK(j.at("samplesTruncated") == false); +} + +TEST_CASE("ExperimentResults serialises the phase-2 magnitude fields", + "[experimentConfig]") { + ExperimentResults results{.RUN_ID = "run-1"}; + results.completedAttempts = experiments::ExcursionStats{ + .samples = 12, + .mfePoints = {.p50 = 495.0, .p90 = 900.0}, + .maePoints = {.p50 = 200.0, .p90 = 350.0}, + .mfePercent = {.p50 = 0.5, .p90 = 0.9}, + .maePercent = {.p50 = 0.2, .p90 = 0.35}, + }; + results.completionSeconds = experiments::QuantilePair{.p50 = 120.0, .p90 = 480.0}; + results.meanSpreadAtTriggerPoints = 20.0; + results.excursionOrientation = "up"; + results.samplesTruncated = true; + + const nlohmann::json j = results; + + CHECK(j.at("completedAttempts").at("samples") == 12); + CHECK(j.at("completedAttempts").at("mfePoints").at("p50") == 495.0); + CHECK(j.at("completedAttempts").at("mfePoints").at("p90") == 900.0); + CHECK(j.at("completedAttempts").at("maePoints").at("p90") == 350.0); + CHECK(j.at("completedAttempts").at("maePercent").at("p50") == 0.2); + CHECK(j.at("failedAttempts").is_null()); // populations stay separate + CHECK(j.at("completionSeconds").at("p50") == 120.0); + CHECK(j.at("meanSpreadAtTriggerPoints") == 20.0); + CHECK(j.at("excursionOrientation") == "up"); + CHECK(j.at("samplesTruncated") == true); +} diff --git a/tests/experimentSweep.cpp b/tests/experimentSweep.cpp new file mode 100644 index 0000000..8310368 --- /dev/null +++ b/tests/experimentSweep.cpp @@ -0,0 +1,147 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// Pins the experiment sweep MACHINERY, not the tuned grid values: the grid +// expands to the product of its range sizes, the factory's combination -> +// chain mapping (signs, minutes -> seconds), and buildExperimentChunk's +// key/payload contract — all without Redis. + +#include + +#include +#include +#include + +#include + +#include "shared/experiments/experimentConfig.hpp" +#include "shared/utilities/queueKeys.hpp" +#include "load/redisLoader.hpp" + +import experimentsCommand; // sweep::buildExperimentChunk (+ re-exported + // dipRecoverySweep: ExperimentSweepSpec, factory) +import makeDipRecovery; // sweep::makeDipRecoveryExperiment + +using experiments::ActivityType; + +TEST_CASE("buildDipRecoverySweep expands to the product of its range sizes", + "[experimentSweep]") { + const auto spec = sweep::buildDipRecoverySweep(); + const auto counts = spec.generator.rangeValueCounts(); + REQUIRE(!counts.empty()); + + // Config-agnostic: however the grid is tuned, its size must be the + // product of the declared range sizes. + std::size_t product = 1; + for (const auto& [name, count] : counts) { + INFO(name << " declares no values"); + CHECK(count > 0); + product *= count; + } + CHECK(spec.generator.combinationCount() == product); + + // Every combination carries every registered range by construction, so + // probing the two grid ends proves the four leg-prefixed names are + // registered (the factory reads them with no has() fallback). + for (const auto& combo : + {spec.generator.combinationAt(0), + spec.generator.combinationAt(spec.generator.combinationCount() - 1)}) { + for (const char* name : {"LEG1_DROP_PERCENT", "LEG1_WINDOW_MINUTES", + "LEG2_RISE_PERCENT", "LEG2_WINDOW_MINUTES"}) { + INFO("Combination is missing " << name); + CHECK(combo.has(name)); + } + } + + // The per-sweep history window (colocated with the grid by design). + CHECK(spec.lastMonths == 9); + CHECK(spec.offsetMonths == 0); + CHECK(spec.factory == sweep::makeDipRecoveryExperiment); +} + +TEST_CASE("makeDipRecoveryExperiment maps a combination onto the chain", + "[experimentSweep]") { + sweep::Combination combo; + combo.set("LEG1_DROP_PERCENT", 1.0); + combo.set("LEG1_WINDOW_MINUTES", 10); + combo.set("LEG2_RISE_PERCENT", 0.5); + combo.set("LEG2_WINDOW_MINUTES", 30); + + const auto config = sweep::makeDipRecoveryExperiment(combo); + + CHECK_FALSE(config.UUID.empty()); + CHECK(config.NAME == "dipRecovery"); + REQUIRE(config.CHAIN.size() == 2); + + // Leg 1: the drop — the grid sweeps a positive magnitude, the factory + // owns the sign; minutes become WINDOW_SECONDS. + CHECK(config.CHAIN[0].TYPE == ActivityType::DirectionalMove); + CHECK(config.CHAIN[0].MOVE_PERCENT == -1.0); + CHECK(config.CHAIN[0].WINDOW_SECONDS == 10 * 60); + + // Leg 2: the recovery — positive, its own window. + CHECK(config.CHAIN[1].TYPE == ActivityType::DirectionalMove); + CHECK(config.CHAIN[1].MOVE_PERCENT == 0.5); + CHECK(config.CHAIN[1].WINDOW_SECONDS == 30 * 60); + + // Each call mints a FRESH UUID — payload keys must never collide. + CHECK(sweep::makeDipRecoveryExperiment(combo).UUID != config.UUID); +} + +TEST_CASE("experiment queue keys embed run id and experiment uuid", + "[experimentSweep]") { + CHECK(queue_keys::experimentKey("run-1") == + std::string("BACKTESTING_QUEUE_EXPERIMENT:run-1")); + CHECK(queue_keys::experimentPayloadKey("run-1", "uuid-2") == + std::string("BACKTESTING_QUEUE_EXPERIMENT_PAYLOAD:run-1:uuid-2")); +} + +// buildExperimentChunk is the producer's streaming seam — buildStrategyChunk's +// exact contract: 1:1 with the lazily-decoded combinations in index order, +// every key derived from the run id and the payload's OWN freshly-minted +// UUID, and the JSON round-trips to an ExperimentConfig carrying that +// combination's swept values. +TEST_CASE("buildExperimentChunk keys each payload by run id and its UUID", + "[experimentSweep]") { + const auto spec = sweep::buildDipRecoverySweep(); + const std::string runId = "test-run-id"; + + const std::size_t begin = 5; + const std::size_t end = 12; + const auto chunk = sweep::buildExperimentChunk(spec.generator, spec.factory, + runId, begin, end); + REQUIRE(chunk.size() == end - begin); + + std::set uuids; + for (std::size_t offset = 0; offset < chunk.size(); ++offset) { + const auto config = nlohmann::json::parse(chunk[offset].rawJson) + .get(); + CHECK_FALSE(config.UUID.empty()); + uuids.insert(config.UUID); + CHECK(chunk[offset].key == + queue_keys::experimentPayloadKey(runId, config.UUID)); + + // The payload parses back to the factory's mapping of the SAME grid + // index (UUIDs aside — those are minted per call). + const auto expected = + spec.factory(spec.generator.combinationAt(begin + offset)); + CHECK(config.NAME == expected.NAME); + REQUIRE(config.CHAIN.size() == expected.CHAIN.size()); + for (std::size_t leg = 0; leg < config.CHAIN.size(); ++leg) { + CHECK(config.CHAIN[leg].TYPE == expected.CHAIN[leg].TYPE); + CHECK(config.CHAIN[leg].MOVE_PERCENT == + expected.CHAIN[leg].MOVE_PERCENT); + CHECK(config.CHAIN[leg].WINDOW_SECONDS == + expected.CHAIN[leg].WINDOW_SECONDS); + } + } + CHECK(uuids.size() == chunk.size()); // no key collisions inside a chunk + + // An empty range (how the stream terminates) yields an empty chunk. + CHECK(sweep::buildExperimentChunk(spec.generator, spec.factory, runId, end, + end) + .empty()); +} diff --git a/tests/fvg.cpp b/tests/fvg.cpp new file mode 100644 index 0000000..f147742 --- /dev/null +++ b/tests/fvg.cpp @@ -0,0 +1,582 @@ +#include + +#include +#include +#include +#include // setenv — keep the bar store off QuestDB +#include +#include +#include +#include +#include + +#include +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import fvgStrategy; +import barStore; // bars::BarStore — the strategy reads bars from it +import priceData; +import trade; +import tradeManager; + +namespace { + +using std::chrono::minutes; +using std::chrono::seconds; + +const std::chrono::system_clock::time_point t0 = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; + +// FVG timeframe: 1m bars, window derived at the ctor minimum LOOKBACK_BARS+3 +// (like the sweep mapper); HTF trend timeframe: HTF_SMA_PERIOD+2 bars of +// `htfMinutes`. With everything on 1 minute the store dedups both consumers +// into one shared series and the HTF trend reduces to "last closed close vs +// the one before it". +tradingDefinitions::StrategyConfig makeConfig(int minGapPips = 3, + int lookbackBars = 2, + int minGapAgeBars = 0, + int htfSmaPeriod = 2, + int htfMinutes = 1, + int maxTradeDurationMinutes = 0) { + tradingDefinitions::StrategyConfig config; + config.UUID = "test-fvg"; + config.TRADING_VARIABLES.STRATEGY = "FvgStrategy"; + config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.TRADING_SIZE = 1; + config.OHLC_VARIABLES = { + tradingDefinitions::OHLCVariables{.OHLC_COUNT = lookbackBars + 3, + .OHLC_MINUTES = 1}, // FVG timeframe + tradingDefinitions::OHLCVariables{.OHLC_COUNT = htfSmaPeriod + 2, + .OHLC_MINUTES = htfMinutes}, // HTF trend + }; + config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES = + tradingDefinitions::FVGStrategyVariables{ + .LOOKBACK_BARS = lookbackBars, + .MIN_GAP_PIPS = minGapPips, + .HTF_SMA_PERIOD = htfSmaPeriod, + .MIN_GAP_AGE_BARS = minGapAgeBars, + .MAX_TRADE_DURATION_MINUTES = maxTradeDurationMinutes}; + return config; +} + +PriceData tickAt(std::chrono::system_clock::duration offset, std::int32_t ask, + std::int32_t bid, const std::string& symbol = "EURUSD") { + return PriceData(ask, bid, t0 + offset, symbol); +} + +// The loop owner's role in miniature: register every configured timeframe +// (same-duration entries dedup into one shared series with the larger window; +// each consumer reads its own tail, like production). +bars::BarStore makeStore(const tradingDefinitions::StrategyConfig& config) { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + bars::BarStore store; + for (const auto& ohlc : config.OHLC_VARIABLES) { + store.registerSeries(minutes{ohlc.OHLC_MINUTES}, ohlc.OHLC_COUNT); + } + return store; +} + +// One tick in run-loop order: the store update first, then decide, then the +// management hook — runTicks feeds the shared bars BEFORE the entry gates, +// so decide() judges the tick against bar state that already includes it. +std::optional step(FvgStrategy& strategy, TradeManager& tm, + bars::BarStore& store, const PriceData& tick) { + store.update(tick); + const auto signal = strategy.decide(tick, store); + strategy.during(tick, store, tm); + return signal; +} + +// The exact OHLC a crafted 1m bar should end up with. +struct BarShape { + std::int32_t o, h, l, c; +}; + +// Four asks inside one 1m bar, fed open/high/low/close: the first tick sets +// the open, later ones only extend the extremes and overwrite the close, so +// the bar lands on exactly this shape. Bars sit 2 minutes apart (slot = bar +// index) so the NEXT bar's first tick rolls this one. Bars are built from the +// ask; the bid trails 10 points and is irrelevant until a SHORT signal tick. +// None of the feed ticks may signal. +void feedBar(FvgStrategy& strategy, TradeManager& tm, bars::BarStore& store, + minutes base, int slot, const BarShape& bar, + const std::string& symbol = "EURUSD") { + const auto barStart = base + minutes{2 * slot}; + const std::array, 4> ticks{{ + {seconds{0}, bar.o}, + {seconds{15}, bar.h}, + {seconds{30}, bar.l}, + {seconds{45}, bar.c}, + }}; + for (const auto& [offset, ask] : ticks) { + CHECK_FALSE(step(strategy, tm, store, + tickAt(barStart + offset, ask, ask - 10, symbol)) + .has_value()); + } +} + +void feedFixture(FvgStrategy& strategy, TradeManager& tm, bars::BarStore& store, + std::span bars, minutes base = minutes{0}, + const std::string& symbol = "EURUSD") { + for (std::size_t slot = 0; slot < bars.size(); ++slot) { + feedBar(strategy, tm, store, base, static_cast(slot), bars[slot], + symbol); + } +} + +// Canonical bullish fixture: flat warm bar, then c1/c2/c3 leaving a 40-point +// bullish gap [c1.high=110020, c3.low=110060], HTF uptrend (c3 closes above +// c2), and c3 closing above the gap. With the default config the window is +// one bar short of full while these feed, so the decision tick (slot 4, +// minutes{8}) is the first one evaluated — it rolls c3 closed and probes the +// gap with its ask. +constexpr std::array kBullishBars{{ + {110000, 110000, 110000, 110000}, + {110000, 110020, 109990, 110010}, // c1 + {110010, 110120, 110005, 110110}, // c2 — the displacement bar + {110110, 110150, 110060, 110140}, // c3 +}}; + +// Bearish mirror: 40-point gap [c3.high=110140, c1.low=110180], HTF downtrend +// (c3 closes below c2), c3 closing below the gap. SHORT probes with the bid. +constexpr std::array kBearishBars{{ + {110200, 110200, 110200, 110200}, + {110200, 110210, 110180, 110190}, // c1 + {110190, 110195, 110080, 110090}, // c2 + {110090, 110140, 110060, 110070}, // c3 +}}; + +// The bearish shape shifted to AUDUSD's price level (gap [65140, 65180]) for +// the cross-symbol isolation test. +constexpr std::array kAudBearishBars{{ + {65200, 65200, 65200, 65200}, + {65200, 65210, 65180, 65190}, // c1 + {65190, 65195, 65080, 65090}, // c2 + {65090, 65140, 65060, 65070}, // c3 +}}; + +} // namespace + +TEST_CASE("FvgStrategy signals LONG when the ask retraces into a bullish gap", + "[fvg]") { + FvgStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedFixture(strategy, tm, store, kBullishBars); + + SECTION("mid-gap") { + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110040, 110030)) == + Direction::LONG); + } + + SECTION("exactly on c3.low: LONG (upper bound inclusive)") { + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110060, 110050)) == + Direction::LONG); + } + + SECTION("exactly on c1.high: LONG (lower bound inclusive)") { + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110020, 110010)) == + Direction::LONG); + } +} + +TEST_CASE("FvgStrategy ignores price outside the gap", "[fvg]") { + FvgStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedFixture(strategy, tm, store, kBullishBars); + + SECTION("one point above c3.low: no signal") { + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{8}, 110061, 110051)) + .has_value()); + } + + SECTION("one point below c1.high: no signal") { + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{8}, 110019, 110009)) + .has_value()); + } +} + +TEST_CASE("FvgStrategy respects the minimum gap size", "[fvg]") { + // The fixture's gap is exactly 40 points = 4 pips on EURUSD (scale 10). + SECTION("gap below MIN_GAP_PIPS: no signal") { + FvgStrategy strategy{makeConfig(5)}; + TradeManager tm; + auto store = makeStore(makeConfig(5)); + feedFixture(strategy, tm, store, kBullishBars); + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{8}, 110040, 110030)) + .has_value()); + } + + SECTION("gap exactly MIN_GAP_PIPS: LONG (>= inclusive)") { + FvgStrategy strategy{makeConfig(4)}; + TradeManager tm; + auto store = makeStore(makeConfig(4)); + feedFixture(strategy, tm, store, kBullishBars); + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110040, 110030)) == + Direction::LONG); + } +} + +TEST_CASE("FvgStrategy converts MIN_GAP_PIPS per symbol scale", "[fvg]") { + // The same 40-point bullish shape on two scales: on EURUSD (10 points per + // pip) 40 points = 4 pips and meets a 4-pip floor; on DEUIDXEUR (100 + // points per pip) the identical shape is only 0.4 pips, so the gap is too + // small. Same config, same bars — only the symbol differs. + FvgStrategy strategy{makeConfig(4)}; + TradeManager tm; + auto store = makeStore(makeConfig(4)); + + SECTION("EURUSD: 40 points meets the 4-pip floor") { + feedFixture(strategy, tm, store, kBullishBars); + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110040, 110030)) == + Direction::LONG); + } + + SECTION("DEUIDXEUR: 40 points is under the 4-pip floor — no signal") { + feedFixture(strategy, tm, store, kBullishBars, minutes{0}, "DEUIDXEUR"); + CHECK_FALSE(step(strategy, tm, store, + tickAt(minutes{8}, 110040, 110030, "DEUIDXEUR")) + .has_value()); + } +} + +TEST_CASE("FvgStrategy signals SHORT when the bid retraces into a bearish gap", + "[fvg]") { + FvgStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedFixture(strategy, tm, store, kBearishBars); + + SECTION("mid-gap") { + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110170, 110160)) == + Direction::SHORT); + } + + SECTION("exactly on c3.high: SHORT (lower bound inclusive)") { + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110150, 110140)) == + Direction::SHORT); + } + + SECTION("exactly on c1.low: SHORT (upper bound inclusive)") { + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110190, 110180)) == + Direction::SHORT); + } + + SECTION("one point above c1.low: no signal") { + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{8}, 110191, 110181)) + .has_value()); + } +} + +TEST_CASE("FvgStrategy trend filter blocks a counter-trend gap", "[fvg]") { + FvgStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + + // Bullish fixture except c3 closes BELOW c2's close (110070 < 110110): + // the HTF reads downtrend while every other bullish condition still + // holds (gap 40, c3 still closes above the gap: 110070 > 110060). + feedFixture(strategy, tm, store, std::span(kBullishBars).first(3)); + feedBar(strategy, tm, store, minutes{0}, 3, {110110, 110150, 110060, 110070}); + + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{8}, 110040, 110030)) + .has_value()); +} + +TEST_CASE("FvgStrategy skips a mitigated gap", "[fvg]") { + // Six-bar window (LOOKBACK_BARS=3) with the gap one pattern old + // (MIN_GAP_AGE_BARS=1): a bar after c3 decides the gap's fate before the + // decision tick at minutes{10}. + const auto config = makeConfig(3, 3, 1); + + SECTION("a later bar traded through the gap: no signal") { + FvgStrategy strategy{config}; + TradeManager tm; + auto store = makeStore(config); + feedFixture(strategy, tm, store, kBullishBars); + // Low 110015 <= c1.high 110020 — fills the whole gap. + feedBar(strategy, tm, store, minutes{0}, 4, {110140, 110148, 110015, 110145}); + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{10}, 110040, 110030)) + .has_value()); + } + + SECTION("a later bar dipped into but not through the gap: still LONG") { + FvgStrategy strategy{config}; + TradeManager tm; + auto store = makeStore(config); + feedFixture(strategy, tm, store, kBullishBars); + // Low 110030 stays above c1.high 110020 — a partial fill leaves the + // gap live. + feedBar(strategy, tm, store, minutes{0}, 4, {110140, 110148, 110030, 110145}); + CHECK(step(strategy, tm, store, tickAt(minutes{10}, 110040, 110030)) == + Direction::LONG); + } +} + +TEST_CASE("FvgStrategy MIN_GAP_AGE_BARS widens the scan backward", "[fvg]") { + // Same six-bar layout with a clean bar after c3 (never touches the gap): + // the gap is one pattern old at the decision tick, so it is only + // reachable when the age allows scanning past the newest pattern. + SECTION("age 0: only the newest pattern is examined — no signal") { + const auto config = makeConfig(3, 3, 0); + FvgStrategy strategy{config}; + TradeManager tm; + auto store = makeStore(config); + feedFixture(strategy, tm, store, kBullishBars); + feedBar(strategy, tm, store, minutes{0}, 4, {110140, 110148, 110100, 110145}); + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{10}, 110040, 110030)) + .has_value()); + } + + SECTION("age 1: the previous pattern is reached — LONG") { + const auto config = makeConfig(3, 3, 1); + FvgStrategy strategy{config}; + TradeManager tm; + auto store = makeStore(config); + feedFixture(strategy, tm, store, kBullishBars); + feedBar(strategy, tm, store, minutes{0}, 4, {110140, 110148, 110100, 110145}); + CHECK(step(strategy, tm, store, tickAt(minutes{10}, 110040, 110030)) == + Direction::LONG); + } +} + +// The last-closed-bar gate ("close still above the gap") can only fail on the +// newest pattern when c3 closes exactly on the gap edge — pin that boundary. +// The HTF must stay in an uptrend while c3 closes low, which needs genuinely +// separated timeframes: four 60m warm bars stepping up 10 points (strictly +// below the 3-pip = 30-point gap floor, so the single-tick 1m warm bars can +// never form an accidental gap) hold the trend up, and the crafted 1m cluster +// then lives inside the in-progress 4th 60m bar, which the closed-bar SMA +// never reads. +TEST_CASE("FvgStrategy requires the last close outside the gap", "[fvg]") { + const auto config = makeConfig(3, 2, 0, 2, 60); + + auto warmUp = [](FvgStrategy& strategy, TradeManager& tm, + bars::BarStore& store) { + const std::array, 4> warmTicks{{ + {minutes{0}, 110000}, + {minutes{61}, 110010}, + {minutes{122}, 110020}, + {minutes{183}, 110030}, + }}; + for (const auto& [offset, ask] : warmTicks) { + CHECK_FALSE(step(strategy, tm, store, + tickAt(offset, ask, ask - 10)) + .has_value()); + } + }; + + SECTION("c3 closes exactly on c3.low: no signal (strictly greater)") { + FvgStrategy strategy{config}; + TradeManager tm; + auto store = makeStore(config); + warmUp(strategy, tm, store); + feedFixture(strategy, tm, store, std::span(kBullishBars).first(3), + minutes{185}); + feedBar(strategy, tm, store, minutes{185}, 3, {110110, 110150, 110060, 110060}); + CHECK_FALSE(step(strategy, tm, store, + tickAt(minutes{185} + minutes{8}, 110040, 110030)) + .has_value()); + } + + SECTION("c3 closes one point above the gap: LONG") { + FvgStrategy strategy{config}; + TradeManager tm; + auto store = makeStore(config); + warmUp(strategy, tm, store); + feedFixture(strategy, tm, store, std::span(kBullishBars).first(3), + minutes{185}); + feedBar(strategy, tm, store, minutes{185}, 3, {110110, 110150, 110060, 110070}); + CHECK(step(strategy, tm, store, + tickAt(minutes{185} + minutes{8}, 110040, 110030)) == + Direction::LONG); + } +} + +TEST_CASE("FvgStrategy keeps per-symbol state isolated", "[fvg]") { + FvgStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + + // Interleave a bullish EURUSD with a bearish AUDUSD at a very different + // price level. If either symbol's ticks leaked into the other's bars, the + // gaps and trend filters below would be wildly wrong. + for (int slot = 0; slot < 4; ++slot) { + feedBar(strategy, tm, store, minutes{0}, slot, + kBullishBars[static_cast(slot)]); + feedBar(strategy, tm, store, minutes{0}, slot, + kAudBearishBars[static_cast(slot)], "AUDUSD"); + } + + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110040, 110030)) == + Direction::LONG); + // AUDUSD's bearish gap is [65140, 65180]; the bid probes it. + CHECK(step(strategy, tm, store, + tickAt(minutes{8} + seconds{30}, 65170, 65160, "AUDUSD")) == + Direction::SHORT); +} + +// The time-cap tests drive during() directly: its exit path is independent of +// bar state (no warm-up needed), and trades are opened straight on the +// TradeManager — same harness as the NyOpenRangeBreakoutStrategy cap tests. +TEST_CASE("FvgStrategy closes trades past the max duration via during()", + "[fvg]") { + FvgStrategy strategy{makeConfig(3, 2, 0, 2, 1, 60)}; + TradeManager tm; + + SECTION("LONG past the cap closes at the bid") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::LONG); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050, 110040), + bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110040); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("SHORT past the cap closes at the ask") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::SHORT); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050, 110040), + bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110050); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("exactly at the cap stays open (strictly greater)") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::LONG); + strategy.during(tickAt(minutes{60}, 110050, 110040), bars::BarStore{}, + tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } +} + +// With the cap at 0 (the pre-cap winner-config default) exits stay owned by +// Operations via SL/TP — during() must not touch open positions. +TEST_CASE("FvgStrategy max duration of zero disables the exit", "[fvg]") { + FvgStrategy strategy{makeConfig()}; + TradeManager tm; + + tm.openTrade(tickAt(seconds{0}, 110000, 109990), 1, Direction::LONG); + strategy.during(tickAt(minutes{600}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); +} + +TEST_CASE("FvgStrategy rejects malformed configuration", "[fvg]") { + SECTION("fewer than two OHLC timeframes") { + auto config = makeConfig(); + config.OHLC_VARIABLES.resize(1); + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } + + SECTION("missing FVG_STRATEGY_VARIABLES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES = std::nullopt; + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } + + SECTION("LOOKBACK_BARS below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES->LOOKBACK_BARS = 0; + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } + + SECTION("MIN_GAP_PIPS below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES->MIN_GAP_PIPS = 0; + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } + + SECTION("HTF_SMA_PERIOD below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES->HTF_SMA_PERIOD = 0; + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } + + SECTION("negative MIN_GAP_AGE_BARS") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES->MIN_GAP_AGE_BARS = -1; + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } + + SECTION("negative MAX_TRADE_DURATION_MINUTES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES + ->MAX_TRADE_DURATION_MINUTES = -1; + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } + + SECTION("FVG window one bar short of LOOKBACK_BARS + 3") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_COUNT -= 1; + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } + + SECTION("HTF window one bar short of HTF_SMA_PERIOD + 2") { + auto config = makeConfig(); + config.OHLC_VARIABLES[1].OHLC_COUNT -= 1; + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } + + SECTION("OHLC_MINUTES below 1") { + auto config = makeConfig(); + config.OHLC_VARIABLES[1].OHLC_MINUTES = 0; + CHECK_THROWS_AS(FvgStrategy{config}, std::invalid_argument); + } +} + +TEST_CASE("StrategyVariables round-trips FVG_STRATEGY_VARIABLES through JSON", + "[fvg]") { + SECTION("present group survives the round-trip") { + tradingDefinitions::StrategyVariables vars; + vars.FVG_STRATEGY_VARIABLES = tradingDefinitions::FVGStrategyVariables{ + .LOOKBACK_BARS = 12, + .MIN_GAP_PIPS = 25, + .HTF_SMA_PERIOD = 30, + .MIN_GAP_AGE_BARS = 4, + .MAX_TRADE_DURATION_MINUTES = 45}; + + const nlohmann::json j = vars; + const auto back = j.get(); + + REQUIRE(back.FVG_STRATEGY_VARIABLES.has_value()); + CHECK(back.FVG_STRATEGY_VARIABLES->LOOKBACK_BARS == 12); + CHECK(back.FVG_STRATEGY_VARIABLES->MIN_GAP_PIPS == 25); + CHECK(back.FVG_STRATEGY_VARIABLES->HTF_SMA_PERIOD == 30); + CHECK(back.FVG_STRATEGY_VARIABLES->MIN_GAP_AGE_BARS == 4); + CHECK(back.FVG_STRATEGY_VARIABLES->MAX_TRADE_DURATION_MINUTES == 45); + } + + SECTION("absent MIN_GAP_AGE_BARS parses as newest-pattern-only") { + // Models a winner config persisted before the field existed: the + // WITH_DEFAULT codec must fall back to 0, not throw. + const auto vars = + nlohmann::json::parse( + R"({"FVG_STRATEGY_VARIABLES":{"LOOKBACK_BARS":12,)" + R"("MIN_GAP_PIPS":25,"HTF_SMA_PERIOD":30}})") + .get(); + + REQUIRE(vars.FVG_STRATEGY_VARIABLES.has_value()); + CHECK(vars.FVG_STRATEGY_VARIABLES->LOOKBACK_BARS == 12); + CHECK(vars.FVG_STRATEGY_VARIABLES->MIN_GAP_AGE_BARS == 0); + // Same doctrine for the cap: absent = 0 = disabled, the pre-cap + // behaviour those persisted winners were scored with. + CHECK(vars.FVG_STRATEGY_VARIABLES->MAX_TRADE_DURATION_MINUTES == 0); + } + + SECTION("absent group serialises as null and stays absent") { + const tradingDefinitions::StrategyVariables vars; + const nlohmann::json j = vars; + + CHECK(j.at("FVG_STRATEGY_VARIABLES").is_null()); + CHECK_FALSE(j.get() + .FVG_STRATEGY_VARIABLES.has_value()); + } +} diff --git a/tests/igRequests.cpp b/tests/igRequests.cpp new file mode 100644 index 0000000..72c2df1 --- /dev/null +++ b/tests/igRequests.cpp @@ -0,0 +1,598 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// IG wire-contract tests: the request bodies POST /positions/otc must carry +// (field names IG validates), the response parsing both engines rely on, +// the Version-header rule for the tunnelled DELETE, the shared Redis +// request-budget key formats, the gate policy matrix, and the open/close +// response mapping over an injected request seam — no Redis, no network. + +#include + +#include +#include +#include +#include +#include + +#include + +#include "shared/aws/dynamoAuth.hpp" +#include "shared/ig/igRestClient.hpp" +#include "shared/redis/apiRequestGate.hpp" +#include "shared/redis/positionManager.hpp" + +import igMarkets; +import igRequests; + +namespace { + +// A recording RequestFn: replies from a SCRIPT consumed one outcome per +// call (the last entry repeats — the confirm poll may retry), captures +// everything including the per-request options. +struct StubRequests { + std::vector script; + std::size_t calls = 0; + std::vector paths; + std::vector methods; + std::vector bodies; + std::vector headerSets; + std::vector options; + std::vector dealKeys; // options[i].dealKey, for terse checks + + static ig::RequestOutcome responded(const long status, std::string body) { + return ig::RequestOutcome{ + .fate = ig::RequestFate::Responded, + .response = ig_rest::HttpResponse{status, std::move(body)}}; + } + static ig::RequestOutcome refused(std::string reason = "duplicateDeal") { + return ig::RequestOutcome{.fate = ig::RequestFate::Refused, + .detail = std::move(reason)}; + } + static ig::RequestOutcome transportFailed() { + return ig::RequestOutcome{.fate = ig::RequestFate::TransportFailed, + .detail = "transport failed"}; + } + + // Single fixed reply — the pre-confirm test idiom. + void respond(ig::RequestOutcome outcome) { + script = {std::move(outcome)}; + } + + ig::RequestFn fn() { + return [this](const std::string& path, const std::string& method, + const std::string& jsonBody, + const ig_rest::Headers& extraHeaders, + const ig::RequestOptions& requestOptions) + -> ig::RequestOutcome { + paths.push_back(path); + methods.push_back(method); + bodies.push_back(jsonBody); + headerSets.push_back(extraHeaders); + options.push_back(requestOptions); + dealKeys.push_back(requestOptions.dealKey); + if (script.empty()) { + return transportFailed(); + } + const std::size_t index = std::min(calls, script.size() - 1); + ++calls; + return script[index]; + }; + } +}; + +ig::TradeOpenObj makeOpenObj() { + return ig::TradeOpenObj{ + .currencyCode = "USD", + .epic = "TEST.EPIC.MINI.IP", + .direction = "BUY", + .size = 1.5, + .stopDistance = 25, + .limitDistance = 50, + .dealReference = "ueur-L1719360000000", + }; +} + +const ig::OrderContext kContext{ + .strategyUuid = "u-eur", + .strategyName = "StubStrategy", + .symbol = "EURUSD", + .openDirection = "BUY", +}; + +} // namespace + +TEST_CASE("the open body carries the IG field names and values", + "[igRequests]") { + const auto body = nlohmann::json::parse(ig::encodeTradeOpen(makeOpenObj())); + CHECK(body.at("currencyCode") == "USD"); + CHECK(body.at("epic") == "TEST.EPIC.MINI.IP"); + CHECK(body.at("expiry") == "-"); + CHECK(body.at("direction") == "BUY"); + CHECK(body.at("size") == 1.5); + CHECK(body.at("forceOpen") == true); + CHECK(body.at("guaranteedStop") == false); + CHECK(body.at("orderType") == "MARKET"); + CHECK(body.at("stopDistance") == 25); + CHECK(body.at("limitDistance") == 50); + CHECK(body.at("dealReference") == "ueur-L1719360000000"); +} + +TEST_CASE("disarmed legs and an empty reference are omitted, not zeroed", + "[igRequests]") { + ig::TradeOpenObj order = makeOpenObj(); + order.stopDistance = 0; + order.limitDistance = 0; + order.dealReference.clear(); + const auto body = nlohmann::json::parse(ig::encodeTradeOpen(order)); + CHECK_FALSE(body.contains("stopDistance")); // IG rejects a literal 0 + CHECK_FALSE(body.contains("limitDistance")); + CHECK_FALSE(body.contains("dealReference")); // empty fails IG's pattern +} + +TEST_CASE("the close body is the C# TradeCloseObj shape", "[igRequests]") { + const auto body = nlohmann::json::parse(ig::encodeTradeClose( + ig::TradeCloseObj{.direction = "SELL", .dealId = "DEAL-77", + .size = 1.5})); + CHECK(body.at("orderType") == "MARKET"); + CHECK(body.at("direction") == "SELL"); + CHECK(body.at("dealId") == "DEAL-77"); + CHECK(body.at("size") == 1.5); +} + +TEST_CASE("the position response parser handles echo, error and garbage", + "[igRequests]") { + const auto ok = ig::parsePositionResponse(R"({"dealReference":"R1"})"); + REQUIRE(ok.has_value()); + CHECK(ok->dealReference == "R1"); + CHECK_FALSE(ok->errorCode.has_value()); + + const auto error = ig::parsePositionResponse( + R"({"dealReference":"","errorCode":"error.public-api.exceeded"})"); + REQUIRE(error.has_value()); + CHECK(error->dealReference.empty()); + CHECK(error->errorCode == "error.public-api.exceeded"); + + // Null errorCode parses as absent (the C# nullable), and non-objects + // are the "Failed to parse response" branch. + const auto nullError = ig::parsePositionResponse( + R"({"dealReference":"R2","errorCode":null})"); + REQUIRE(nullError.has_value()); + CHECK_FALSE(nullError->errorCode.has_value()); + CHECK_FALSE(ig::parsePositionResponse("gateway").has_value()); + CHECK_FALSE(ig::parsePositionResponse("[1,2]").has_value()); + CHECK_FALSE(ig::parsePositionResponse("").has_value()); +} + +TEST_CASE("closing direction flips the open side", "[igRequests]") { + CHECK(ig::closingDirection("BUY") == "SELL"); + CHECK(ig::closingDirection("SELL") == "BUY"); +} + +TEST_CASE("the Version header is 1 only for the tunnelled DELETE", + "[igRequests]") { + CHECK(ig_rest::versionFor({}) == "2"); + CHECK(ig_rest::versionFor({{"_method", "DELETE"}}) == "1"); + CHECK(ig_rest::versionFor({{"_method", "delete"}}) == "1"); // C# ignores case + CHECK(ig_rest::versionFor({{"_method", "PATCH"}}) == "2"); + CHECK(ig_rest::versionFor({{"X-Other", "DELETE"}}) == "2"); +} + +TEST_CASE("the request budget keys match the C# IGMarketRequests formats", + "[igRequests]") { + CHECK(redis_api::requestWindowKey(7) == "REQ#7"); + CHECK(redis_api::requestWindowKey(59) == "REQ#59"); + CHECK(redis_api::dealRequestKey("u-eurBUY") == "API#u-eurBUY"); +} + +TEST_CASE("the gate policy fails closed for opens and open for closes", + "[igRequests]") { + using ig::gatePolicy; + using ig::GateRefusal; + constexpr bool kOpen = false; // riskReducing + constexpr bool kClose = true; + + // A definite duplicate refuses BOTH classes (for a close this is pacing + // on its own close# marker — safe now that a refused close maps + // to Failed and keeps the book entry). + CHECK(gatePolicy(true, 0, kOpen) == GateRefusal::DuplicateDeal); + CHECK(gatePolicy(true, 0, kClose) == GateRefusal::DuplicateDeal); + + // Nothing suspicious: proceed. + CHECK_FALSE(gatePolicy(false, 0, kOpen).has_value()); + CHECK_FALSE(gatePolicy(false, 0, kClose).has_value()); + + // Redis uncertainty: opens fail closed, closes fail OPEN — an unsent + // close leaves live exposure, the one outcome worse than a duplicate. + CHECK(gatePolicy(std::nullopt, 0, kOpen) == GateRefusal::DuplicateUnknown); + CHECK_FALSE(gatePolicy(std::nullopt, std::nullopt, kClose).has_value()); + CHECK(gatePolicy(false, std::nullopt, kOpen) == GateRefusal::BudgetUnknown); + CHECK_FALSE(gatePolicy(false, std::nullopt, kClose).has_value()); + + // Soft budget exhausted (known): opens brake, closes bypass — IG's hard + // limiter is the backstop and a rate-rejected close retries via sync. + const int over = ig::IGMarketRequests::kMaxRequestsPerMinute + 1; + CHECK(gatePolicy(false, over, kOpen) == GateRefusal::RateLimited); + CHECK_FALSE(gatePolicy(false, over, kClose).has_value()); +} + +TEST_CASE("makeOpen confirms the deal and resolves the dealId", + "[igRequests]") { + StubRequests stub; + stub.script = { + StubRequests::responded(200, R"({"dealReference":"IGREF-1"})"), + StubRequests::responded( + 200, + R"({"dealId":"DIAAA-1","dealReference":"IGREF-1",)" + R"("dealStatus":"ACCEPTED","reason":"SUCCESS"})"), + }; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 3, std::chrono::milliseconds{0}); + + const ig::OpenResult result = open(makeOpenObj(), kContext); + CHECK(result.status == ig::OpenStatus::Accepted); + CHECK(result.dealReference == "IGREF-1"); + CHECK(result.dealId == "DIAAA-1"); + + // POST to the OTC endpoint (deduplicated per strategy+direction, the C# + // $"{strategyId}{direction}"), then the confirm GET: Version 1, no + // body, and an EMPTY deal key — the POST just recorded API#u-eurBUY for + // 30s, so reusing it would refuse this very confirm as a duplicate. + REQUIRE(stub.paths.size() == 2); + CHECK(stub.paths[0] == "/positions/otc"); + CHECK(stub.methods[0] == "POST"); + CHECK(stub.dealKeys[0] == "u-eurBUY"); + CHECK(stub.headerSets[0].empty()); + CHECK(nlohmann::json::parse(stub.bodies[0]).at("epic") + == "TEST.EPIC.MINI.IP"); + CHECK(stub.paths[1] == "/confirms/IGREF-1"); + CHECK(stub.methods[1] == "GET"); + CHECK(stub.bodies[1].empty()); + CHECK(stub.headerSets[1] + == ig_rest::Headers{{"Version", "1"}}); + CHECK(stub.dealKeys[1].empty()); + + // The open POST is NOT idempotent: zero transport retries (ambiguity + // resolves via /confirms, never a blind re-send), fail-closed gating. + // The confirm GET is idempotent and keeps the retried default. + CHECK(stub.options[0].transportRetries == 0); + CHECK_FALSE(stub.options[0].riskReducing); + CHECK(stub.options[1].transportRetries == 2); + CHECK_FALSE(stub.options[1].riskReducing); +} + +TEST_CASE("a REJECTED confirm maps to Rejected with the broker's reason", + "[igRequests]") { + StubRequests stub; + stub.script = { + StubRequests::responded(200, R"({"dealReference":"IGREF-1"})"), + StubRequests::responded( + 200, R"({"dealStatus":"REJECTED","reason":"INSUFFICIENT_FUNDS"})"), + }; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 3, std::chrono::milliseconds{0}); + + const ig::OpenResult result = open(makeOpenObj(), kContext); + CHECK(result.status == ig::OpenStatus::Rejected); + CHECK(result.reason == "INSUFFICIENT_FUNDS"); + CHECK(stub.paths.size() == 2); // definitive answer — no further polling +} + +TEST_CASE("confirm 404s exhaust the retries and keep Accepted with an empty " + "dealId", + "[igRequests]") { + StubRequests stub; + stub.script = { + StubRequests::responded(200, R"({"dealReference":"IGREF-1"})"), + StubRequests::responded(404, R"({"errorCode":"not found"})"), + }; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 3, std::chrono::milliseconds{0}); + + // The POST succeeded, so IG has the order: mapping to Rejected would + // release the lock and skip booking a possibly-live deal. Accepted + // with an empty dealId books it and fails closed at close time. + const ig::OpenResult result = open(makeOpenObj(), kContext); + CHECK(result.status == ig::OpenStatus::Accepted); + CHECK(result.dealReference == "IGREF-1"); + CHECK(result.dealId.empty()); + CHECK(stub.paths.size() == 4); // 1 POST + 3 confirm attempts +} + +TEST_CASE("an unparseable confirm keeps Accepted with an empty dealId", + "[igRequests]") { + StubRequests stub; + stub.script = { + StubRequests::responded(200, R"({"dealReference":"IGREF-1"})"), + StubRequests::responded(200, "proxy"), + }; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 2, std::chrono::milliseconds{0}); + + const ig::OpenResult result = open(makeOpenObj(), kContext); + CHECK(result.status == ig::OpenStatus::Accepted); + CHECK(result.dealId.empty()); + CHECK(stub.paths.size() == 3); // 1 POST + 2 confirm attempts +} + +TEST_CASE("a confirm poll uses IG's echoed reference, falling back to ours", + "[igRequests]") { + StubRequests stub; + stub.script = { + StubRequests::responded(200, R"({"dealReference":""})"), // no echo + StubRequests::responded( + 200, R"({"dealId":"DIAAA-2","dealStatus":"ACCEPTED"})"), + }; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 3, std::chrono::milliseconds{0}); + + const ig::OpenResult result = open(makeOpenObj(), kContext); + CHECK(result.status == ig::OpenStatus::Accepted); + CHECK(result.dealId == "DIAAA-2"); + REQUIRE(stub.paths.size() == 2); + // Our minted reference names the deal when IG echoes nothing. + CHECK(stub.paths[1] == "/confirms/ueur-L1719360000000"); +} + +TEST_CASE("a transport-failed open resolves through the confirms poll, " + "never a re-send", + "[igRequests]") { + // The lost-ACK case: the POST may or may not have reached IG. A blind + // retry could double a live position — the ONLY acceptable resolution + // is asking /confirms under the dealReference we minted into the body. + StubRequests stub; + stub.script = { + StubRequests::transportFailed(), + StubRequests::responded( + 200, R"({"dealId":"DIAAA-9","dealStatus":"ACCEPTED"})"), + }; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 3, std::chrono::milliseconds{0}); + + const ig::OpenResult result = open(makeOpenObj(), kContext); + CHECK(result.status == ig::OpenStatus::Accepted); + CHECK(result.dealReference == "ueur-L1719360000000"); + CHECK(result.dealId == "DIAAA-9"); + + // Exactly ONE POST — the order body was never re-sent. + REQUIRE(stub.paths.size() == 2); + CHECK(stub.methods[0] == "POST"); + CHECK(stub.paths[1] == "/confirms/ueur-L1719360000000"); + CHECK(stub.methods[1] == "GET"); +} + +TEST_CASE("a transport-failed open whose confirm says REJECTED maps to " + "Rejected", + "[igRequests]") { + StubRequests stub; + stub.script = { + StubRequests::transportFailed(), + StubRequests::responded( + 200, R"({"dealStatus":"REJECTED","reason":"MARKET_CLOSED"})"), + }; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 3, std::chrono::milliseconds{0}); + + const ig::OpenResult result = open(makeOpenObj(), kContext); + CHECK(result.status == ig::OpenStatus::Rejected); + CHECK(result.reason == "MARKET_CLOSED"); + CHECK(stub.paths.size() == 2); +} + +TEST_CASE("an unconfirmable transport-failed open is Failed with exactly " + "one POST", + "[igRequests]") { + StubRequests stub; + stub.script = { + StubRequests::transportFailed(), + StubRequests::responded(404, R"({"errorCode":"not found"})"), + }; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 3, std::chrono::milliseconds{0}); + + // No confirm ever appeared: the order presumably never reached IG. It + // is NOT re-sent (the channel's failure TTL brakes re-entry, and the + // producer's book sync seeds the position if it DID land). + const ig::OpenResult result = open(makeOpenObj(), kContext); + CHECK(result.status == ig::OpenStatus::Failed); + CHECK(result.reason.contains("not re-sent")); + const auto posts = std::count(stub.methods.begin(), stub.methods.end(), + std::string{"POST"}); + CHECK(posts == 1); + CHECK(stub.paths.size() == 4); // 1 POST + 3 confirm attempts +} + +TEST_CASE("a 502 on the open POST is ambiguous and resolves like a lost ACK", + "[igRequests]") { + // A gateway 5xx does not say whether IG holds the order — same + // confirm-driven resolution, same single-POST guarantee. + StubRequests stub; + stub.script = { + StubRequests::responded(502, "bad gateway"), + StubRequests::responded( + 200, R"({"dealId":"DIAAA-5","dealStatus":"ACCEPTED"})"), + }; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 3, std::chrono::milliseconds{0}); + + const ig::OpenResult result = open(makeOpenObj(), kContext); + CHECK(result.status == ig::OpenStatus::Accepted); + CHECK(result.dealId == "DIAAA-5"); + const auto posts = std::count(stub.methods.begin(), stub.methods.end(), + std::string{"POST"}); + CHECK(posts == 1); +} + +TEST_CASE("makeOpen fails on refusal, non-200, garbage and errorCode", + "[igRequests]") { + StubRequests stub; + const auto open = + ig::IGMarketCalls::makeOpen(stub.fn(), 3, std::chrono::milliseconds{0}); + + stub.respond(StubRequests::refused("budgetUnknown")); // gate refused + const auto refused = open(makeOpenObj(), kContext); + CHECK(refused.status == ig::OpenStatus::Failed); + CHECK(refused.reason.contains("budgetUnknown")); + + stub.respond(StubRequests::responded(401, R"({"errorCode":"invalid"})")); + const auto denied = open(makeOpenObj(), kContext); + CHECK(denied.status == ig::OpenStatus::Failed); + CHECK(denied.reason.contains("401")); + + stub.respond(StubRequests::responded(200, "proxy error")); + CHECK(open(makeOpenObj(), kContext).status == ig::OpenStatus::Failed); + + stub.respond(StubRequests::responded( + 200, R"({"dealReference":"","errorCode":"error.margin"})")); + const auto margin = open(makeOpenObj(), kContext); + CHECK(margin.status == ig::OpenStatus::Failed); + CHECK(margin.reason.contains("error.margin")); +} + +TEST_CASE("parseDealConfirmation handles the IG shape, partial fields and " + "garbage", + "[igRequests]") { + const auto full = ig::parseDealConfirmation( + R"({"dealId":"DIAAA-3","dealReference":"IGREF-7",)" + R"("dealStatus":"ACCEPTED","reason":"SUCCESS"})"); + REQUIRE(full.has_value()); + CHECK(full->dealId == "DIAAA-3"); + CHECK(full->dealReference == "IGREF-7"); + CHECK(full->dealStatus == "ACCEPTED"); + CHECK(full->reason == "SUCCESS"); + + // Missing fields parse as empty (empty dealStatus = still pending). + const auto partial = ig::parseDealConfirmation(R"({"dealId":"D"})"); + REQUIRE(partial.has_value()); + CHECK(partial->dealStatus.empty()); + + CHECK_FALSE(ig::parseDealConfirmation("oops").has_value()); + CHECK_FALSE(ig::parseDealConfirmation("[]").has_value()); + CHECK_FALSE(ig::parseDealConfirmation("").has_value()); +} + +TEST_CASE("an explicit Version header suppresses the default", + "[igRequests]") { + CHECK(ig_rest::hasHeader({{"Version", "1"}}, "Version")); + CHECK(ig_rest::hasHeader({{"version", "1"}}, "Version")); // case-insensitive + CHECK(ig_rest::hasHeader({{"_method", "DELETE"}, {"Version", "1"}}, + "version")); + CHECK_FALSE(ig_rest::hasHeader({}, "Version")); + CHECK_FALSE(ig_rest::hasHeader({{"_method", "DELETE"}}, "Version")); +} + +TEST_CASE("makeClose tunnels DELETE and maps outcomes", "[igRequests]") { + StubRequests stub; + const auto close = ig::IGMarketCalls::makeClose(stub.fn()); + const ig::TradeCloseObj closeObj{.direction = "SELL", .dealId = "DEAL-77", + .size = 1.5}; + + stub.respond(StubRequests::responded( + 200, R"({"dealReference":"IGREF-C"})")); + CHECK(close(closeObj, kContext).status == ig::CloseStatus::Ok); + REQUIRE(stub.headerSets.size() == 1); + REQUIRE(stub.headerSets[0].size() == 1); + CHECK(stub.headerSets[0][0] + == std::pair{"_method", "DELETE"}); + CHECK(stub.paths[0] == "/positions/otc"); + CHECK(stub.methods[0] == "POST"); + CHECK(nlohmann::json::parse(stub.bodies[0]).at("dealId") == "DEAL-77"); + + // The C# null-response branch, now reserved for a REAL lost exchange. + stub.respond(StubRequests::transportFailed()); + CHECK(close(closeObj, kContext).status == ig::CloseStatus::Gone); + + stub.respond(StubRequests::responded(500, "oops")); + CHECK(close(closeObj, kContext).status == ig::CloseStatus::Failed); + + stub.respond(StubRequests::responded(200, "not json")); + CHECK(close(closeObj, kContext).status == ig::CloseStatus::Failed); +} + +TEST_CASE("a close is keyed on its dealId, never the open's uuid+direction", + "[igRequests]") { + // The L1 regression: open and close used to SHARE API#, + // so the open's 30s marker refused the close that followed it — and the + // refusal read as "position missing from IG", pruning a live position + // from the book. Distinct namespaces make that collision impossible. + StubRequests stub; + stub.respond(StubRequests::responded( + 200, R"({"dealReference":"IGREF-C"})")); + const auto close = ig::IGMarketCalls::makeClose(stub.fn()); + + close(ig::TradeCloseObj{.direction = "SELL", .dealId = "DEAL-77", + .size = 1.5}, + kContext); + REQUIRE(stub.dealKeys.size() == 1); + CHECK(stub.dealKeys[0] == "close#DEAL-77"); + CHECK(stub.dealKeys[0] != kContext.strategyUuid + kContext.openDirection); + + // And the close is risk-reducing with transport retries kept on + // (re-sending a close of the same dealId is safe, unlike the open). + CHECK(stub.options[0].riskReducing); + CHECK(stub.options[0].transportRetries == 2); +} + +TEST_CASE("a refused close maps to Failed and never to Gone", "[igRequests]") { + // The L2 regression: a refusal means the request was NEVER SENT, so + // "the position is missing from IG" is unknowable — Gone deleted the + // very book entry the sync-driven close retry depends on. Failed keeps + // it, and the strategy re-closes on the next book sync. + StubRequests stub; + stub.respond(StubRequests::refused("duplicateDeal")); + const auto close = ig::IGMarketCalls::makeClose(stub.fn()); + + const ig::CloseResult result = + close(ig::TradeCloseObj{.direction = "SELL", .dealId = "DEAL-77", + .size = 1.5}, + kContext); + CHECK(result.status == ig::CloseStatus::Failed); + CHECK(result.reason.contains("duplicateDeal")); +} + +TEST_CASE("the deal receipt key matches the C# format", "[igRequests]") { + // (Declared in positionManager but pinned here with the rest of the IG + // wire contract.) + CHECK(redis_positions::dealReceiptKey("IGREF-9", "EURUSD") + == "DealId#IGREF-9#EURUSD"); +} + +TEST_CASE("the DynamoDB auth key matches the C# Auth# convention", + "[igRequests]") { + CHECK(aws_auth::authKey("live") == "Auth#live"); + CHECK(aws_auth::authKey("demo") == "Auth#demo"); + CHECK(aws_auth::kAuthTable == "MarketDataLive"); +} + +TEST_CASE("a DynamoDB session item maps to an IG auth only when complete", + "[igRequests]") { + const std::map full{ + {"id", "Auth#demo"}, + {"sort", "null"}, + {"url", "https://demo-api.ig.com/gateway/deal"}, + {"apikey", "key-1"}, + {"CST", "cst-1"}, + {"xSecurityToken", "xst-1"}, + {"date", "2026-07-05T00:00:00Z"}, // ignored, like authRoot + }; + const auto auth = aws_auth::authFromItem(full); + REQUIRE(auth.has_value()); + CHECK(auth->url == "https://demo-api.ig.com/gateway/deal"); + CHECK(auth->apiKey == "key-1"); + CHECK(auth->cst == "cst-1"); + CHECK(auth->xSecurityToken == "xst-1"); + + // Any missing or empty session field is unusable — must read as no auth. + for (const char* required : {"url", "apikey", "CST", "xSecurityToken"}) { + auto incomplete = full; + incomplete.erase(required); + CHECK_FALSE(aws_auth::authFromItem(incomplete).has_value()); + auto blank = full; + blank[required] = ""; + CHECK_FALSE(aws_auth::authFromItem(blank).has_value()); + } + CHECK_FALSE(aws_auth::authFromItem({}).has_value()); +} diff --git a/tests/jsonParser.cpp b/tests/jsonParser.cpp index afb77f1..fdf41d1 100644 --- a/tests/jsonParser.cpp +++ b/tests/jsonParser.cpp @@ -6,10 +6,15 @@ #include +#include #include +#include + #include "shared/utilities/jsonParser.hpp" #include "shared/utilities/base64.hpp" +#include "shared/tradingDefinitions/strategyConfig.hpp" +#include "shared/tradingDefinitions/variables/tradingVariables.hpp" TEST_CASE("JsonParser parses a valid base64 configuration", "[jsonParser]") { // A sample configuration JSON with all required fields. @@ -21,8 +26,8 @@ TEST_CASE("JsonParser parses a valid base64 configuration", "[jsonParser]") { "UUID": "", "TRADING_VARIABLES": { "STRATEGY": "RandomStrategy", - "STOP_DISTANCE_IN_PIPS": "1", - "LIMIT_DISTANCE_IN_PIPS": "1", + "STOP_DISTANCE_IN_ATR": "1", + "LIMIT_DISTANCE_IN_ATR": "1", "TRADING_SIZE": "1" }, "OHLC_VARIABLES": [ @@ -46,4 +51,87 @@ TEST_CASE("JsonParser parses a valid base64 configuration", "[jsonParser]") { JsonParser::parseConfigurationFromBase64(base64Input); INFO("Parsing should succeed with valid JSON"); CHECK_FALSE(result.RUN_ID.empty()); + // Payloads written before the slippage stress toggle existed parse with + // it off — 0.0 pip of adverse entry slippage. + CHECK(result.ENTRY_SLIPPAGE_TENTH_PIPS == 0); + // Payloads written before the weekly batch identity existed parse with + // it empty — unsuffixed legacy index routing, no batch metadata. + CHECK(result.BATCH.empty()); + CHECK(result.EXECUTION_TS.empty()); + // Likewise for payloads written before range bars existed: no + // RANGE_VARIABLES key parses as "builds no range bars", never a parse + // failure — the most important regression in the range-bar plumbing + // (a required key would silently drop every ES winner and Redis payload). + CHECK(result.STRATEGY.RANGE_VARIABLES.empty()); +} + +TEST_CASE("StrategyConfig round-trips RANGE_VARIABLES when present", + "[jsonParser]") { + nlohmann::json j = { + {"UUID", "u-range"}, + {"TRADING_VARIABLES", + {{"STRATEGY", "RandomStrategy"}, + {"STOP_DISTANCE_IN_ATR", "1"}, + {"LIMIT_DISTANCE_IN_ATR", "1"}, + {"TRADING_SIZE", "1"}}}, + {"OHLC_VARIABLES", nlohmann::json::array()}, + {"RANGE_VARIABLES", + {{{"RANGE_ATR_TICK_WINDOW", 5000}, + {"RANGE_ATR_PERCENT", 40}, + {"RANGE_COUNT", 50}}}}, + {"STRATEGY_VARIABLES", nlohmann::json::object()}, + }; + + const auto config = j.get(); + REQUIRE(config.RANGE_VARIABLES.size() == 1); + CHECK(config.RANGE_VARIABLES[0].RANGE_ATR_TICK_WINDOW == 5000); + CHECK(config.RANGE_VARIABLES[0].RANGE_ATR_PERCENT == 40); + CHECK(config.RANGE_VARIABLES[0].RANGE_COUNT == 50); + + // to_json always writes the key, so new documents round-trip exactly. + const nlohmann::json out = config; + CHECK(out.at("RANGE_VARIABLES") == j.at("RANGE_VARIABLES")); +} + +// Zero is the "unused" sentinel; negatives are poison-pill payloads that must +// fail the parse before reaching the bar builder — same doctrine as +// OHLCVariables. +TEST_CASE("RangeBarVariables rejects negative fields", "[jsonParser]") { + nlohmann::json j = {{"RANGE_ATR_TICK_WINDOW", -1}, + {"RANGE_ATR_PERCENT", 40}, + {"RANGE_COUNT", 50}}; + CHECK_THROWS_AS(j.get(), + std::invalid_argument); + + j["RANGE_ATR_TICK_WINDOW"] = 5000; + j["RANGE_COUNT"] = -50; + CHECK_THROWS_AS(j.get(), + std::invalid_argument); + + j["RANGE_COUNT"] = 50; // sane values still parse + CHECK(j.get().RANGE_ATR_TICK_WINDOW == + 5000); +} + +// readIntField reads pip distances / size through double: the old unchecked +// lround->int32 cast silently wrapped huge values into garbage (possibly +// negative) integers, and lround on "nan" is UB. Both must reject the config +// instead — a rejected payload is a contained poison pill, a wrapped size is a +// silently wrong backtest. +TEST_CASE("TradingVariables rejects out-of-range integer fields", "[jsonParser]") { + nlohmann::json j = { + {"STRATEGY", "RandomStrategy"}, + {"STOP_DISTANCE_IN_ATR", "10"}, + {"LIMIT_DISTANCE_IN_ATR", "10"}, + {"TRADING_SIZE", "99999999999"}, // wraps int32 under the old cast + }; + CHECK_THROWS_AS(j.get(), + std::invalid_argument); + + j["TRADING_SIZE"] = "nan"; // lround(NaN) is UB + CHECK_THROWS_AS(j.get(), + std::invalid_argument); + + j["TRADING_SIZE"] = "2"; // sane value still parses + CHECK(j.get().TRADING_SIZE == 2); } diff --git a/tests/keltnerFade.cpp b/tests/keltnerFade.cpp new file mode 100644 index 0000000..dc485e5 --- /dev/null +++ b/tests/keltnerFade.cpp @@ -0,0 +1,350 @@ +#include + +#include +#include +#include +#include // setenv — keep the bar store off QuestDB +#include +#include +#include +#include +#include + +#include +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import keltnerFadeStrategy; +import barStore; // bars::BarStore — the strategy reads bars from it +import priceData; +import trade; +import tradeManager; + +namespace { + +using std::chrono::minutes; +using std::chrono::seconds; + +const std::chrono::system_clock::time_point t0 = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; + +// Signal timeframe: 1m bars, window derived at the ctor minimum +// BAND_SMA_PERIOD + 2 (like the sweep mapper): period closed bars for the +// SMA, period + 1 for the ATR, plus the in-progress last bar. +tradingDefinitions::StrategyConfig makeConfig(int bandSmaPeriod = 4, + int bandAtrMultTenths = 15, + int ohlcMinutes = 1, + int maxTradeDurationMinutes = 0) { + tradingDefinitions::StrategyConfig config; + config.UUID = "test-keltner"; + config.TRADING_VARIABLES.STRATEGY = "KeltnerFadeStrategy"; + config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.TRADING_SIZE = 1; + config.OHLC_VARIABLES = { + tradingDefinitions::OHLCVariables{.OHLC_COUNT = bandSmaPeriod + 2, + .OHLC_MINUTES = ohlcMinutes}, + }; + config.STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES = + tradingDefinitions::KeltnerFadeVariables{ + .BAND_SMA_PERIOD = bandSmaPeriod, + .BAND_ATR_MULT_TENTHS = bandAtrMultTenths, + .MAX_TRADE_DURATION_MINUTES = maxTradeDurationMinutes}; + return config; +} + +PriceData tickAt(std::chrono::system_clock::duration offset, std::int32_t ask, + std::int32_t bid, const std::string& symbol = "EURUSD") { + return PriceData(ask, bid, t0 + offset, symbol); +} + +// The loop owner's role in miniature: register every configured timeframe. +bars::BarStore makeStore(const tradingDefinitions::StrategyConfig& config) { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + bars::BarStore store; + for (const auto& ohlc : config.OHLC_VARIABLES) { + store.registerSeries(minutes{ohlc.OHLC_MINUTES}, ohlc.OHLC_COUNT); + } + return store; +} + +// One tick in run-loop order: the store update first, then decide, then the +// management hook — runTicks feeds the shared bars BEFORE the entry gates, +// so decide() judges the tick against bar state that already includes it. +std::optional step(KeltnerFadeStrategy& strategy, TradeManager& tm, + bars::BarStore& store, const PriceData& tick) { + store.update(tick); + const auto signal = strategy.decide(tick, store); + strategy.during(tick, store, tm); + return signal; +} + +// The exact OHLC a crafted 1m bar should end up with. +struct BarShape { + std::int32_t o, h, l, c; +}; + +// Four asks inside one 1m bar, fed open/high/low/close: the first tick sets +// the open, later ones only extend the extremes and overwrite the close, so +// the bar lands on exactly this shape. Bars sit 2 minutes apart (slot = bar +// index) so the NEXT bar's first tick rolls this one. Bars are built from the +// ask; the bid trails 10 points and is irrelevant until a SHORT signal tick. +// None of the feed ticks may signal. +void feedBar(KeltnerFadeStrategy& strategy, TradeManager& tm, + bars::BarStore& store, minutes base, int slot, const BarShape& bar, + const std::string& symbol = "EURUSD") { + const auto barStart = base + minutes{2 * slot}; + const std::array, 4> ticks{{ + {seconds{0}, bar.o}, + {seconds{15}, bar.h}, + {seconds{30}, bar.l}, + {seconds{45}, bar.c}, + }}; + for (const auto& [offset, ask] : ticks) { + CHECK_FALSE(step(strategy, tm, store, + tickAt(barStart + offset, ask, ask - 10, symbol)) + .has_value()); + } +} + +// Six identical bars around a 110000 centre with a 20-point true range: +// SMA(4) of the closed closes = 110000, ATR(4) = 20, so with +// BAND_ATR_MULT_TENTHS = 15 the band half-width is 1.5 x 20 = 30 points and +// the band is [109970, 110030]. Every feed tick sits inside it. The decision +// ticks land at minutes{12} (slot 6), whose first tick rolls the last warm +// bar closed — the band above is exactly what they are judged against. +void feedWarmBand(KeltnerFadeStrategy& strategy, TradeManager& tm, + bars::BarStore& store) { + for (int slot = 0; slot < 6; ++slot) { + feedBar(strategy, tm, store, minutes{0}, slot, + {110000, 110010, 109990, 110000}); + } +} + +} // namespace + +TEST_CASE("KeltnerFadeStrategy fades a stretch below the lower band", "[keltnerFade]") { + KeltnerFadeStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedWarmBand(strategy, tm, store); + + SECTION("ask one point below the band: LONG") { + CHECK(step(strategy, tm, store, tickAt(minutes{12}, 109969, 109959)) == + Direction::LONG); + } + + SECTION("ask exactly on the band: no signal (strictly outside)") { + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{12}, 109970, 109960)) + .has_value()); + } +} + +TEST_CASE("KeltnerFadeStrategy fades a stretch above the upper band", "[keltnerFade]") { + KeltnerFadeStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedWarmBand(strategy, tm, store); + + SECTION("bid one point above the band: SHORT") { + CHECK(step(strategy, tm, store, tickAt(minutes{12}, 110041, 110031)) == + Direction::SHORT); + } + + SECTION("bid exactly on the band: no signal (strictly outside)") { + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{12}, 110040, 110030)) + .has_value()); + } +} + +TEST_CASE("KeltnerFadeStrategy judges the trade's own fill side", "[keltnerFade]") { + KeltnerFadeStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedWarmBand(strategy, tm, store); + + // The bid sits below the lower band but the ask (what a LONG would pay) + // does not — the spread must not flatter the stretch. + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{12}, 109975, 109965)) + .has_value()); +} + +TEST_CASE("KeltnerFadeStrategy builds the band from closed bars only", "[keltnerFade]") { + KeltnerFadeStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedWarmBand(strategy, tm, store); + + // First decision tick sets the in-progress bar's close to 109975 (inside + // the band — no signal). Had that close leaked into the SMA, the centre + // would sag to 109993.75 and the lower band to ~109964, hiding the LONG + // below. The closed-bar band keeps its edge at 109970, so the second tick + // still signals. + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{12}, 109975, 109965)) + .has_value()); + CHECK(step(strategy, tm, store, + tickAt(minutes{12} + seconds{30}, 109969, 109959)) == + Direction::LONG); +} + +TEST_CASE("KeltnerFadeStrategy treats a dead-flat market as untradeable", "[keltnerFade]") { + KeltnerFadeStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + + // Every bar is a single price: all true ranges are 0, ATR = 0, and a + // zero-width band would fade every tick of noise — decide() must refuse. + for (int slot = 0; slot < 6; ++slot) { + feedBar(strategy, tm, store, minutes{0}, slot, + {110000, 110000, 110000, 110000}); + } + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{12}, 109900, 109890)) + .has_value()); +} + +TEST_CASE("KeltnerFadeStrategy keeps per-symbol state isolated", "[keltnerFade]") { + KeltnerFadeStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedWarmBand(strategy, tm, store); + + // AUDUSD's history is only the single bar this tick opens — nowhere near + // a full window, so no signal at any price: the EURUSD band must not + // leak across symbols. + CHECK_FALSE(step(strategy, tm, store, + tickAt(minutes{12}, 109969, 109959, "AUDUSD")) + .has_value()); + // EURUSD itself still signals (interleaving did not disturb its series). + CHECK(step(strategy, tm, store, + tickAt(minutes{12} + seconds{30}, 109969, 109959)) == + Direction::LONG); +} + +// The time-cap tests drive during() directly: its exit path is independent of +// bar state (no warm-up needed), and trades are opened straight on the +// TradeManager — same harness as the SessionRangeBreakoutStrategy cap tests. +TEST_CASE("KeltnerFadeStrategy closes trades past the max duration via during()", + "[keltnerFade]") { + KeltnerFadeStrategy strategy{makeConfig(4, 15, 1, 60)}; + TradeManager tm; + + SECTION("LONG past the cap closes at the bid") { + tm.openTrade(tickAt(seconds{0}, 110000, 109990), 1, Direction::LONG); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050, 110040), bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110040); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("SHORT past the cap closes at the ask") { + tm.openTrade(tickAt(seconds{0}, 110000, 109990), 1, Direction::SHORT); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050, 110040), bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110050); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("exactly at the cap stays open (strictly greater)") { + tm.openTrade(tickAt(seconds{0}, 110000, 109990), 1, Direction::LONG); + strategy.during(tickAt(minutes{60}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } +} + +// With the cap disabled (0, the default) exits stay owned by Operations via +// SL/TP — during() must not touch open positions, otherwise the sweep's +// stop/limit parameters stop being the only exit mechanism under test. +TEST_CASE("KeltnerFadeStrategy max duration of zero disables the exit", + "[keltnerFade]") { + KeltnerFadeStrategy strategy{makeConfig()}; + TradeManager tm; + + tm.openTrade(tickAt(seconds{0}, 110000, 109990), 1, Direction::LONG); + strategy.during(tickAt(minutes{600}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); +} + +TEST_CASE("KeltnerFadeStrategy rejects malformed configuration", "[keltnerFade]") { + SECTION("no OHLC timeframe") { + auto config = makeConfig(); + config.OHLC_VARIABLES.clear(); + CHECK_THROWS_AS(KeltnerFadeStrategy{config}, std::invalid_argument); + } + + SECTION("missing KELTNER_FADE_VARIABLES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES = std::nullopt; + CHECK_THROWS_AS(KeltnerFadeStrategy{config}, std::invalid_argument); + } + + SECTION("BAND_SMA_PERIOD below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES->BAND_SMA_PERIOD = 0; + CHECK_THROWS_AS(KeltnerFadeStrategy{config}, std::invalid_argument); + } + + SECTION("BAND_ATR_MULT_TENTHS below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES->BAND_ATR_MULT_TENTHS = 0; + CHECK_THROWS_AS(KeltnerFadeStrategy{config}, std::invalid_argument); + } + + SECTION("window one bar short of BAND_SMA_PERIOD + 2") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_COUNT -= 1; + CHECK_THROWS_AS(KeltnerFadeStrategy{config}, std::invalid_argument); + } + + SECTION("OHLC_MINUTES below 1") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_MINUTES = 0; + CHECK_THROWS_AS(KeltnerFadeStrategy{config}, std::invalid_argument); + } +} + +TEST_CASE("StrategyVariables round-trips KELTNER_FADE_VARIABLES through JSON", + "[keltnerFade]") { + SECTION("present group survives the round-trip") { + tradingDefinitions::StrategyVariables vars; + vars.KELTNER_FADE_VARIABLES = tradingDefinitions::KeltnerFadeVariables{ + .BAND_SMA_PERIOD = 20, + .BAND_ATR_MULT_TENTHS = 25, + .MAX_TRADE_DURATION_MINUTES = 90}; + + const nlohmann::json j = vars; + const auto back = j.get(); + + REQUIRE(back.KELTNER_FADE_VARIABLES.has_value()); + CHECK(back.KELTNER_FADE_VARIABLES->BAND_SMA_PERIOD == 20); + CHECK(back.KELTNER_FADE_VARIABLES->BAND_ATR_MULT_TENTHS == 25); + CHECK(back.KELTNER_FADE_VARIABLES->MAX_TRADE_DURATION_MINUTES == 90); + } + + SECTION("absent MAX_TRADE_DURATION_MINUTES parses as disabled") { + // Models a winner config persisted before the field existed: the + // WITH_DEFAULT codec must fall back to 0, not throw. + const auto vars = + nlohmann::json::parse( + R"({"KELTNER_FADE_VARIABLES":{"BAND_SMA_PERIOD":20,)" + R"("BAND_ATR_MULT_TENTHS":25}})") + .get(); + + REQUIRE(vars.KELTNER_FADE_VARIABLES.has_value()); + CHECK(vars.KELTNER_FADE_VARIABLES->BAND_SMA_PERIOD == 20); + CHECK(vars.KELTNER_FADE_VARIABLES->MAX_TRADE_DURATION_MINUTES == 0); + } + + SECTION("absent group serialises as null and stays absent") { + const tradingDefinitions::StrategyVariables vars; + const nlohmann::json j = vars; + + CHECK(j.at("KELTNER_FADE_VARIABLES").is_null()); + CHECK_FALSE(j.get() + .KELTNER_FADE_VARIABLES.has_value()); + } +} diff --git a/tests/liquiditySweepReversal.cpp b/tests/liquiditySweepReversal.cpp new file mode 100644 index 0000000..343018a --- /dev/null +++ b/tests/liquiditySweepReversal.cpp @@ -0,0 +1,639 @@ +#include + +#include +#include +#include +#include // setenv — keep the bar store off QuestDB +#include +#include +#include +#include + +#include +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import liquiditySweepReversalStrategy; +import barStore; // bars::BarStore — the strategy reads bars from it +import priceData; +import trade; +import tradeManager; + +namespace { + +using std::chrono::minutes; +using std::chrono::seconds; + +const std::chrono::system_clock::time_point kBase = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5}; + +// Window derived exactly like the sweep mapper — the ctor minimum: +// max(LOOKBACK_BARS + PIVOT_BARS + 1, VALID_BARS + 11). +int deriveOhlcCount(int pivotBars, int lookbackBars, int validBars) { + return std::max(lookbackBars + pivotBars + 1, validBars + 11); +} + +tradingDefinitions::StrategyConfig makeConfig(int pivotBars = 2, + int lookbackBars = 8, + int minSweepPips = 2, + int displacementAtrTenths = 0, + int validBars = 2, + int ohlcMinutes = 15, + int maxTradeDurationMinutes = 0) { + tradingDefinitions::StrategyConfig config; + config.UUID = "test-liquidity-sweep"; + config.TRADING_VARIABLES.STRATEGY = "LiquiditySweepReversalStrategy"; + config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.TRADING_SIZE = 1; + config.OHLC_VARIABLES = { + tradingDefinitions::OHLCVariables{ + .OHLC_COUNT = deriveOhlcCount(pivotBars, lookbackBars, validBars), + .OHLC_MINUTES = ohlcMinutes}, + }; + config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES = + tradingDefinitions::LiquiditySweepReversalVariables{ + .PIVOT_BARS = pivotBars, + .LOOKBACK_BARS = lookbackBars, + .MIN_SWEEP_PIPS = minSweepPips, + .DISPLACEMENT_ATR_TENTHS = displacementAtrTenths, + .VALID_BARS = validBars, + .MAX_TRADE_DURATION_MINUTES = maxTradeDurationMinutes}; + return config; +} + +PriceData tickAt(std::chrono::system_clock::time_point base, + std::chrono::system_clock::duration offset, std::int32_t ask, + std::int32_t bid, const std::string& symbol = "EURUSD") { + return PriceData(ask, bid, base + offset, symbol); +} + +// The loop owner's role in miniature: register every configured timeframe. +bars::BarStore makeStore(const tradingDefinitions::StrategyConfig& config) { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + bars::BarStore store; + for (const auto& ohlc : config.OHLC_VARIABLES) { + store.registerSeries(minutes{ohlc.OHLC_MINUTES}, ohlc.OHLC_COUNT); + } + return store; +} + +// One tick in run-loop order: the store update first, then decide, then the +// management hook. +std::optional step(LiquiditySweepReversalStrategy& strategy, + TradeManager& tm, bars::BarStore& store, + const PriceData& tick) { + store.update(tick); + const auto signal = strategy.decide(tick, store); + strategy.during(tick, store, tm); + return signal; +} + +// The exact OHLC a crafted 15m bar should end up with. +struct BarShape { + std::int32_t o, h, l, c; +}; + +// Four asks inside one 15m bar. Unlike the session strategies there is no +// clock gate to guarantee silence while feeding, and live setups DO fire on +// feed ticks once the window is warm — so feedBar deliberately ignores +// signals; the tests assert on explicit decision ticks (and the no-lookahead +// test steps the sweep bar tick by tick itself). +void feedBar(LiquiditySweepReversalStrategy& strategy, TradeManager& tm, + bars::BarStore& store, std::chrono::system_clock::time_point base, + minutes offset, const BarShape& bar, + const std::string& symbol = "EURUSD") { + const std::array, 4> ticks{{ + {seconds{0}, bar.o}, + {seconds{15}, bar.h}, + {seconds{30}, bar.l}, + {seconds{45}, bar.c}, + }}; + for (const auto& [tickOffset, ask] : ticks) { + step(strategy, tm, store, + tickAt(base, offset + tickOffset, ask, ask - 10, symbol)); + } +} + +// Feeds bars every 20 minutes (first-tick-anchored 15m bars roll on the next +// feed), starting at `startOffset`. +template +void feedBars(LiquiditySweepReversalStrategy& strategy, TradeManager& tm, + bars::BarStore& store, std::chrono::system_clock::time_point base, + minutes startOffset, const std::array& bars) { + for (std::size_t i = 0; i < N; ++i) { + feedBar(strategy, tm, store, base, + startOffset + minutes{20} * static_cast(i), bars[i]); + } +} + +// Canonical swept-high fixture: two pad bars (keep the warm-up gate ahead of +// the interesting bars), then a gentle up-drift whose bar [6] is a strict +// 2-wing swing high at 110100. EURUSD scale 10, MIN_SWEEP_PIPS 2 -> the +// sweep must poke >= 20 points beyond 110100. +constexpr std::array kSweptHighFixture{{ + {110000, 110020, 109980, 110010}, // pad + {110000, 110020, 109980, 110010}, // pad + {110000, 110020, 109980, 110010}, // [0] + {110010, 110030, 109990, 110020}, // [1] + {110020, 110040, 110000, 110030}, // [2] + {110030, 110050, 110010, 110040}, // [3] + {110040, 110060, 110020, 110050}, // [4] + {110050, 110070, 110030, 110060}, // [5] + {110060, 110100, 110040, 110080}, // [6] swing high: level 110100 + {110080, 110090, 110050, 110070}, // [7] + {110070, 110080, 110040, 110060}, // [8] + {110060, 110070, 110030, 110050}, // [9] +}}; + +// The rejection sweep: pokes 30 points through 110100, closes back inside +// with a 40-point bearish body (open 110120 -> close 110080). +constexpr BarShape kRejectionSweep{110120, 110130, 110040, 110080}; +// A benign tail bar that closes the sweep bar without touching the level. +constexpr BarShape kBenignTail{110080, 110090, 110030, 110060}; + +// Offsets: fixture bar i sits at 20 x i minutes; the next free slot follows. +constexpr minutes kSweepOffset{240}; // right after the 12 fixture bars +constexpr minutes kTailOffset{260}; +constexpr minutes kDecisionOffset{280}; + +} // namespace + +TEST_CASE("LiquiditySweepReversalStrategy fades a swept swing high", + "[liquiditySweepReversal]") { + LiquiditySweepReversalStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedBars(strategy, tm, store, kBase, minutes{0}, kSweptHighFixture); + + SECTION("rejection deep enough: SHORT while the bid is back below") { + feedBar(strategy, tm, store, kBase, kSweepOffset, kRejectionSweep); + feedBar(strategy, tm, store, kBase, kTailOffset, kBenignTail); + CHECK(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 110090, 110080)) == + Direction::SHORT); + } + + SECTION("exactly MIN_SWEEP_PIPS deep still rejects") { + feedBar(strategy, tm, store, kBase, kSweepOffset, + {110110, 110120, 110040, 110080}); + feedBar(strategy, tm, store, kBase, kTailOffset, kBenignTail); + CHECK(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 110090, 110080)) == + Direction::SHORT); + } + + SECTION("fill-side gate: no SHORT while the bid still sits at the level") { + feedBar(strategy, tm, store, kBase, kSweepOffset, kRejectionSweep); + feedBar(strategy, tm, store, kBase, kTailOffset, kBenignTail); + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 110120, 110110)) + .has_value()); + } +} + +TEST_CASE("LiquiditySweepReversalStrategy never trades an unconfirmed rejection", + "[liquiditySweepReversal]") { + // No-lookahead: while the sweep bar is still IN PROGRESS its wick beyond + // the level is visible in the store, but the rejection isn't a closed + // fact yet — every tick of the bar must stay silent, including the ones + // where the bid is already back below the level. The tick that ROLLS the + // bar closed is the first one allowed to trade it. + LiquiditySweepReversalStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedBars(strategy, tm, store, kBase, minutes{0}, kSweptHighFixture); + + const std::array, 4> sweepTicks{{ + {seconds{0}, kRejectionSweep.o}, + {seconds{15}, kRejectionSweep.h}, + {seconds{30}, kRejectionSweep.l}, + {seconds{45}, kRejectionSweep.c}, + }}; + for (const auto& [tickOffset, ask] : sweepTicks) { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kSweepOffset + tickOffset, ask, ask - 10)) + .has_value()); + } + + // 20 minutes on, the first tick of the next bar closes the sweep bar — + // the rejection now exists and this same tick may trade it. + CHECK(step(strategy, tm, store, + tickAt(kBase, kTailOffset, 110080, 110070)) == Direction::SHORT); +} + +TEST_CASE("LiquiditySweepReversalStrategy lets the first touch consume the level", + "[liquiditySweepReversal]") { + LiquiditySweepReversalStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedBars(strategy, tm, store, kBase, minutes{0}, kSweptHighFixture); + + SECTION("a breakout close kills the level; a later poke never revives it") { + // First touch closes ABOVE the level -> breakout, level dead. + feedBar(strategy, tm, store, kBase, kSweepOffset, + {110120, 110130, 110090, 110110}); + // A textbook rejection shape follows — but it is the SECOND touch. + feedBar(strategy, tm, store, kBase, kTailOffset, + {110105, 110125, 110060, 110080}); + feedBar(strategy, tm, store, kBase, kDecisionOffset, + {110080, 110090, 110050, 110070}); + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset + minutes{20}, 110090, + 110080)) + .has_value()); + } + + SECTION("a shallow tap kills the level; a later deep sweep never revives it") { + // First touch pokes only 10 points (< MIN_SWEEP_PIPS x 10) -> tap. + feedBar(strategy, tm, store, kBase, kSweepOffset, + {110105, 110110, 110040, 110080}); + // A deep, well-formed sweep follows — but the level is already spent. + feedBar(strategy, tm, store, kBase, kTailOffset, kRejectionSweep); + feedBar(strategy, tm, store, kBase, kDecisionOffset, kBenignTail); + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset + minutes{20}, 110090, + 110080)) + .has_value()); + } +} + +TEST_CASE("LiquiditySweepReversalStrategy gates the rejection on displacement", + "[liquiditySweepReversal]") { + // ATR(10) frozen at the rejection bar is 47 points for this fixture; the + // rejection body is 40. tenths = 10 demands a full ATR -> refused; + // tenths = 5 demands half -> passes. + SECTION("body smaller than the demanded displacement: no trade") { + LiquiditySweepReversalStrategy strategy{makeConfig(2, 8, 2, 10)}; + TradeManager tm; + auto store = makeStore(makeConfig(2, 8, 2, 10)); + feedBars(strategy, tm, store, kBase, minutes{0}, kSweptHighFixture); + feedBar(strategy, tm, store, kBase, kSweepOffset, kRejectionSweep); + feedBar(strategy, tm, store, kBase, kTailOffset, kBenignTail); + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 110090, 110080)) + .has_value()); + } + + SECTION("body clearing the demanded displacement: SHORT") { + LiquiditySweepReversalStrategy strategy{makeConfig(2, 8, 2, 5)}; + TradeManager tm; + auto store = makeStore(makeConfig(2, 8, 2, 5)); + feedBars(strategy, tm, store, kBase, minutes{0}, kSweptHighFixture); + feedBar(strategy, tm, store, kBase, kSweepOffset, kRejectionSweep); + feedBar(strategy, tm, store, kBase, kTailOffset, kBenignTail); + CHECK(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 110090, 110080)) == + Direction::SHORT); + } + + SECTION("a wrong-way body is no displacement, whatever its size") { + // Same sweep depth but a BULLISH body off a swept high. + LiquiditySweepReversalStrategy strategy{makeConfig(2, 8, 2, 5)}; + TradeManager tm; + auto store = makeStore(makeConfig(2, 8, 2, 5)); + feedBars(strategy, tm, store, kBase, minutes{0}, kSweptHighFixture); + feedBar(strategy, tm, store, kBase, kSweepOffset, + {110040, 110130, 110035, 110080}); + feedBar(strategy, tm, store, kBase, kTailOffset, kBenignTail); + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 110090, 110080)) + .has_value()); + } + + SECTION("tenths of zero disables the gate, body direction included") { + LiquiditySweepReversalStrategy strategy{makeConfig(2, 8, 2, 0)}; + TradeManager tm; + auto store = makeStore(makeConfig(2, 8, 2, 0)); + feedBars(strategy, tm, store, kBase, minutes{0}, kSweptHighFixture); + feedBar(strategy, tm, store, kBase, kSweepOffset, + {110040, 110130, 110035, 110080}); + feedBar(strategy, tm, store, kBase, kTailOffset, kBenignTail); + CHECK(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 110090, 110080)) == + Direction::SHORT); + } +} + +TEST_CASE("LiquiditySweepReversalStrategy ages a rejection out after VALID_BARS", + "[liquiditySweepReversal]") { + LiquiditySweepReversalStrategy strategy{makeConfig()}; // VALID_BARS = 2 + TradeManager tm; + auto store = makeStore(makeConfig()); + feedBars(strategy, tm, store, kBase, minutes{0}, kSweptHighFixture); + feedBar(strategy, tm, store, kBase, kSweepOffset, kRejectionSweep); + feedBar(strategy, tm, store, kBase, kTailOffset, kBenignTail); + // A second benign bar pushes the rejection to three closed bars back — + // past the freshness window. + feedBar(strategy, tm, store, kBase, kDecisionOffset, + {110060, 110070, 110020, 110050}); + + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset + minutes{20}, 110090, + 110080)) + .has_value()); +} + +TEST_CASE("LiquiditySweepReversalStrategy drops a rejection once price closes back beyond", + "[liquiditySweepReversal]") { + // VALID_BARS = 3 so the extra bar cannot age the setup out — what kills + // it (or not) is that bar's CLOSE relative to the level. + LiquiditySweepReversalStrategy strategy{makeConfig(2, 8, 2, 0, 3)}; + TradeManager tm; + auto store = makeStore(makeConfig(2, 8, 2, 0, 3)); + feedBars(strategy, tm, store, kBase, minutes{0}, kSweptHighFixture); + feedBar(strategy, tm, store, kBase, kSweepOffset, kRejectionSweep); + + SECTION("a later close above the level invalidates the setup") { + feedBar(strategy, tm, store, kBase, kTailOffset, + {110090, 110120, 110080, 110110}); + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 110090, 110080)) + .has_value()); + } + + SECTION("control: the same bar closing back inside leaves it tradeable") { + feedBar(strategy, tm, store, kBase, kTailOffset, + {110090, 110120, 110080, 110090}); + CHECK(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 110090, 110080)) == + Direction::SHORT); + } +} + +TEST_CASE("LiquiditySweepReversalStrategy fades a swept swing low with a LONG", + "[liquiditySweepReversal]") { + // Exact mirror of the swept-high fixture: down-drift, swing low 109900 at + // bar [6], bullish rejection sweeping 30 points below. + LiquiditySweepReversalStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + const std::array sweptLowFixture{{ + {110000, 110020, 109980, 109990}, // pad + {110000, 110020, 109980, 109990}, // pad + {110000, 110020, 109980, 109990}, // [0] + {109990, 110010, 109970, 109980}, // [1] + {109980, 110000, 109960, 109970}, // [2] + {109970, 109990, 109950, 109960}, // [3] + {109960, 109980, 109940, 109950}, // [4] + {109950, 109970, 109930, 109940}, // [5] + {109940, 109960, 109900, 109920}, // [6] swing low: level 109900 + {109920, 109950, 109910, 109930}, // [7] + {109930, 109960, 109920, 109940}, // [8] + {109940, 109970, 109930, 109950}, // [9] + }}; + feedBars(strategy, tm, store, kBase, minutes{0}, sweptLowFixture); + feedBar(strategy, tm, store, kBase, kSweepOffset, + {109880, 109960, 109870, 109920}); + feedBar(strategy, tm, store, kBase, kTailOffset, + {109920, 109970, 109910, 109940}); + + SECTION("LONG while the ask is back above the level") { + CHECK(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 109920, 109910)) == + Direction::LONG); + } + + SECTION("fill-side gate: no LONG while the ask still sits at the level") { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kDecisionOffset, 109900, 109890)) + .has_value()); + } +} + +TEST_CASE("LiquiditySweepReversalStrategy refuses an outside bar rejecting both ways", + "[liquiditySweepReversal]") { + // One bar sweeps a swing high AND a swing low and closes inside both — + // the direction is ambiguous, so the strategy must stand aside even + // though either side alone would have been tradeable. VALID_BARS = 3 + // keeps the shared rejection fresh at the decision tick. + LiquiditySweepReversalStrategy strategy{makeConfig(2, 8, 2, 0, 3)}; + TradeManager tm; + auto store = makeStore(makeConfig(2, 8, 2, 0, 3)); + const std::array rangeFixture{{ + {110000, 110040, 109960, 110000}, // pad + {110000, 110040, 109960, 110000}, // pad + {110000, 110050, 109970, 110010}, // [0] + {110010, 110060, 109960, 110000}, // [1] + {110000, 110070, 109950, 110010}, // [2] + {110010, 110100, 109940, 110020}, // [3] swing high: level 110100 + {110020, 110080, 109930, 110000}, // [4] + {110000, 110060, 109900, 109990}, // [5] swing low: level 109900 + {109990, 110050, 109940, 110000}, // [6] + {110000, 110040, 109950, 110010}, // [7] + {110010, 110130, 109870, 110000}, // [8] outside bar: sweeps both + {110000, 110040, 109950, 110000}, // [9] + }}; + feedBars(strategy, tm, store, kBase, minutes{0}, rangeFixture); + feedBar(strategy, tm, store, kBase, kSweepOffset, + {110000, 110050, 109940, 110010}); + + CHECK_FALSE(step(strategy, tm, store, + tickAt(kBase, kTailOffset, 110010, 110000)) + .has_value()); +} + +// The time-cap tests drive during() directly: its exit path is independent of +// bar state (no warm-up needed), and trades are opened straight on the +// TradeManager — same harness as the NyOpenRangeBreakoutStrategy cap tests. +TEST_CASE("LiquiditySweepReversalStrategy closes trades past the max duration via during()", + "[liquiditySweepReversal]") { + LiquiditySweepReversalStrategy strategy{makeConfig(2, 8, 2, 0, 2, 15, 60)}; + TradeManager tm; + + SECTION("LONG past the cap closes at the bid") { + tm.openTrade(tickAt(kBase, seconds{0}, 110000, 109998), 1, + Direction::LONG); + strategy.during( + tickAt(kBase, minutes{60} + seconds{1}, 110050, 110040), + bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110040); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("SHORT past the cap closes at the ask") { + tm.openTrade(tickAt(kBase, seconds{0}, 110000, 109998), 1, + Direction::SHORT); + strategy.during( + tickAt(kBase, minutes{60} + seconds{1}, 110050, 110040), + bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110050); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("exactly at the cap stays open (strictly greater)") { + tm.openTrade(tickAt(kBase, seconds{0}, 110000, 109998), 1, + Direction::LONG); + strategy.during(tickAt(kBase, minutes{60}, 110050, 110040), + bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } +} + +// With the cap at 0 (the pre-cap winner-config default) exits stay owned by +// Operations via SL/TP — during() must not touch open positions. +TEST_CASE("LiquiditySweepReversalStrategy max duration of zero disables the exit", + "[liquiditySweepReversal]") { + LiquiditySweepReversalStrategy strategy{makeConfig()}; + TradeManager tm; + + tm.openTrade(tickAt(kBase, seconds{0}, 110000, 109990), 1, + Direction::LONG); + strategy.during(tickAt(kBase, minutes{600}, 110050, 110040), + bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); +} + +TEST_CASE("LiquiditySweepReversalStrategy rejects malformed configuration", + "[liquiditySweepReversal]") { + SECTION("no OHLC timeframe") { + auto config = makeConfig(); + config.OHLC_VARIABLES.clear(); + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("missing LIQUIDITY_SWEEP_REVERSAL_VARIABLES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES = + std::nullopt; + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("PIVOT_BARS below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->PIVOT_BARS = 0; + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("lookback too small to hold a pivot plus its wing") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->LOOKBACK_BARS = 2; // PIVOT_BARS is 2 -> needs >= 3 + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("negative MIN_SWEEP_PIPS") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->MIN_SWEEP_PIPS = -1; + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("negative DISPLACEMENT_ATR_TENTHS") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->DISPLACEMENT_ATR_TENTHS = -1; + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("VALID_BARS below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->VALID_BARS = 0; + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("VALID_BARS beyond the lookback") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->VALID_BARS = 9; // LOOKBACK_BARS is 8 + // Keep the window minimum satisfied so the ordering rule is what + // throws. + config.OHLC_VARIABLES[0].OHLC_COUNT = 40; + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("negative MAX_TRADE_DURATION_MINUTES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->MAX_TRADE_DURATION_MINUTES = -1; + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("window below the scan + warm-ATR minimum") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_COUNT -= 1; + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } + + SECTION("OHLC_MINUTES below 1") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_MINUTES = 0; + CHECK_THROWS_AS(LiquiditySweepReversalStrategy{config}, + std::invalid_argument); + } +} + +TEST_CASE("StrategyVariables round-trips LIQUIDITY_SWEEP_REVERSAL_VARIABLES through JSON", + "[liquiditySweepReversal]") { + SECTION("present group survives the round-trip") { + tradingDefinitions::StrategyVariables vars; + vars.LIQUIDITY_SWEEP_REVERSAL_VARIABLES = + tradingDefinitions::LiquiditySweepReversalVariables{ + .PIVOT_BARS = 3, + .LOOKBACK_BARS = 48, + .MIN_SWEEP_PIPS = 5, + .DISPLACEMENT_ATR_TENTHS = 10, + .VALID_BARS = 4, + .MAX_TRADE_DURATION_MINUTES = 45}; + + const nlohmann::json j = vars; + const auto back = j.get(); + + REQUIRE(back.LIQUIDITY_SWEEP_REVERSAL_VARIABLES.has_value()); + CHECK(back.LIQUIDITY_SWEEP_REVERSAL_VARIABLES->PIVOT_BARS == 3); + CHECK(back.LIQUIDITY_SWEEP_REVERSAL_VARIABLES->LOOKBACK_BARS == 48); + CHECK(back.LIQUIDITY_SWEEP_REVERSAL_VARIABLES->MIN_SWEEP_PIPS == 5); + CHECK(back.LIQUIDITY_SWEEP_REVERSAL_VARIABLES->DISPLACEMENT_ATR_TENTHS == + 10); + CHECK(back.LIQUIDITY_SWEEP_REVERSAL_VARIABLES->VALID_BARS == 4); + CHECK(back.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 45); + } + + SECTION("absent DISPLACEMENT_ATR_TENTHS parses as disabled") { + // Models a winner config persisted before the field existed: the + // WITH_DEFAULT codec must fall back to 0, not throw. + const auto vars = + nlohmann::json::parse( + R"({"LIQUIDITY_SWEEP_REVERSAL_VARIABLES":{"PIVOT_BARS":2,)" + R"("LOOKBACK_BARS":48,"MIN_SWEEP_PIPS":2,"VALID_BARS":2}})") + .get(); + + REQUIRE(vars.LIQUIDITY_SWEEP_REVERSAL_VARIABLES.has_value()); + CHECK(vars.LIQUIDITY_SWEEP_REVERSAL_VARIABLES->LOOKBACK_BARS == 48); + CHECK(vars.LIQUIDITY_SWEEP_REVERSAL_VARIABLES->DISPLACEMENT_ATR_TENTHS == + 0); + // Same doctrine for the cap: absent = 0 = disabled, the pre-cap + // behaviour those persisted winners were scored with. + CHECK(vars.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 0); + } + + SECTION("absent group serialises as null and stays absent") { + const tradingDefinitions::StrategyVariables vars; + const nlohmann::json j = vars; + + CHECK(j.at("LIQUIDITY_SWEEP_REVERSAL_VARIABLES").is_null()); + CHECK_FALSE(j.get() + .LIQUIDITY_SWEEP_REVERSAL_VARIABLES.has_value()); + } +} diff --git a/tests/liveRunner.cpp b/tests/liveRunner.cpp new file mode 100644 index 0000000..1bf7e2d --- /dev/null +++ b/tests/liveRunner.cpp @@ -0,0 +1,847 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// StrategyRunner machinery tests: symbol routing, per-worker fan-out, the +// drain-on-stop guarantee, and the gate/sink seam. The gate and sink are +// injected functors, so no Redis (and no Elasticsearch) is involved. All +// counters are read after stop() — the join gives the happens-before edge. + +#include + +#include +#include // setenv — keep the ATR gate test off QuestDB +#include +#include +#include +#include +#include + +#include "shared/tradingDefinitions/variables/tradingVariables.hpp" + +import barStore; // bars::BarStore / bars::SeriesSpec +import liveStrategyRunner; +import priceData; +import strategy; +import trade; +import tradeManager; + +namespace { + +// Deterministic IStrategy: a fixed decide() answer, call counting for both +// hooks. Owned by the runner via WorkerSpec; the test keeps a raw pointer. +class StubStrategy final : public IStrategy { +public: + explicit StubStrategy(std::optional signal) : signal_(signal) {} + + std::optional decide(const PriceData& /*tick*/, + const bars::BarStore& /*barStore*/) override { + ++decideCalls; + return signal_; + } + + void during(const PriceData& /*price*/, const bars::BarStore& /*barStore*/, + TradeManager& /*tm*/) override { + ++duringCalls; + } + + int decideCalls = 0; + int duringCalls = 0; + +private: + std::optional signal_; +}; + +// With the ATR gate disengaged (these specs set no gateSeries), the values +// flow into OrderIntent literally, so the exact-value assertions below hold. +tradingDefinitions::TradingVariables stubVars() { + tradingDefinitions::TradingVariables vars; + vars.STRATEGY = "StubStrategy"; + vars.STOP_DISTANCE_IN_ATR = 25; + vars.LIMIT_DISTANCE_IN_ATR = 50; + vars.TRADING_SIZE = 3; + return vars; +} + +// Builds a spec and hands back the stub for post-run inspection. Cap +// defaults match WorkerSpec's own (no open-trades cap, per-minute brake on). +std::pair makeSpec( + const std::string& symbol, const std::string& uuid, + std::optional signal, const int maxOpenTrades = 0, + const int maxTradesPerMinute = 60) { + auto stub = std::make_unique(signal); + StubStrategy* raw = stub.get(); + live::WorkerSpec spec{ + .symbol = symbol, + .strategyName = "StubStrategy", + .strategyUuid = uuid, + .vars = stubVars(), + .maxOpenTrades = maxOpenTrades, + .maxTradesPerMinute = maxTradesPerMinute, + .strategy = std::move(stub), + }; + return {std::move(spec), raw}; +} + +PriceData makeTick(const std::string& symbol, const std::int32_t bid = 110000, + const std::int32_t ask = 110002, + const std::chrono::seconds offset = std::chrono::seconds{0}) { + return PriceData{ask, bid, + std::chrono::system_clock::time_point{ + std::chrono::microseconds{1'719'360'000'000'000LL}} + + offset, + symbol}; +} + +const live::TradeGate kAlwaysOpen = [](const std::string&, const std::string&) { + return true; +}; +const live::OrderSink kDiscard = [](const live::OrderIntent&) {}; + +// Throws a non-std type from decide() — the worker guard must catch this too +// (an escaping non-std exception would std::terminate the whole process). +class NonStdThrowingStrategy final : public IStrategy { +public: + std::optional decide(const PriceData& /*tick*/, + const bars::BarStore& /*barStore*/) override { + ++decideCalls; + throw 42; + } + + void during(const PriceData& /*price*/, const bars::BarStore& /*barStore*/, + TradeManager& /*tm*/) override {} + + int decideCalls = 0; +}; + +// Consults the book from during() — the divergence the position feed +// exists to close — recording what it saw; optionally closes the trade, +// which is the close-signal path under test. +class BookStrategy final : public IStrategy { +public: + explicit BookStrategy(const bool closeWhenActive) + : closeWhenActive_(closeWhenActive) {} + + std::optional decide(const PriceData& /*tick*/, + const bars::BarStore& /*barStore*/) override { + return std::nullopt; + } + + void during(const PriceData& price, const bars::BarStore& /*barStore*/, + TradeManager& tradeManager) override { + if (const Trade* trade = tradeManager.findActiveTrade(price.symbol)) { + ++sawActive; + seenEntryPrice = trade->entryPrice; + seenSize = trade->size; + seenDirection = trade->direction; + if (closeWhenActive_) { + tradeManager.closeTrade(price.symbol, trade->lastMarkPrice, + price); + } + } + } + + int sawActive = 0; + std::int32_t seenEntryPrice = 0; + std::int32_t seenSize = 0; + Direction seenDirection = Direction::LONG; + +private: + bool closeWhenActive_; +}; + +std::pair makeBookSpec( + const std::string& symbol, const std::string& uuid, + const bool closeWhenActive) { + auto strategy = std::make_unique(closeWhenActive); + BookStrategy* raw = strategy.get(); + live::WorkerSpec spec{ + .symbol = symbol, + .strategyName = "BookStrategy", + .strategyUuid = uuid, + .vars = stubVars(), + .strategy = std::move(strategy), + }; + return {std::move(spec), raw}; +} + +live::BookedPosition makeBooked(const std::string& dealId = "DIAAA-1") { + return live::BookedPosition{ + .dealId = dealId, + .dealReference = "IGREF-1", + .direction = Direction::LONG, + .brokerSize = 1.5, + .engineSize = 3, + .level = 110002, + .openedAt = std::chrono::system_clock::time_point{ + std::chrono::microseconds{1'719'360'000'000'000LL}}, + }; +} + +// Scripted PositionFeed: one entry per sync (the last repeats), argument +// capture for the per-worker scoping. Worker-thread-only writes; read after +// stop() joins. +struct ScriptedFeed { + std::vector>> script; + std::vector> calls; // uuid, symbol + + live::PositionFeed fn() { + return [this](const std::string& uuid, const std::string& symbol) { + calls.emplace_back(uuid, symbol); + const std::size_t index = + std::min(calls.size() - 1, script.size() - 1); + return script[index]; + }; + } +}; + +const live::CloseSink kDiscardClose = [](const live::CloseIntent&) {}; + +} // namespace + +TEST_CASE("StrategyRunner routes ticks by symbol and counts uncached symbols", + "[liveRunner]") { + auto [eurSpec, eurStub] = makeSpec("EURUSD", "u-eur", std::nullopt); + auto [jpySpec, jpyStub] = makeSpec("USDJPY", "u-jpy", std::nullopt); + std::vector specs; + specs.push_back(std::move(eurSpec)); + specs.push_back(std::move(jpySpec)); + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard); + runner.start(); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("GBPUSD")); // no worker cached for this symbol + runner.stop(); + + CHECK(eurStub->duringCalls == 3); + CHECK(eurStub->decideCalls == 3); + CHECK(jpyStub->duringCalls == 0); + + const live::RunnerStats stats = runner.stats(); + CHECK(stats.routed == 3); + CHECK(stats.ignoredSymbol == 1); + CHECK(stats.queueDropped == 0); +} + +TEST_CASE("StrategyRunner fans one tick out to every worker on the symbol", + "[liveRunner]") { + auto [specA, stubA] = makeSpec("EURUSD", "u-a", std::nullopt); + auto [specB, stubB] = makeSpec("EURUSD", "u-b", std::nullopt); + std::vector specs; + specs.push_back(std::move(specA)); + specs.push_back(std::move(specB)); + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard); + runner.start(); + for (int i = 0; i < 5; ++i) { + runner.onTick(makeTick("EURUSD")); + } + runner.stop(); + + CHECK(stubA->duringCalls == 5); + CHECK(stubB->duringCalls == 5); + CHECK(runner.stats().routed == 10); // per-worker deliveries +} + +TEST_CASE("stop() drains queued ticks before the workers exit", "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-drain", std::nullopt); + std::vector specs; + specs.push_back(std::move(spec)); + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard); + // Enqueue before start: the whole backlog sits in the queue, so this only + // passes if stop() really drains rather than aborting mid-queue. + constexpr int kTicks = 200; + for (int i = 0; i < kTicks; ++i) { + runner.onTick(makeTick("EURUSD")); + } + runner.start(); + runner.stop(); + + CHECK(stub->duringCalls == kTicks); + CHECK(runner.stats().routed == kTicks); +} + +TEST_CASE("StrategyRunner processes ticks again after a stop()/start() cycle", + "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-restart", std::nullopt); + std::vector specs; + specs.push_back(std::move(spec)); + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard); + runner.start(); + runner.onTick(makeTick("EURUSD")); + runner.stop(); + + runner.start(); + // Give a regressed runner (stop flag never reset) time to let its fresh + // workers drain-and-exit before the tick arrives; a correct runner's + // workers are parked on the condition variable, so this costs nothing + // but the sleep. + std::this_thread::sleep_for(std::chrono::milliseconds{50}); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.stop(); + + CHECK(stub->duringCalls == 3); + CHECK(runner.stats().routed == 3); +} + +TEST_CASE("a non-std exception from a strategy does not kill the worker", + "[liveRunner]") { + auto strategy = std::make_unique(); + NonStdThrowingStrategy* raw = strategy.get(); + live::WorkerSpec spec{ + .symbol = "EURUSD", + .strategyName = "NonStdThrowingStrategy", + .strategyUuid = "u-throw", + .vars = stubVars(), + .strategy = std::move(strategy), + }; + std::vector specs; + specs.push_back(std::move(spec)); + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard); + runner.start(); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.stop(); + + // Three throws, three survivals: the worker kept draining its queue. + CHECK(raw->decideCalls == 3); + CHECK(runner.stats().routed == 3); +} + +TEST_CASE("a closed gate blocks the order and counts lockBlocked", + "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-gated", Direction::LONG); + std::vector specs; + specs.push_back(std::move(spec)); + + // Counters are only read after stop() joins the worker, so plain ints + // behind a reference are safe here. + int gateCalls = 0; + int sinkCalls = 0; + live::TradeGate gate = [&gateCalls](const std::string&, const std::string&) { + ++gateCalls; + return false; // lock held (or Redis fail-closed) + }; + live::OrderSink sink = [&sinkCalls](const live::OrderIntent&) { ++sinkCalls; }; + + live::StrategyRunner runner(std::move(specs), std::move(gate), std::move(sink)); + runner.start(); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.stop(); + + CHECK(gateCalls == 3); + CHECK(sinkCalls == 0); + const live::RunnerStats stats = runner.stats(); + CHECK(stats.signals == 3); + CHECK(stats.lockBlocked == 3); + CHECK(stats.ordersLogged == 0); + CHECK(stub->duringCalls == 3); // during still runs when the gate blocks +} + +TEST_CASE("an open gate emits a fully-populated OrderIntent", "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-order", Direction::SHORT); + std::vector specs; + specs.push_back(std::move(spec)); + + std::vector> gateArgs; + std::vector orders; + live::TradeGate gate = [&gateArgs](const std::string& uuid, + const std::string& direction) { + gateArgs.emplace_back(uuid, direction); + return true; + }; + live::OrderSink sink = [&orders](const live::OrderIntent& order) { + orders.push_back(order); + }; + + live::StrategyRunner runner(std::move(specs), std::move(gate), std::move(sink)); + runner.start(); + runner.onTick(makeTick("EURUSD", /*bid=*/110010, /*ask=*/110013)); + runner.stop(); + + REQUIRE(gateArgs.size() == 1); + CHECK(gateArgs[0].first == "u-order"); + CHECK(gateArgs[0].second == "SHORT"); + + REQUIRE(orders.size() == 1); + const live::OrderIntent& order = orders[0]; + CHECK(order.strategyName == "StubStrategy"); + CHECK(order.strategyUuid == "u-order"); + CHECK(order.symbol == "EURUSD"); + CHECK(order.direction == Direction::SHORT); + CHECK(order.size == 3); + CHECK(order.stopDistancePips == 25); + CHECK(order.limitDistancePips == 50); + CHECK(order.bid == 110010); + CHECK(order.ask == 110013); + + const live::RunnerStats stats = runner.stats(); + CHECK(stats.signals == 1); + CHECK(stats.ordersLogged == 1); + CHECK(stats.lockBlocked == 0); +} + +// Peak-hours entry gate: out-of-session ticks skip decide() entirely (counted +// as sessionSkipped) while during() still runs; an in-session tick trades as +// normal. The fixed tick epoch is Wed 2024-06-26 00:00 UTC — under BST, so +// EURUSD's Europe window is 07:00-10:00 UTC. +TEST_CASE("peakHoursOnly skips decide() outside the session window", + "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-session", Direction::LONG); + spec.peakHoursOnly = true; + std::vector specs; + specs.push_back(std::move(spec)); + + std::vector orders; + live::OrderSink sink = [&orders](const live::OrderIntent& order) { + orders.push_back(order); + }; + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, std::move(sink)); + runner.start(); + runner.onTick(makeTick("EURUSD")); // 00:00 UTC — out of session + runner.onTick(makeTick("EURUSD", 110000, 110002, + std::chrono::hours{5})); // 05:00 — out + runner.onTick(makeTick("EURUSD", 110000, 110002, + std::chrono::hours{8})); // 08:00 — in + runner.stop(); + + CHECK(stub->decideCalls == 1); // only the 08:00 tick reached decide() + CHECK(stub->duringCalls == 3); // during() is never gated + + REQUIRE(orders.size() == 1); + CHECK(orders[0].timestamp == + makeTick("EURUSD", 110000, 110002, std::chrono::hours{8}).timestamp); + + const live::RunnerStats stats = runner.stats(); + CHECK(stats.sessionSkipped == 2); + CHECK(stats.signals == 1); + CHECK(stats.ordersLogged == 1); +} + +// The gate covers entries only: on an out-of-session tick the book still +// syncs and a strategy close from during() still flows out as a CloseIntent +// (mirrors the strategy-close test below, with the filter on). +TEST_CASE("peakHoursOnly never gates the book sync or strategy closes", + "[liveRunner]") { + auto [spec, strategy] = makeBookSpec("EURUSD", "u-session-close", + /*closeWhenActive=*/true); + spec.peakHoursOnly = true; + std::vector specs; + specs.push_back(std::move(spec)); + + ScriptedFeed feed; + feed.script = {std::vector{makeBooked()}, + std::vector{}}; // gone after close + std::vector intents; + live::CloseSink closeSink = [&intents](const live::CloseIntent& intent) { + intents.push_back(intent); + }; + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, {}, + feed.fn(), std::move(closeSink), + std::chrono::seconds{0}); + runner.start(); + runner.onTick(makeTick("EURUSD")); // 00:00 UTC — out of session + runner.stop(); + + REQUIRE(intents.size() == 1); + CHECK(intents[0].dealId == "DIAAA-1"); + const live::RunnerStats stats = runner.stats(); + CHECK(stats.sessionSkipped == 1); // the entry path was gated... + CHECK(stats.bookSeeded == 1); // ...but the sync still seeded + CHECK(stats.strategyCloses == 1); // ...and the close still flowed out +} + +TEST_CASE("MAX_TRADES_PER_MINUTE caps entries in a sliding tick-time window", + "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-rate", Direction::LONG, + /*maxOpenTrades=*/0, /*maxTradesPerMinute=*/2); + std::vector specs; + specs.push_back(std::move(spec)); + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard); + runner.start(); + using std::chrono::seconds; + runner.onTick(makeTick("EURUSD", 110000, 110002, seconds{0})); // order 1 + runner.onTick(makeTick("EURUSD", 110000, 110002, seconds{1})); // order 2 + runner.onTick(makeTick("EURUSD", 110000, 110002, seconds{2})); // window full + // 60s after the first order: the half-open window has aged it out, so a + // slot is free again — the same boundary the backtest cap uses. + runner.onTick(makeTick("EURUSD", 110000, 110002, seconds{60})); // order 3 + runner.stop(); + + const live::RunnerStats stats = runner.stats(); + CHECK(stats.signals == 4); + CHECK(stats.ordersLogged == 3); + CHECK(stats.rateBlocked == 1); + CHECK(stats.lockBlocked == 0); + CHECK(stub->duringCalls == 4); // during still runs when the cap blocks +} + +TEST_CASE("maxTradesPerMinute <= 0 disables the rate cap", "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-uncapped", Direction::LONG, + /*maxOpenTrades=*/0, /*maxTradesPerMinute=*/0); + std::vector specs; + specs.push_back(std::move(spec)); + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard); + runner.start(); + for (int i = 0; i < 5; ++i) { + runner.onTick(makeTick("EURUSD")); // all in the same tick instant + } + runner.stop(); + + const live::RunnerStats stats = runner.stats(); + CHECK(stats.ordersLogged == 5); + CHECK(stats.rateBlocked == 0); + CHECK(stub->decideCalls == 5); +} + +TEST_CASE("MAX_OPEN_TRADES consults the position counter and fails closed on " + "an unknown count", + "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-pos", Direction::LONG, + /*maxOpenTrades=*/2); + std::vector specs; + specs.push_back(std::move(spec)); + + // One scripted count per signal; ticks drain in order on the single + // worker thread and the vector is only read after stop() joins it. + std::vector> counts{2, 1, std::nullopt}; + std::vector counterArgs; + live::PositionCounter counter = + [&counts, &counterArgs](const std::string& uuid) { + counterArgs.push_back(uuid); + return counts[counterArgs.size() - 1]; + }; + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, + std::move(counter)); + runner.start(); + runner.onTick(makeTick("EURUSD")); // count 2, at the cap -> blocked + runner.onTick(makeTick("EURUSD")); // count 1, below -> order + runner.onTick(makeTick("EURUSD")); // unknown (Redis down) -> blocked + runner.stop(); + + REQUIRE(counterArgs.size() == 3); + CHECK(counterArgs[0] == "u-pos"); + CHECK(stub->decideCalls == 3); + const live::RunnerStats stats = runner.stats(); + CHECK(stats.signals == 3); + CHECK(stats.positionBlocked == 2); + CHECK(stats.ordersLogged == 1); + CHECK(stats.lockBlocked == 0); +} + +TEST_CASE("maxOpenTrades <= 0 never consults the position counter", + "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-nocap", Direction::LONG, + /*maxOpenTrades=*/0); + std::vector specs; + specs.push_back(std::move(spec)); + + int counterCalls = 0; + live::PositionCounter counter = + [&counterCalls](const std::string&) -> std::optional { + ++counterCalls; + return 0; + }; + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, + std::move(counter)); + runner.start(); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.stop(); + + CHECK(counterCalls == 0); + CHECK(stub->decideCalls == 2); + CHECK(runner.stats().ordersLogged == 2); + CHECK(runner.stats().positionBlocked == 0); +} + +TEST_CASE("an open-trades cap with no counter wired fails closed", + "[liveRunner]") { + auto [spec, stub] = makeSpec("EURUSD", "u-nocounter", Direction::LONG, + /*maxOpenTrades=*/1); + std::vector specs; + specs.push_back(std::move(spec)); + + // No PositionCounter argument: the position state is unknowable, so a + // capped spec must block every entry rather than trade ungated. + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard); + runner.start(); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.stop(); + + CHECK(stub->decideCalls == 2); + const live::RunnerStats stats = runner.stats(); + CHECK(stats.positionBlocked == 2); + CHECK(stats.ordersLogged == 0); +} + +TEST_CASE("the position feed seeds the book at the booked level", + "[liveRunner]") { + auto [spec, strategy] = makeBookSpec("EURUSD", "u-book", + /*closeWhenActive=*/false); + std::vector specs; + specs.push_back(std::move(spec)); + + ScriptedFeed feed; + feed.script = {std::vector{makeBooked()}}; + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, {}, + feed.fn(), kDiscardClose, + std::chrono::seconds{0}); + runner.start(); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); // second sync must NOT re-seed + runner.stop(); + + // The strategy's during() saw a real trade, entered exactly at the + // booked level (synthetic zero-spread tick) with the engine-lots size. + CHECK(strategy->sawActive == 2); + CHECK(strategy->seenEntryPrice == 110002); + CHECK(strategy->seenSize == 3); + CHECK(strategy->seenDirection == Direction::LONG); + REQUIRE(feed.calls.size() == 2); + CHECK(feed.calls[0] + == std::pair{"u-book", "EURUSD"}); + const live::RunnerStats stats = runner.stats(); + CHECK(stats.bookSeeded == 1); + CHECK(stats.bookRemoved == 0); + CHECK(stats.strategyCloses == 0); +} + +TEST_CASE("a deal gone from the feed leaves the book without a close intent", + "[liveRunner]") { + auto [spec, strategy] = makeBookSpec("EURUSD", "u-gone", + /*closeWhenActive=*/false); + std::vector specs; + specs.push_back(std::move(spec)); + + ScriptedFeed feed; + feed.script = {std::vector{makeBooked()}, + std::vector{}}; // closed at broker + int closeIntents = 0; + live::CloseSink closeSink = [&closeIntents](const live::CloseIntent&) { + ++closeIntents; + }; + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, {}, + feed.fn(), std::move(closeSink), + std::chrono::seconds{0}); + runner.start(); + runner.onTick(makeTick("EURUSD")); // seeds + runner.onTick(makeTick("EURUSD")); // removal — broker closed it + runner.stop(); + + CHECK(strategy->sawActive == 1); // gone before the second during() + CHECK(closeIntents == 0); // broker closes never echo back out + const live::RunnerStats stats = runner.stats(); + CHECK(stats.bookSeeded == 1); + CHECK(stats.bookRemoved == 1); + CHECK(stats.strategyCloses == 0); +} + +TEST_CASE("an unknown feed keeps the book unchanged", "[liveRunner]") { + auto [spec, strategy] = makeBookSpec("EURUSD", "u-blip", + /*closeWhenActive=*/false); + std::vector specs; + specs.push_back(std::move(spec)); + + ScriptedFeed feed; + feed.script = {std::vector{makeBooked()}, + std::nullopt}; // Redis blip: state UNKNOWN + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, {}, + feed.fn(), kDiscardClose, + std::chrono::seconds{0}); + runner.start(); + runner.onTick(makeTick("EURUSD")); + runner.onTick(makeTick("EURUSD")); + runner.stop(); + + // Fail-open for book state: the trade survived the blip. + CHECK(strategy->sawActive == 2); + const live::RunnerStats stats = runner.stats(); + CHECK(stats.bookSeeded == 1); + CHECK(stats.bookRemoved == 0); + CHECK(stats.bookSyncFailed == 1); +} + +TEST_CASE("the book sync respects its minimum interval", "[liveRunner]") { + auto [spec, strategy] = makeBookSpec("EURUSD", "u-cadence", + /*closeWhenActive=*/false); + std::vector specs; + specs.push_back(std::move(spec)); + + ScriptedFeed feed; + feed.script = {std::vector{}}; + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, {}, + feed.fn(), kDiscardClose, + std::chrono::hours{1}); + runner.start(); + for (int i = 0; i < 5; ++i) { + runner.onTick(makeTick("EURUSD")); + } + runner.stop(); + + CHECK(feed.calls.size() == 1); // first tick syncs, the rest are inside + // the interval +} + +TEST_CASE("a strategy close emits a CloseIntent carrying the booked deal", + "[liveRunner]") { + auto [spec, strategy] = makeBookSpec("EURUSD", "u-close", + /*closeWhenActive=*/true); + std::vector specs; + specs.push_back(std::move(spec)); + + ScriptedFeed feed; + feed.script = {std::vector{makeBooked()}, + std::vector{}}; // gone after close + std::vector intents; + live::CloseSink closeSink = [&intents](const live::CloseIntent& intent) { + intents.push_back(intent); + }; + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, {}, + feed.fn(), std::move(closeSink), + std::chrono::seconds{0}); + runner.start(); + runner.onTick(makeTick("EURUSD")); // seed, then during() closes + runner.onTick(makeTick("EURUSD")); // book empty — nothing more happens + runner.stop(); + + REQUIRE(intents.size() == 1); + const live::CloseIntent& intent = intents[0]; + CHECK(intent.strategyName == "BookStrategy"); + CHECK(intent.strategyUuid == "u-close"); + CHECK(intent.symbol == "EURUSD"); + CHECK(intent.direction == Direction::LONG); // the OPEN direction + CHECK(intent.brokerSize == 1.5); // exact broker units + CHECK(intent.dealId == "DIAAA-1"); + CHECK(intent.dealReference == "IGREF-1"); + const live::RunnerStats stats = runner.stats(); + CHECK(stats.strategyCloses == 1); + CHECK(stats.closeDropped == 0); + CHECK(stats.bookRemoved == 0); // the strategy closed it, not the sync +} + +TEST_CASE("a strategy close without a booked dealId is dropped and counted", + "[liveRunner]") { + auto [spec, strategy] = makeBookSpec("EURUSD", "u-unconfirmed", + /*closeWhenActive=*/true); + std::vector specs; + specs.push_back(std::move(spec)); + + ScriptedFeed feed; + // The deal never confirmed — booked with an empty dealId; then gone (so + // the second tick cannot re-seed and re-close). + feed.script = {std::vector{makeBooked("")}, + std::vector{}}; + int closeIntents = 0; + live::CloseSink closeSink = [&closeIntents](const live::CloseIntent&) { + ++closeIntents; + }; + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, {}, + feed.fn(), std::move(closeSink), + std::chrono::seconds{0}); + runner.start(); + runner.onTick(makeTick("EURUSD")); + runner.stop(); + + CHECK(closeIntents == 0); // nothing addressable at the broker + const live::RunnerStats stats = runner.stats(); + CHECK(stats.closeDropped == 1); + CHECK(stats.strategyCloses == 0); +} + +TEST_CASE("a feed for another symbol's deals books nothing", "[liveRunner]") { + // Belt-and-braces on the per-worker symbol scoping: the feed adapter + // filters by symbol, so a worker whose symbol has no deals gets an + // empty vector — which must seed nothing. + auto [spec, strategy] = makeBookSpec("USDJPY", "u-other", + /*closeWhenActive=*/false); + std::vector specs; + specs.push_back(std::move(spec)); + + ScriptedFeed feed; + feed.script = {std::vector{}}; + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, kDiscard, {}, + feed.fn(), kDiscardClose, + std::chrono::seconds{0}); + runner.start(); + runner.onTick(makeTick("USDJPY")); + runner.stop(); + + REQUIRE(feed.calls.size() == 1); + CHECK(feed.calls[0].second == "USDJPY"); // the worker asks for ITS symbol + CHECK(strategy->sawActive == 0); + CHECK(runner.stats().bookSeeded == 0); +} + +// ATR entry conditions: with gateSeries engaged, signals on a cold worker are +// skipped BEFORE decide() (counted as conditionsSkipped); once the gate +// series warms, orders carry the dynamic ATR-derived pip distances instead of +// the raw multipliers. +TEST_CASE("ATR entry gate skips cold workers then emits dynamic distances", + "[liveRunner]") { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + + auto [spec, stub] = makeSpec("EURUSD", "u-atr", Direction::LONG); + spec.vars.STOP_DISTANCE_IN_ATR = 1; + spec.vars.LIMIT_DISTANCE_IN_ATR = 3; + spec.gateSeries = bars::SeriesSpec{std::chrono::minutes{15}, 11}; + std::vector specs; + specs.push_back(std::move(spec)); + + std::vector orders; + live::OrderSink sink = [&orders](const live::OrderIntent& order) { + orders.push_back(order); + }; + + live::StrategyRunner runner(std::move(specs), kAlwaysOpen, std::move(sink)); + runner.start(); + // Zero-spread ticks 16 minutes apart: each rolls a fresh 15m bar + // (calculateOHLC rolls on STRICTLY-greater than the duration) and steps + // the price 200 points (20 EURUSD pips), so every true range is 200 and + // ATR(10) reads exactly 200 points once 11 bars exist. The store updates + // at the TOP of processTick, so the gate on tick i sees i+1 bars: ticks + // 0-9 are the cold phase and ticks 10-11 are warm. + constexpr int kTicks = 12; + for (int i = 0; i < kTicks; ++i) { + const std::int32_t price = 110000 + i * 200; + runner.onTick( + makeTick("EURUSD", price, price, std::chrono::minutes{16 * i})); + } + runner.stop(); + + // Warm ticks: stop = 20 pips x 1, limit = 20 pips x 3 — the dynamic + // distances, not the raw multipliers. (No open-position gate in the + // runner, so both warm ticks emit.) + CHECK(stub->decideCalls == 2); + REQUIRE(orders.size() == 2); + CHECK(orders[0].stopDistancePips == 20); + CHECK(orders[0].limitDistancePips == 60); + CHECK(orders[1].stopDistancePips == 20); + CHECK(orders[1].limitDistancePips == 60); + + const live::RunnerStats stats = runner.stats(); + CHECK(stats.conditionsSkipped == 10); + CHECK(stats.signals == 2); + CHECK(stats.ordersLogged == 2); +} diff --git a/tests/liveStrategyCache.cpp b/tests/liveStrategyCache.cpp new file mode 100644 index 0000000..87e8935 --- /dev/null +++ b/tests/liveStrategyCache.cpp @@ -0,0 +1,105 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// StrategyCache::build gatekeeping: a winner only becomes a live worker when +// its symbol is tradeable (known to symbol_scale — the tick decoder drops +// unknown symbols, so an unvalidated worker would look cached but never +// receive a tick) and its config carries the trade-lock UUID. One bad winner +// must not affect the rest. + +#include + +#include +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import liveStrategyCache; +import liveWinners; + +namespace { + +live::Winner makeWinner(const std::string& symbol, const std::string& uuid) { + live::Winner winner; + winner.symbol = symbol; + winner.strategyName = "RandomStrategy"; + winner.runId = "run-" + uuid; + winner.performanceScore = 20.0; + winner.config.UUID = uuid; + winner.config.TRADING_VARIABLES.STRATEGY = "RandomStrategy"; + winner.config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR = 25; + winner.config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR = 50; + winner.config.TRADING_VARIABLES.TRADING_SIZE = 3; + return winner; +} + +} // namespace + +TEST_CASE("StrategyCache skips winners whose symbol is not in symbol_scale", + "[liveStrategyCache]") { + std::vector winners; + winners.push_back(makeWinner("EURUSD", "u-known")); + winners.push_back(makeWinner("DOGEUSD", "u-unknown")); + + const auto specs = live::StrategyCache::build(winners); + + REQUIRE(specs.size() == 1); + CHECK(specs[0].symbol == "EURUSD"); + CHECK(specs[0].strategyUuid == "u-known"); +} + +TEST_CASE("StrategyCache skips winners without a trade-lock UUID", + "[liveStrategyCache]") { + std::vector winners; + winners.push_back(makeWinner("EURUSD", "")); + winners.push_back(makeWinner("USDJPY", "u-ok")); + + const auto specs = live::StrategyCache::build(winners); + + REQUIRE(specs.size() == 1); + CHECK(specs[0].symbol == "USDJPY"); +} + +TEST_CASE("StrategyCache carries the winner's trading variables into the spec", + "[liveStrategyCache]") { + live::Winner winner = makeWinner("EURUSD", "u-vars"); + winner.peakHoursOnly = true; + const auto specs = live::StrategyCache::build({std::move(winner)}); + + REQUIRE(specs.size() == 1); + CHECK(specs[0].strategyName == "RandomStrategy"); + CHECK(specs[0].vars.STOP_DISTANCE_IN_ATR == 25); + CHECK(specs[0].vars.LIMIT_DISTANCE_IN_ATR == 50); + CHECK(specs[0].vars.TRADING_SIZE == 3); + CHECK(specs[0].peakHoursOnly == true); + CHECK(specs[0].strategy != nullptr); +} + +TEST_CASE("StrategyCache turns RANGE_VARIABLES into worker range series, " + "skipping the all-zeros sentinel", + "[liveStrategyCache]") { + live::Winner ranged = makeWinner("EURUSD", "u-range"); + ranged.config.RANGE_VARIABLES = { + {.RANGE_ATR_TICK_WINDOW = 5000, .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 50}, + {}, // the all-zeros "unused" sentinel — registers nothing + }; + // A pre-range winner parses to an empty RANGE_VARIABLES and must build a + // spec with no range series, exactly as before the field existed. + live::Winner plain = makeWinner("USDJPY", "u-plain"); + + std::vector winners; + winners.push_back(std::move(ranged)); + winners.push_back(std::move(plain)); + const auto specs = live::StrategyCache::build(winners); + + REQUIRE(specs.size() == 2); + REQUIRE(specs[0].rangeSeries.size() == 1); + CHECK(specs[0].rangeSeries[0].atrTickWindow == 5000); + CHECK(specs[0].rangeSeries[0].atrPercent == 40); + CHECK(specs[0].rangeSeries[0].count == 50); + CHECK(specs[1].rangeSeries.empty()); +} diff --git a/tests/liveTrace.cpp b/tests/liveTrace.cpp new file mode 100644 index 0000000..8f422be --- /dev/null +++ b/tests/liveTrace.cpp @@ -0,0 +1,272 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// liveTrace: the armed gate (inert until init(), LIVE_TRACE_ENABLED=0 keeps +// it disarmed, the environment is recorded either way) and the document +// envelope (@timestamp to the millisecond, env/hostname/event, empty +// correlation ids and empty string fields omitted, each Field alternative +// serialising as its JSON type). The machinery only — no pinned config +// values, and nothing here arms tracing without disarming again (an armed +// gate in another TU's tests would try to deliver documents at process +// exit). +// +// ELASTIC_ENABLED=0 is set before every init() as a second fence: even if an +// emit slipped through while armed, the publisher would drop it instead of +// spinning up its flusher against a dead localhost. + +#include + +#include + +#include +#include +#include +#include +#include + +#include "shared/utilities/backtestLog.hpp" + +import backtestLog; // backtest_log::logLine — the stdout half of the sink +import liveTrace; + +namespace { + +// "YYYY-MM-DDTHH:MM:SS.mmmZ" — 24 chars, digits everywhere but the fixed +// separators. +bool isIsoUtcMillis(const std::string& ts) { + if (ts.size() != 24) { + return false; + } + for (std::size_t i = 0; i < ts.size(); ++i) { + const char c = ts[i]; + switch (i) { + case 4: + case 7: + if (c != '-') return false; + break; + case 10: + if (c != 'T') return false; + break; + case 13: + case 16: + if (c != ':') return false; + break; + case 19: + if (c != '.') return false; + break; + case 23: + if (c != 'Z') return false; + break; + default: + if (std::isdigit(static_cast(c)) == 0) return false; + break; + } + } + return true; +} + +} // namespace + +// Declared first in this TU on purpose: in a single-process run it must see +// the gate before any test below arms it (ctest runs each case in its own +// process anyway). +TEST_CASE("tracing is inert before init", "[liveTrace]") { + CHECK_FALSE(live_trace::enabled()); + CHECK(live_trace::tradingEnv() == "unknown"); + // emit while disarmed is a no-op — must not throw or touch the publisher. + live_trace::emit("orderIntent", {}, {{"size", std::int64_t{3}}}); + CHECK_FALSE(live_trace::enabled()); +} + +TEST_CASE("init arms tracing and records the environment", "[liveTrace]") { + ::setenv("ELASTIC_ENABLED", "0", 1); + ::unsetenv("LIVE_TRACE_ENABLED"); // default: on + + live_trace::init("demo"); + CHECK(live_trace::enabled()); + CHECK(live_trace::tradingEnv() == "demo"); + // The default-on ship gate registers the live-logs sink too. + CHECK(backtest_log::sinkArmed()); + // An armed emit routes to the (disabled) publisher without throwing. + live_trace::emit("startup", {}, {{"workers", std::uint64_t{2}}}); + + live_trace::disarm(); + CHECK_FALSE(live_trace::enabled()); + CHECK_FALSE(backtest_log::sinkArmed()); +} + +TEST_CASE("LIVE_TRACE_ENABLED=0 keeps tracing disarmed but records the env", + "[liveTrace]") { + ::setenv("ELASTIC_ENABLED", "0", 1); + ::setenv("LIVE_TRACE_ENABLED", "0", 1); + + live_trace::init("live"); + CHECK_FALSE(live_trace::enabled()); + // The env is stored regardless, for the live-trades audit stamp. + CHECK(live_trace::tradingEnv() == "live"); + + ::unsetenv("LIVE_TRACE_ENABLED"); + live_trace::disarm(); +} + +TEST_CASE("document envelope carries timestamp, env, hostname and event", + "[liveTrace]") { + ::setenv("ELASTIC_ENABLED", "0", 1); + ::setenv("LIVE_TRACE_ENABLED", "0", 1); // record env without arming + live_trace::init("demo"); + ::unsetenv("LIVE_TRACE_ENABLED"); + + const auto doc = nlohmann::json::parse( + live_trace::buildDocumentJson("orderAccepted", {}, {})); + + CHECK(isIsoUtcMillis(doc.at("@timestamp").get())); + CHECK(doc.at("env").get() == "demo"); + CHECK_FALSE(doc.at("hostname").get().empty()); + CHECK(doc.at("event").get() == "orderAccepted"); + + live_trace::disarm(); +} + +TEST_CASE("non-empty correlation ids are indexed, empty ones omitted", + "[liveTrace]") { + const auto doc = nlohmann::json::parse(live_trace::buildDocumentJson( + "closeOk", + {.strategyUuid = "f47ac10b-58cc-4372-a567-0e02b2c3d479", + .strategyName = "StubStrategy", + .symbol = "EURUSD", + .dealReference = "f47ac10b-L1719360000000"}, + {})); + + CHECK(doc.at("strategyUuid").get() + == "f47ac10b-58cc-4372-a567-0e02b2c3d479"); + CHECK(doc.at("strategyName").get() == "StubStrategy"); + CHECK(doc.at("symbol").get() == "EURUSD"); + CHECK(doc.at("dealReference").get() + == "f47ac10b-L1719360000000"); + CHECK_FALSE(doc.contains("dealId")); // empty -> omitted +} + +TEST_CASE("each field alternative serialises as its JSON type", + "[liveTrace]") { + const auto doc = nlohmann::json::parse(live_trace::buildDocumentJson( + "stats", {}, + {{"flag", true}, + {"delta", std::int64_t{-7}}, + {"count", std::uint64_t{42}}, + {"score", 1.5}, + {"reason", "rateCap"}, + {"empty", std::string_view{}}})); + + CHECK(doc.at("flag").is_boolean()); + CHECK(doc.at("flag").get() == true); + CHECK(doc.at("delta").is_number_integer()); + CHECK(doc.at("delta").get() == -7); + CHECK(doc.at("count").is_number_unsigned()); + CHECK(doc.at("count").get() == 42); + CHECK(doc.at("score").is_number_float()); + CHECK(doc.at("score").get() == 1.5); + CHECK(doc.at("reason").is_string()); + CHECK(doc.at("reason").get() == "rateCap"); + // Empty string fields follow the empty-id convention: omitted, not "". + CHECK_FALSE(doc.contains("empty")); +} + +TEST_CASE("log documents carry the shared envelope, a level and the message", + "[liveTrace]") { + ::setenv("ELASTIC_ENABLED", "0", 1); + ::setenv("LIVE_TRACE_ENABLED", "0", 1); + ::setenv("LIVE_LOG_SHIP_ENABLED", "0", 1); // envelope only, no sink + live_trace::init("demo"); + ::unsetenv("LIVE_TRACE_ENABLED"); + ::unsetenv("LIVE_LOG_SHIP_ENABLED"); + + const auto errorDoc = nlohmann::json::parse( + live_trace::buildLogDocumentJson(true, "OrderChannel: it broke")); + CHECK(isIsoUtcMillis(errorDoc.at("@timestamp").get())); + CHECK(errorDoc.at("env").get() == "demo"); + CHECK_FALSE(errorDoc.at("hostname").get().empty()); + CHECK(errorDoc.at("level").get() == "error"); + CHECK(errorDoc.at("message").get() + == "OrderChannel: it broke"); + + const auto infoDoc = + nlohmann::json::parse(live_trace::buildLogDocumentJson(false, "ok")); + CHECK(infoDoc.at("level").get() == "info"); + + // The cap: a pathological line is truncated, not dropped or indexed raw. + const std::string huge(10000, 'x'); + const auto cappedDoc = + nlohmann::json::parse(live_trace::buildLogDocumentJson(false, huge)); + CHECK(cappedDoc.at("message").get().size() == 4096); + + live_trace::disarm(); +} + +namespace { + +// Collecting sink for the registration tests — capture-free, matching +// backtest_log::Sink; state lives in the accessor's static. +std::vector>& collectedLines() { + static std::vector> lines; + return lines; +} + +void collectingSink(const bool isError, const std::string_view message) { + collectedLines().emplace_back(isError, std::string{message}); +} + +} // namespace + +TEST_CASE("the backtest_log sink receives error and logLine text, honours " + "suppression, and disarms", + "[liveTrace]") { + collectedLines().clear(); + backtest_log::setSink(&collectingSink); + + backtest_log::error("stderr line"); + backtest_log::logLine("stdout {} {}", "line", 7); + REQUIRE(collectedLines().size() == 2); + CHECK(collectedLines()[0] == std::pair{true, std::string{"stderr line"}}); + CHECK(collectedLines()[1] == std::pair{false, std::string{"stdout line 7"}}); + + // The publisher's delivery paths log under this guard — nothing ships. + { + const backtest_log::SinkSuppression suppression; + backtest_log::error("publisher-origin line"); + } + CHECK(collectedLines().size() == 2); + + // Suppression is scoped: shipping resumes when the guard leaves. + backtest_log::error("after the guard"); + CHECK(collectedLines().size() == 3); + + backtest_log::setSink(nullptr); + backtest_log::error("after disarm"); + CHECK(collectedLines().size() == 3); +} + +TEST_CASE("LIVE_LOG_SHIP_ENABLED=0 keeps the live-logs sink unregistered " + "and clears a previously-armed one", + "[liveTrace]") { + ::setenv("ELASTIC_ENABLED", "0", 1); + ::setenv("LIVE_TRACE_ENABLED", "0", 1); + ::setenv("LIVE_LOG_SHIP_ENABLED", "0", 1); + + // Register BEFORE init: a gate-off init must clear the slot (the gate is + // authoritative in both directions), and must not install shipLogLine. + // Observing the slot directly (sinkArmed) catches the register-anyway + // regression; the collecting sink catches the fails-to-clear one. + collectedLines().clear(); + backtest_log::setSink(&collectingSink); + live_trace::init("demo"); + CHECK_FALSE(backtest_log::sinkArmed()); + backtest_log::error("into the void"); + CHECK(collectedLines().empty()); + + ::unsetenv("LIVE_TRACE_ENABLED"); + ::unsetenv("LIVE_LOG_SHIP_ENABLED"); + live_trace::disarm(); +} diff --git a/tests/liveWinners.cpp b/tests/liveWinners.cpp new file mode 100644 index 0000000..2662193 --- /dev/null +++ b/tests/liveWinners.cpp @@ -0,0 +1,950 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// selectTopWinners is the pure half of liveWinners (no Elasticsearch): these +// tests drive it with synthetic _search response bodies, pinning the +// selection machinery — filtering, symbol splitting, per-group ranking, +// behavioural-clone dedup, the behavioural diversity gate (trade count / +// PnL / long-short mix, with the config-space gate as the legacy fallback +// for documents predating results.tradesClosed) and malformed-hit +// tolerance — not any currently-configured sweep values. Fixtures that omit +// tradesClosed parse to the -1 fallback and thus exercise the LEGACY +// diversity path; behavioural-gate tests must set it. buildWinnersQueryBody +// is pinned too (the strategy .keyword term and the analyzed-SYMBOLS match +// are both silent-failure-shaped if they regress). + +#include + +#include +#include +#include +#include +#include + +#include + +#include "shared/tradingDefinitions/config/runConfiguration.hpp" +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import liveWinners; + +namespace { + +constexpr std::array kActive{"RandomStrategy", + "OhlcBreakoutStrategy"}; + +// Optional knobs for the dedup/diversity machinery. Defaults reproduce the +// pre-existing fixture shape: empty OHLC_VARIABLES and a results object +// carrying only performanceScore — which doubles as the "old document" shape +// for the fallback-key tests. +struct HitExtras { + std::vector ohlc{}; + // Emitted only when non-empty, so the default fixture keeps the + // pre-range document shape — every test without it doubles as the + // backward-compat pin for RANGE_VARIABLES-less winners. + std::vector range{}; + std::optional finalPnl{}; + std::optional tradesClosed{}; + // Emitted only when set — every fixture without them doubles as the + // mid-vintage pin (docs carrying tradesClosed but predating the + // opened counts), where the direction axis must stand down. + std::optional openedLong{}; + std::optional openedShort{}; +}; + +// One winners-index hit as the reporting path writes it. The ES documents +// carry TRADING_VARIABLES re-typed as JSON numbers (reportConfigJson); the +// Redis sweep payloads carry them string-encoded — both shapes must parse, so +// the fixture can produce either. +nlohmann::json makeHit(const std::string& runId, const std::string& symbols, + const std::string& strategyName, const std::string& uuid, + const double score, const bool numericVars = true, + const HitExtras& extras = {}) { + nlohmann::json vars{{"STRATEGY", strategyName}}; + if (numericVars) { + vars["STOP_DISTANCE_IN_ATR"] = 25; + vars["LIMIT_DISTANCE_IN_ATR"] = 50; + vars["TRADING_SIZE"] = 3; + } else { + vars["STOP_DISTANCE_IN_ATR"] = "25"; + vars["LIMIT_DISTANCE_IN_ATR"] = "50"; + vars["TRADING_SIZE"] = "3"; + } + nlohmann::json results{{"performanceScore", score}}; + if (extras.finalPnl) { + results["finalPnl"] = *extras.finalPnl; + } + if (extras.tradesClosed) { + results["tradesClosed"] = *extras.tradesClosed; + } + if (extras.openedLong) { + results["openedLong"] = *extras.openedLong; + } + if (extras.openedShort) { + results["openedShort"] = *extras.openedShort; + } + nlohmann::json hit{{"_source", + {{"RUN_ID", runId}, + {"config", + {{"SYMBOLS", symbols}, + {"STRATEGY", + {{"UUID", uuid}, + {"TRADING_VARIABLES", vars}, + {"OHLC_VARIABLES", nlohmann::json(extras.ohlc)}, + {"STRATEGY_VARIABLES", nlohmann::json::object()}}}}}, + {"results", results}}}}; + if (!extras.range.empty()) { + hit["_source"]["config"]["STRATEGY"]["RANGE_VARIABLES"] = extras.range; + } + return hit; +} + +std::string makeResponse(const std::vector& hits) { + return nlohmann::json{ + {"hits", {{"total", {{"value", hits.size()}}}, {"hits", hits}}}} + .dump(); +} + +tradingDefinitions::OHLCVariables series(const int count, const int minutes) { + return {.OHLC_COUNT = count, .OHLC_MINUTES = minutes}; +} + +} // namespace + +TEST_CASE("selectTopWinners keeps the top N per (symbol, strategy) by score", + "[liveWinners]") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "uuid-40", 40.0), + makeHit("r2", "EURUSD", "RandomStrategy", "uuid-42", 42.0), + makeHit("r3", "EURUSD", "RandomStrategy", "uuid-39", 39.0), + makeHit("r4", "EURUSD", "RandomStrategy", "uuid-41", 41.0), + }); + + const auto winners = live::selectTopWinners(body, kActive, 2); + + REQUIRE(winners.size() == 2); + CHECK(winners[0].config.UUID == "uuid-42"); + CHECK(winners[1].config.UUID == "uuid-41"); + CHECK(winners[0].performanceScore == 42.0); + CHECK(winners[0].symbol == "EURUSD"); + CHECK(winners[0].strategyName == "RandomStrategy"); + CHECK(winners[0].runId == "r2"); +} + +TEST_CASE("selectTopWinners ranks strategies independently per group", + "[liveWinners]") { + // Two strategies on one symbol, one strategy on another: each group keeps + // its own top 2, so a dominant strategy cannot crowd out the others. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-r1", 50.0), + makeHit("r2", "EURUSD", "RandomStrategy", "u-r2", 49.0), + makeHit("r3", "EURUSD", "RandomStrategy", "u-r3", 48.0), + makeHit("r4", "EURUSD", "OhlcBreakoutStrategy", "u-o1", 11.0), + makeHit("r5", "USDJPY", "OhlcBreakoutStrategy", "u-o2", 12.0), + }); + + const auto winners = live::selectTopWinners(body, kActive, 2); + + REQUIRE(winners.size() == 4); + // Output is ordered by symbol then strategy name (std::map iteration). + CHECK(winners[0].config.UUID == "u-o1"); // EURUSD / OhlcBreakout + CHECK(winners[1].config.UUID == "u-r1"); // EURUSD / Random, top score + CHECK(winners[2].config.UUID == "u-r2"); // EURUSD / Random, second + CHECK(winners[3].config.UUID == "u-o2"); // USDJPY / OhlcBreakout +} + +TEST_CASE("selectTopWinners drops strategies not in the active list", + "[liveWinners]") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RetiredStrategy", "u-x", 99.0), + makeHit("r2", "EURUSD", "RandomStrategy", "u-r", 20.0), + }); + + const auto winners = live::selectTopWinners(body, kActive, 2); + + REQUIRE(winners.size() == 1); + CHECK(winners[0].config.UUID == "u-r"); +} + +TEST_CASE("selectTopWinners splits a multi-symbol run into one winner per symbol", + "[liveWinners]") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD,USDJPY", "RandomStrategy", "u-multi", 30.0), + }); + + const auto winners = live::selectTopWinners(body, kActive, 2); + + REQUIRE(winners.size() == 2); + CHECK(winners[0].symbol == "EURUSD"); + CHECK(winners[1].symbol == "USDJPY"); + CHECK(winners[0].config.UUID == "u-multi"); + CHECK(winners[1].config.UUID == "u-multi"); +} + +TEST_CASE("selectTopWinners trims whitespace and drops empty fields in SYMBOLS", + "[liveWinners]") { + // Historical documents can carry hand-written symbol groups; an untrimmed + // " USDJPY" would cache a worker no decoded tick could ever route to. + const std::string body = makeResponse({ + makeHit("r1", " EURUSD , USDJPY ,", "RandomStrategy", "u-ws", 30.0), + }); + + const auto winners = live::selectTopWinners(body, kActive, 2); + + REQUIRE(winners.size() == 2); + CHECK(winners[0].symbol == "EURUSD"); + CHECK(winners[1].symbol == "USDJPY"); +} + +TEST_CASE("selectTopWinners skips a malformed hit and keeps the rest", + "[liveWinners]") { + nlohmann::json broken = makeHit("r1", "EURUSD", "RandomStrategy", "u-b", 44.0); + broken["_source"]["config"].erase("STRATEGY"); + + const std::string body = makeResponse({ + broken, + makeHit("r2", "EURUSD", "RandomStrategy", "u-ok", 33.0), + }); + + const auto winners = live::selectTopWinners(body, kActive, 2); + + REQUIRE(winners.size() == 1); + CHECK(winners[0].config.UUID == "u-ok"); +} + +TEST_CASE("selectTopWinners parses numeric and string TRADING_VARIABLES", + "[liveWinners]") { + // numericVars=true is the reporting-path document shape + // (reportConfigJson re-types the pip/size fields as JSON numbers); + // numericVars=false is the shared string-encoded convention. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-num", 20.0, true), + makeHit("r2", "USDJPY", "RandomStrategy", "u-str", 21.0, false), + }); + + const auto winners = live::selectTopWinners(body, kActive, 2); + + REQUIRE(winners.size() == 2); + for (const auto& winner : winners) { + CHECK(winner.config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == 25); + CHECK(winner.config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == 50); + CHECK(winner.config.TRADING_VARIABLES.TRADING_SIZE == 3); + CHECK(winner.config.TRADING_VARIABLES.STRATEGY == "RandomStrategy"); + } +} + +TEST_CASE("selectTopWinners carries the run-level risk caps and defaults " + "documents that predate them", + "[liveWinners]") { + nlohmann::json capped = + makeHit("r1", "EURUSD", "RandomStrategy", "u-cap", 30.0); + capped["_source"]["config"]["MAX_OPEN_TRADES"] = 4; + capped["_source"]["config"]["MAX_TRADES_PER_MINUTE"] = 7; + capped["_source"]["config"]["PEAK_HOURS_ONLY"] = true; + + const std::string body = makeResponse({ + capped, + makeHit("r2", "USDJPY", "RandomStrategy", "u-old", 30.0), // no caps + }); + + const auto winners = live::selectTopWinners(body, kActive, 2); + + REQUIRE(winners.size() == 2); + CHECK(winners[0].config.UUID == "u-cap"); + CHECK(winners[0].maxOpenTrades == 4); + CHECK(winners[0].maxTradesPerMinute == 7); + CHECK(winners[0].peakHoursOnly == true); + + // Pre-cap documents fall back to the same defaults the backtest itself + // would have applied — RunConfiguration's, not literals repeated here. + const tradingDefinitions::RunConfiguration defaults{}; + CHECK(winners[1].config.UUID == "u-old"); + CHECK(winners[1].maxOpenTrades == defaults.MAX_OPEN_TRADES); + CHECK(winners[1].maxTradesPerMinute == defaults.MAX_TRADES_PER_MINUTE); + CHECK(winners[1].peakHoursOnly == defaults.PEAK_HOURS_ONLY); +} + +TEST_CASE("selectTopWinners returns empty on garbage or empty responses", + "[liveWinners]") { + CHECK(live::selectTopWinners("not json at all", kActive, 2).empty()); + CHECK(live::selectTopWinners("{}", kActive, 2).empty()); + CHECK(live::selectTopWinners(makeResponse({}), kActive, 2).empty()); +} + +TEST_CASE("selectTopWinners survives 2xx bodies that are not ES envelopes", + "[liveWinners]") { + // A proxy in front of Elasticsearch can return 200 with its own JSON + // (error envelopes, arrays, nulls). These parse fine but are not + // _search-shaped; they must take the empty-return path, not throw an + // uncaught type_error out of live startup. + CHECK(live::selectTopWinners("[1, 2, 3]", kActive, 2).empty()); + CHECK(live::selectTopWinners("\"gateway timeout\"", kActive, 2).empty()); + CHECK(live::selectTopWinners("null", kActive, 2).empty()); + CHECK(live::selectTopWinners(R"({"hits": null})", kActive, 2).empty()); + CHECK(live::selectTopWinners(R"({"hits": {"hits": "bogus"}})", kActive, 2) + .empty()); + CHECK(live::selectTopWinners(R"({"hits": {"hits": null}})", kActive, 2) + .empty()); +} + +TEST_CASE("selectTopWinners skips behavioural clones with identical results " + "tuples", + "[liveWinners]") { + // Re-running `load` mints a fresh UUID for the identical config; the + // deterministic backtest then writes an identical results tuple. The + // clone must not consume a slot even though its OHLC config is diverse. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-a", 50.0, true, + {.ohlc = {series(20, 5)}, .finalPnl = 12.5, .tradesClosed = 40}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-clone", 50.0, true, + {.ohlc = {series(20, 15)}, .finalPnl = 12.5, .tradesClosed = 40}), + makeHit("r3", "EURUSD", "OhlcBreakoutStrategy", "u-b", 49.0, true, + {.ohlc = {series(20, 30)}, .finalPnl = 8.0, .tradesClosed = 31}), + makeHit("r4", "EURUSD", "OhlcBreakoutStrategy", "u-c", 48.0, true, + {.ohlc = {series(20, 60)}, .finalPnl = 5.0, .tradesClosed = 22}), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 3); + CHECK(winners[0].config.UUID == "u-a"); + CHECK(winners[1].config.UUID == "u-b"); + CHECK(winners[2].config.UUID == "u-c"); +} + +TEST_CASE("selectTopWinners books a doubly-fetched multi-symbol document " + "once per symbol", + "[liveWinners]") { + // fetchWinners queries per (strategy, symbol), so a multi-symbol + // document matches each of its symbols' queries and lands in the merged + // candidate map once per fetch. The copies are exact duplicates — + // identical results tuples — and must dedup as behavioural clones, not + // book two workers per group. + const nlohmann::json hit = + makeHit("r1", "EURUSD,USDJPY", "RandomStrategy", "u-multi", 30.0, true, + {.finalPnl = 3.0, .tradesClosed = 7}); + const std::string body = makeResponse({hit, hit}); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 2); + CHECK(winners[0].symbol == "EURUSD"); + CHECK(winners[1].symbol == "USDJPY"); + CHECK(winners[0].config.UUID == "u-multi"); + CHECK(winners[1].config.UUID == "u-multi"); +} + +TEST_CASE("selectTopWinners treats equal scores with differing finalPnl or " + "tradesClosed as distinct, not clones", + "[liveWinners]") { + // Pins the dedup KEY: equal scores alone must not read as clones when + // finalPnl or tradesClosed differ. The differences here clear the + // behavioural neighbour bands so the diversity gate books both — + // near-identical tuples are that gate's business, pinned by the + // noise-floor tests below. + const std::string pnlDiffers = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-1", 50.0, true, + {.ohlc = {series(20, 5)}, .finalPnl = 1000.0, .tradesClosed = 40}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-2", 50.0, true, + {.ohlc = {series(20, 15)}, .finalPnl = 2000.0, .tradesClosed = 40}), + }); + CHECK(live::selectTopWinners(pnlDiffers, kActive, 3).size() == 2); + + const std::string tradesDiffer = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-1", 50.0, true, + {.ohlc = {series(20, 5)}, .finalPnl = 12.5, .tradesClosed = 40}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-2", 50.0, true, + {.ohlc = {series(20, 15)}, .finalPnl = 12.5, .tradesClosed = 60}), + }); + CHECK(live::selectTopWinners(tradesDiffer, kActive, 3).size() == 2); +} + +TEST_CASE("selectTopWinners excludes behavioural neighbours whose results " + "sit close on every axis", + "[liveWinners]") { + // LSR-shaped (2026-07-12 batch): a sweep-pips jiggle closed 41 vs 38 + // trades for 1692 vs 1610 PnL — the market barely noticed the knob. + // The OHLC minutes DIFFER (5 vs 15), which the config-space gate called + // diverse; behavioural closeness must win over config distance. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-top", 50.0, true, + {.ohlc = {series(20, 5)}, + .finalPnl = 1692.0, .tradesClosed = 41}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-jiggle", 49.0, true, + {.ohlc = {series(20, 15)}, + .finalPnl = 1610.0, .tradesClosed = 38}), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 1); + CHECK(winners[0].config.UUID == "u-top"); +} + +TEST_CASE("selectTopWinners admits candidates whose trade counts sit apart", + "[liveWinners]") { + // The VALID_BARS case (2026-07-12): 41 vs 60 trades at similar PnL is + // a different behaviour. The configs share one bar series with counts + // inside the legacy gap (20 vs 22 on 5m) — the old gate called this a + // jiggle and booked one; the trades axis books both. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-41", 50.0, true, + {.ohlc = {series(20, 5)}, + .finalPnl = 1692.0, .tradesClosed = 41}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-60", 49.0, true, + {.ohlc = {series(22, 5)}, + .finalPnl = 1897.0, .tradesClosed = 60}), + }); + + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); +} + +TEST_CASE("selectTopWinners admits candidates whose PnL sits apart beyond " + "band and floor", + "[liveWinners]") { + // Same trade count; 1000 vs 2000 clears the relative band (20% of + // 2000 = 400) and the absolute floor. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-1k", 50.0, true, + {.finalPnl = 1000.0, .tradesClosed = 50}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-2k", 49.0, true, + {.finalPnl = 2000.0, .tradesClosed = 50}), + }); + + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); +} + +TEST_CASE("selectTopWinners treats sub-floor PnL differences as noise, not " + "diversity", + "[liveWinners]") { + SECTION("10 vs 20: 100% apart relatively, spread noise absolutely") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-20", 50.0, true, + {.finalPnl = 20.0, .tradesClosed = 50}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-10", 49.0, true, + {.finalPnl = 10.0, .tradesClosed = 50}), + }); + const auto winners = live::selectTopWinners(body, kActive, 3); + REQUIRE(winners.size() == 1); + CHECK(winners[0].config.UUID == "u-20"); + } + SECTION("a sign flip inside the noise floor is still noise") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-plus", 50.0, true, + {.finalPnl = 5.0, .tradesClosed = 50}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-minus", 49.0, true, + {.finalPnl = -5.0, .tradesClosed = 50}), + }); + CHECK(live::selectTopWinners(body, kActive, 3).size() == 1); + } +} + +TEST_CASE("selectTopWinners treats results exactly at the band as " + "neighbours", + "[liveWinners]") { + // 100 vs 80 trades: the delta equals the 20% band exactly — distinct + // requires STRICTLY exceeding it, the same boundary doctrine the + // legacy count gap uses. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-100", 50.0, true, + {.finalPnl = 1600.0, .tradesClosed = 100}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-80", 49.0, true, + {.finalPnl = 1700.0, .tradesClosed = 80}), + }); + + CHECK(live::selectTopWinners(body, kActive, 3).size() == 1); +} + +TEST_CASE("selectTopWinners applies the absolute trade floor to small " + "counts", + "[liveWinners]") { + // 5 vs 8 trades is 60% apart relatively, but small counts are noisy — + // the absolute floor says neighbour. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-5", 50.0, true, + {.finalPnl = 600.0, .tradesClosed = 5}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-8", 49.0, true, + {.finalPnl = 700.0, .tradesClosed = 8}), + }); + + CHECK(live::selectTopWinners(body, kActive, 3).size() == 1); +} + +TEST_CASE("selectTopWinners admits a flipped long/short mix as diversity", + "[liveWinners]") { + // 40L/10S vs 10L/40S on near-identical counts and PnL: two + // parameterisations trading opposite sides of the same market are + // partially hedging — exactly the pair diversification wants kept. + // Without the direction axis these would merge as neighbours. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-long", 50.0, true, + {.finalPnl = 1500.0, .tradesClosed = 50, + .openedLong = 40, .openedShort = 10}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-short", 49.0, true, + {.finalPnl = 1520.0, .tradesClosed = 50, + .openedLong = 10, .openedShort = 40}), + }); + + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); +} + +TEST_CASE("selectTopWinners stands the direction axis down when a document " + "predates the opened counts", + "[liveWinners]") { + // Mid-vintage doc: carries tradesClosed but not openedLong/Short. The + // axis cannot claim distinctness it cannot measure, so the pair is + // judged on the remaining axes — close there, one booked. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-new", 50.0, true, + {.finalPnl = 1500.0, .tradesClosed = 50, + .openedLong = 40, .openedShort = 10}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-mid", 49.0, true, + {.finalPnl = 1520.0, .tradesClosed = 50}), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 1); + CHECK(winners[0].config.UUID == "u-new"); +} + +TEST_CASE("selectTopWinners requires behavioural diversity against every " + "already-picked winner", + "[liveWinners]") { + // A(100 trades) books; B(140) clears A (delta 40 > 20% of 140 = 28); + // C(120) sits within the band of BOTH picks (20 <= 24 vs A, 20 <= 28 + // vs B) — skipped; D(180) clears both (80 and 40 > 36) and takes the + // third slot. PnL identical throughout so only the trades axis moves. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-100", 50.0, true, + {.finalPnl = 2000.0, .tradesClosed = 100}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-140", 49.0, true, + {.finalPnl = 2000.0, .tradesClosed = 140}), + makeHit("r3", "EURUSD", "RandomStrategy", "u-120", 48.0, true, + {.finalPnl = 2000.0, .tradesClosed = 120}), + makeHit("r4", "EURUSD", "RandomStrategy", "u-180", 47.0, true, + {.finalPnl = 2000.0, .tradesClosed = 180}), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 3); + CHECK(winners[0].config.UUID == "u-100"); + CHECK(winners[1].config.UUID == "u-140"); + CHECK(winners[2].config.UUID == "u-180"); +} + +TEST_CASE("selectTopWinners falls back to the config gate when either " + "document predates tradesClosed", + "[liveWinners]") { + // Mixed pair: one old-shape doc (no tradesClosed -> -1 fallback), one + // behavioural doc. The pair cannot be judged behaviourally, so the + // legacy config-space gate governs it in both directions. + SECTION("differing bar minutes: config gate books both") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-old", 30.0, + true, {.ohlc = {series(20, 5)}}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-new", 29.0, + true, + {.ohlc = {series(20, 15)}, + .finalPnl = 1000.0, .tradesClosed = 40}), + }); + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); + } + SECTION("same series within the count gap: config gate books one") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-old", 30.0, + true, {.ohlc = {series(20, 5)}}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-new", 29.0, + true, + {.ohlc = {series(25, 5)}, + .finalPnl = 1000.0, .tradesClosed = 40}), + }); + const auto winners = live::selectTopWinners(body, kActive, 3); + REQUIRE(winners.size() == 1); + CHECK(winners[0].config.UUID == "u-old"); + } +} + +TEST_CASE("selectTopWinners dedups documents missing finalPnl/tradesClosed " + "on the fallback key", + "[liveWinners]") { + // Two old-shape documents (results carries only performanceScore) with + // the same score collide on the fallback key and dedup to one... + const std::string oldClones = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-1", 30.0, true, + {.ohlc = {series(20, 5)}}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-2", 30.0, true, + {.ohlc = {series(20, 15)}}), + }); + const auto deduped = live::selectTopWinners(oldClones, kActive, 3); + REQUIRE(deduped.size() == 1); + CHECK(deduped[0].config.UUID == "u-1"); + + // ...but a real finalPnl differs from the 0.0 fallback, so an old doc + // and a new same-score doc stay distinct. + const std::string oldVsNew = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-old", 30.0, true, + {.ohlc = {series(20, 5)}}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-new", 30.0, true, + {.ohlc = {series(20, 15)}, .finalPnl = 1.0}), + }); + CHECK(live::selectTopWinners(oldVsNew, kActive, 3).size() == 2); +} + +TEST_CASE("selectTopWinners rejects same-minutes candidates within the count " + "gap as not diverse", + "[liveWinners]") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-20", 50.0, true, + {.ohlc = {series(20, 5)}, .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-25", 49.0, true, + {.ohlc = {series(25, 5)}, .finalPnl = 2.0}), + makeHit("r3", "EURUSD", "OhlcBreakoutStrategy", "u-35", 48.0, true, + {.ohlc = {series(35, 5)}, .finalPnl = 3.0}), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + // 25 sits 5 bars from the picked 20 (a parameter jiggle); 35 sits 15 + // away and earns the second slot. + REQUIRE(winners.size() == 2); + CHECK(winners[0].config.UUID == "u-20"); + CHECK(winners[1].config.UUID == "u-35"); +} + +TEST_CASE("selectTopWinners treats differing bar minutes as diverse " + "regardless of count", + "[liveWinners]") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-m5", 50.0, true, + {.ohlc = {series(20, 5)}, .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-m15", 49.0, true, + {.ohlc = {series(20, 15)}, .finalPnl = 2.0}), + }); + + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); +} + +TEST_CASE("selectTopWinners counts one differing series element as diverse " + "in multi-series configs", + "[liveWinners]") { + // FVG-shaped: [0] scan series, [1] HTF series. Element 0 sits within the + // count gap, but element 1's minutes differ — diverse. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-htf60", 50.0, true, + {.ohlc = {series(20, 5), series(50, 60)}, .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-htf240", 49.0, true, + {.ohlc = {series(22, 5), series(50, 240)}, .finalPnl = 2.0}), + }); + + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); +} + +TEST_CASE("selectTopWinners treats differing series counts as diverse, " + "including empty vs non-empty", + "[liveWinners]") { + const std::string oneVsTwo = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-one", 50.0, true, + {.ohlc = {series(20, 5)}, .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-two", 49.0, true, + {.ohlc = {series(20, 5), series(50, 60)}, .finalPnl = 2.0}), + }); + CHECK(live::selectTopWinners(oneVsTwo, kActive, 3).size() == 2); + + // The both-empty guard must not swallow empty-vs-non-empty. + const std::string emptyVsOne = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-none", 50.0, true, + {.finalPnl = 1.0}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-bars", 49.0, true, + {.ohlc = {series(20, 5)}, .finalPnl = 2.0}), + }); + CHECK(live::selectTopWinners(emptyVsOne, kActive, 3).size() == 2); +} + +TEST_CASE("selectTopWinners lets empty-OHLC strategies through on clone " + "dedup alone", + "[liveWinners]") { + // RandomStrategy builds no bars: both-empty OHLC is vacuously diverse, + // so only the results tuple separates candidates. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-1", 50.0, true, + {.finalPnl = 1.0}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-2", 49.0, true, + {.finalPnl = 2.0}), + makeHit("r3", "EURUSD", "RandomStrategy", "u-3", 50.0, true, + {.finalPnl = 1.0}), // clone of u-1 + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 2); + CHECK(winners[0].config.UUID == "u-1"); + CHECK(winners[1].config.UUID == "u-2"); +} + +TEST_CASE("selectTopWinners requires diversity against every already-picked " + "winner", + "[liveWinners]") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-20", 50.0, true, + {.ohlc = {series(20, 5)}, .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-45", 49.0, true, + {.ohlc = {series(45, 5)}, .finalPnl = 2.0}), + makeHit("r3", "EURUSD", "OhlcBreakoutStrategy", "u-40", 48.0, true, + {.ohlc = {series(40, 5)}, .finalPnl = 3.0}), + makeHit("r4", "EURUSD", "OhlcBreakoutStrategy", "u-60", 47.0, true, + {.ohlc = {series(60, 5)}, .finalPnl = 4.0}), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + // 40 clears the gap against 20 but sits 5 from the picked 45 — skipped; + // 60 clears both picked candidates and takes the third slot. + REQUIRE(winners.size() == 3); + CHECK(winners[0].config.UUID == "u-20"); + CHECK(winners[1].config.UUID == "u-45"); + CHECK(winners[2].config.UUID == "u-60"); +} + +TEST_CASE("selectTopWinners books fewer than topPerGroup when the group " + "lacks diverse survivors", + "[liveWinners]") { + // Every candidate sits within the count gap of the top pick — including + // the exact-boundary |30 - 20| == 10, which is NOT diverse (the gap is + // strictly-greater-than). No backfill: one winner, not three. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-20", 50.0, true, + {.ohlc = {series(20, 5)}, .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-22", 49.0, true, + {.ohlc = {series(22, 5)}, .finalPnl = 2.0}), + makeHit("r3", "EURUSD", "OhlcBreakoutStrategy", "u-25", 48.0, true, + {.ohlc = {series(25, 5)}, .finalPnl = 3.0}), + makeHit("r4", "EURUSD", "OhlcBreakoutStrategy", "u-28", 47.0, true, + {.ohlc = {series(28, 5)}, .finalPnl = 4.0}), + makeHit("r5", "EURUSD", "OhlcBreakoutStrategy", "u-30", 46.0, true, + {.ohlc = {series(30, 5)}, .finalPnl = 5.0}), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 1); + CHECK(winners[0].config.UUID == "u-20"); +} + +TEST_CASE("selectTopWinners carries finalPnl and tradesClosed onto the " + "Winner, defaulting old documents", + "[liveWinners]") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-new", 30.0, true, + {.finalPnl = 7.25, .tradesClosed = 19}), + makeHit("r2", "USDJPY", "RandomStrategy", "u-old", 30.0), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 2); + CHECK(winners[0].config.UUID == "u-new"); + CHECK(winners[0].finalPnl == 7.25); + CHECK(winners[0].tradesClosed == 19); + // The fallbacks: 0.0 PnL, -1 trades — -1 so an old document can never + // collide with a genuine 0-trade run. + CHECK(winners[1].config.UUID == "u-old"); + CHECK(winners[1].finalPnl == 0.0); + CHECK(winners[1].tradesClosed == -1); +} + +TEST_CASE("selectTopWinners treats range-bar winners on one series identity " + "as parameter jiggles, not diverse", + "[liveWinners]") { + // Same (TICK_WINDOW, PERCENT); only the derived RANGE_COUNT differs — + // comparing it would mistake a derivation artefact for diversity. OHLC + // is empty on both (the range-bar strategy shape), and the results + // tuples differ so clone dedup cannot mask the diversity verdict. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-1", 50.0, true, + {.range = {{.RANGE_ATR_TICK_WINDOW = 5000, + .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 37}}, + .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-2", 49.0, true, + {.range = {{.RANGE_ATR_TICK_WINDOW = 5000, + .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 9}}, + .finalPnl = 2.0}), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 1); + CHECK(winners[0].config.UUID == "u-1"); +} + +TEST_CASE("selectTopWinners treats differing range window or percent as " + "diverse", + "[liveWinners]") { + SECTION("TICK_WINDOW differs") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-1", 50.0, true, + {.range = {{.RANGE_ATR_TICK_WINDOW = 2500, + .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 37}}, + .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-2", 49.0, true, + {.range = {{.RANGE_ATR_TICK_WINDOW = 5000, + .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 37}}, + .finalPnl = 2.0}), + }); + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); + } + SECTION("PERCENT differs") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-1", 50.0, true, + {.range = {{.RANGE_ATR_TICK_WINDOW = 5000, + .RANGE_ATR_PERCENT = 25, + .RANGE_COUNT = 37}}, + .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-2", 49.0, true, + {.range = {{.RANGE_ATR_TICK_WINDOW = 5000, + .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 37}}, + .finalPnl = 2.0}), + }); + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); + } + SECTION("range series present vs absent") { + // Both OHLC-empty: without the range axis these would be vacuously + // diverse-by-emptiness; the size mismatch is real diversity. + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-range", 50.0, true, + {.range = {{.RANGE_ATR_TICK_WINDOW = 5000, + .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 37}}, + .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "RandomStrategy", "u-none", 49.0, true, + {.finalPnl = 2.0}), + }); + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); + } + SECTION("OHLC diversity still wins when the range series are identical") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "OhlcBreakoutStrategy", "u-m5", 50.0, true, + {.ohlc = {series(20, 5)}, + .range = {{.RANGE_ATR_TICK_WINDOW = 5000, + .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 37}}, + .finalPnl = 1.0}), + makeHit("r2", "EURUSD", "OhlcBreakoutStrategy", "u-m15", 49.0, true, + {.ohlc = {series(20, 15)}, + .range = {{.RANGE_ATR_TICK_WINDOW = 5000, + .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 37}}, + .finalPnl = 2.0}), + }); + CHECK(live::selectTopWinners(body, kActive, 3).size() == 2); + } +} + +TEST_CASE("selectTopWinners carries RANGE_VARIABLES and defaults documents " + "that predate them", + "[liveWinners]") { + const std::string body = makeResponse({ + makeHit("r1", "EURUSD", "RandomStrategy", "u-range", 30.0, true, + {.range = {{.RANGE_ATR_TICK_WINDOW = 5000, + .RANGE_ATR_PERCENT = 40, + .RANGE_COUNT = 50}}, + .finalPnl = 1.0}), + makeHit("r2", "USDJPY", "RandomStrategy", "u-old", 30.0), + }); + + const auto winners = live::selectTopWinners(body, kActive, 3); + + REQUIRE(winners.size() == 2); + CHECK(winners[0].config.UUID == "u-range"); + REQUIRE(winners[0].config.RANGE_VARIABLES.size() == 1); + CHECK(winners[0].config.RANGE_VARIABLES[0].RANGE_ATR_TICK_WINDOW == 5000); + CHECK(winners[0].config.RANGE_VARIABLES[0].RANGE_ATR_PERCENT == 40); + CHECK(winners[0].config.RANGE_VARIABLES[0].RANGE_COUNT == 50); + // A pre-range document parses with the field empty, not a parse failure + // that would silently drop the winner. + CHECK(winners[1].config.UUID == "u-old"); + CHECK(winners[1].config.RANGE_VARIABLES.empty()); +} + +TEST_CASE("buildWinnersQueryBody terms the strategy name on the .keyword " + "subfield and matches the symbol on the analyzed field", + "[liveWinners]") { + // The strategy base field is analyzed text: a term against it silently + // matches nothing and live starts with zero winners. The symbol clause + // has the opposite shape — a match on the ANALYZED config.SYMBOLS, NOT a + // .keyword term, which would silently drop legacy multi-symbol documents + // ("EURUSD,USDJPY"). This pins both paths and the query's load-bearing + // clauses (not the page size — config). + const auto body = nlohmann::json::parse( + live::buildWinnersQueryBody(12.5, 5.0, 30.0, "FvgStrategy", + "EURUSD")); + + const auto& boolQuery = body.at("query").at("bool"); + const auto& filter = boolQuery.at("filter"); + REQUIRE(filter.is_array()); + + bool sawStrategyTerm = false; + bool sawScoreFloor = false; + bool sawDrawdownCeiling = false; + bool sawCalmarFloor = false; + bool sawSymbolMatch = false; + for (const auto& clause : filter) { + if (clause.contains("term") + && clause["term"].contains( + "config.STRATEGY.TRADING_VARIABLES.STRATEGY.keyword")) { + sawStrategyTerm = true; + CHECK(clause["term"] + ["config.STRATEGY.TRADING_VARIABLES.STRATEGY.keyword"] + == "FvgStrategy"); + } + if (clause.contains("range") + && clause["range"].contains("results.performanceScore")) { + sawScoreFloor = true; + CHECK(clause["range"]["results.performanceScore"]["gt"] == 12.5); + } + // The ceiling must be lte INSIDE the filter array: a should/penalty + // shape would let a spiky run through on score alone, and a missing + // maxDrawdownPercent field must fail the filter (fail-closed). + if (clause.contains("range") + && clause["range"].contains("results.maxDrawdownPercent")) { + sawDrawdownCeiling = true; + CHECK(clause["range"]["results.maxDrawdownPercent"]["lte"] + == 5.0); + } + // Same fail-closed shape for the calmar floor: gte inside filter. + if (clause.contains("range") + && clause["range"].contains("results.calmarScore")) { + sawCalmarFloor = true; + CHECK(clause["range"]["results.calmarScore"]["gte"] == 30.0); + } + if (clause.contains("match") + && clause["match"].contains("config.SYMBOLS")) { + sawSymbolMatch = true; + CHECK(clause["match"]["config.SYMBOLS"] == "EURUSD"); + } + } + CHECK(sawStrategyTerm); + CHECK(sawScoreFloor); + CHECK(sawDrawdownCeiling); + CHECK(sawCalmarFloor); + CHECK(sawSymbolMatch); + + const auto& mustNot = boolQuery.at("must_not"); + REQUIRE(mustNot.is_array()); + REQUIRE(mustNot.size() == 1); + CHECK(mustNot[0].contains("range")); + CHECK(mustNot[0]["range"].contains("config.OFFSET_MONTHS")); + + const auto& sort = body.at("sort"); + REQUIRE(sort.is_array()); + CHECK(sort[0] == nlohmann::json{{"results.performanceScore", "desc"}}); +} diff --git a/tests/marketHours.cpp b/tests/marketHours.cpp new file mode 100644 index 0000000..8cc54f2 --- /dev/null +++ b/tests/marketHours.cpp @@ -0,0 +1,212 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// market_hours — the peak-hours entry filter. Session mapping, the hand-rolled +// DST rules (pinned against known 2026 transition dates), the weekend block, +// and every session window's edges in both summer and winter time. + +#include + +#include + +import marketHours; +import symbolScale; + +namespace { + +using namespace std::chrono; +using market_hours::Session; + +// UTC instant on a calendar date at hh:mm. +constexpr system_clock::time_point at(const year_month_day date, const hours h, + const minutes m = minutes{0}) { + return sys_days{date} + h + m; +} + +// 2026 calendar facts the window tests lean on: 2026-07-15 and 2026-01-14 are +// Wednesdays (mid-week, no weekend rule in play); BST runs [Sun 2026-03-29, +// Sun 2026-10-25) and EDT runs [Sun 2026-03-08, Sun 2026-11-01). +constexpr year_month_day kSummerWed = 2026y / 7 / 15; +constexpr year_month_day kWinterWed = 2026y / 1 / 14; + +} // namespace + +TEST_CASE("sessionFor maps each symbol to its market session", "[marketHours]") { + CHECK(market_hours::sessionFor("USDJPY") == Session::Asia); + CHECK(market_hours::sessionFor("JPNIDXJPY") == Session::Asia); + CHECK(market_hours::sessionFor("AUDNZD") == Session::Asia); + CHECK(market_hours::sessionFor("EURUSD") == Session::Europe); + CHECK(market_hours::sessionFor("GBRIDXGBP") == Session::Europe); + CHECK(market_hours::sessionFor("USDSEK") == Session::Europe); + CHECK(market_hours::sessionFor("USA500IDXUSD") == Session::NewYork); + CHECK(market_hours::sessionFor("XAUUSD") == Session::NewYork); + CHECK(market_hours::sessionFor("BRENTCMDUSD") == Session::NewYork); + CHECK(market_hours::sessionFor("DOGEUSD") == Session::Unknown); + CHECK(market_hours::sessionFor("") == Session::Unknown); +} + +// The session table and the price table must cover the SAME symbol universe +// (same pin as the marketDefinitions cross-table test): a symbol priced but +// unmapped here would silently never trade under the filter, and a mapped +// symbol without a price scale could not be traded at all. +TEST_CASE("every priced symbol has a session and vice versa", "[marketHours]") { + CHECK(market_hours::kTable.size() == symbol_scale::kTable.size()); + for (const auto& entry : symbol_scale::kTable) { + INFO("symbol_scale entry missing a session: " << entry.symbol); + CHECK(market_hours::sessionFor(entry.symbol) != Session::Unknown); + } + for (const auto& entry : market_hours::kTable) { + INFO("market_hours entry missing a price scale: " << entry.symbol); + CHECK(symbol_scale::get(entry.symbol) != symbol_scale::kUnknown); + } +} + +TEST_CASE("DST helpers pin the 2026 transition dates", "[marketHours]") { + SECTION("London: [last Sunday of March, last Sunday of October)") { + CHECK_FALSE(market_hours::isLondonSummer(sys_days{2026y / 3 / 28})); + CHECK(market_hours::isLondonSummer(sys_days{2026y / 3 / 29})); + CHECK(market_hours::isLondonSummer(sys_days{2026y / 10 / 24})); + CHECK_FALSE(market_hours::isLondonSummer(sys_days{2026y / 10 / 25})); + } + + SECTION("New York: [second Sunday of March, first Sunday of November)") { + CHECK_FALSE(market_hours::isNewYorkSummer(sys_days{2026y / 3 / 7})); + CHECK(market_hours::isNewYorkSummer(sys_days{2026y / 3 / 8})); + CHECK(market_hours::isNewYorkSummer(sys_days{2026y / 10 / 31})); + CHECK_FALSE(market_hours::isNewYorkSummer(sys_days{2026y / 11 / 1})); + } +} + +// Times below sit inside the symbol's session window, so only the weekend +// rule varies. +TEST_CASE("tradePermitted blocks the weekend and its shoulders", "[marketHours]") { + SECTION("Friday cuts off at 16:00 UTC") { + // US summer window 13:30-16:30 straddles the cutoff. + CHECK(market_hours::tradePermitted( + "USA500IDXUSD", at(2026y / 7 / 17, hours{15}, minutes{59}))); + CHECK_FALSE(market_hours::tradePermitted( + "USA500IDXUSD", at(2026y / 7 / 17, hours{16}))); + } + + SECTION("all of Sunday is blocked") { + CHECK_FALSE(market_hours::tradePermitted( + "USDJPY", at(2026y / 7 / 19, hours{3}))); + } + + SECTION("Monday opens at 02:00 UTC") { + CHECK_FALSE(market_hours::tradePermitted( + "USDJPY", at(2026y / 7 / 20, hours{1}, minutes{59}))); + CHECK(market_hours::tradePermitted( + "USDJPY", at(2026y / 7 / 20, hours{2}))); + } +} + +TEST_CASE("Asia window is 00:00-06:00 UTC in every season", "[marketHours]") { + for (const year_month_day date : {kSummerWed, kWinterWed}) { + INFO("date " << static_cast(static_cast(date.month()))); + CHECK(market_hours::tradePermitted("USDJPY", at(date, hours{0}))); + CHECK(market_hours::tradePermitted("USDJPY", + at(date, hours{5}, minutes{59}))); + CHECK_FALSE(market_hours::tradePermitted("USDJPY", at(date, hours{6}))); + } +} + +TEST_CASE("Europe window tracks the London open across DST", "[marketHours]") { + SECTION("summer (BST): 07:00-10:00 UTC") { + CHECK_FALSE(market_hours::tradePermitted( + "EURUSD", at(kSummerWed, hours{6}, minutes{59}))); + CHECK(market_hours::tradePermitted("EURUSD", at(kSummerWed, hours{7}))); + CHECK(market_hours::tradePermitted( + "EURUSD", at(kSummerWed, hours{9}, minutes{59}))); + CHECK_FALSE(market_hours::tradePermitted("EURUSD", + at(kSummerWed, hours{10}))); + } + + SECTION("winter (GMT): 08:00-11:00 UTC") { + CHECK_FALSE(market_hours::tradePermitted( + "EURUSD", at(kWinterWed, hours{7}, minutes{59}))); + CHECK(market_hours::tradePermitted("EURUSD", at(kWinterWed, hours{8}))); + CHECK(market_hours::tradePermitted( + "EURUSD", at(kWinterWed, hours{10}, minutes{59}))); + CHECK_FALSE(market_hours::tradePermitted("EURUSD", + at(kWinterWed, hours{11}))); + } +} + +TEST_CASE("US window tracks the New York open across DST", "[marketHours]") { + SECTION("summer (EDT): 13:30-16:30 UTC") { + CHECK_FALSE(market_hours::tradePermitted( + "USA500IDXUSD", at(kSummerWed, hours{13}, minutes{29}))); + CHECK(market_hours::tradePermitted( + "USA500IDXUSD", at(kSummerWed, hours{13}, minutes{30}))); + CHECK(market_hours::tradePermitted( + "USA500IDXUSD", at(kSummerWed, hours{16}, minutes{29}))); + CHECK_FALSE(market_hours::tradePermitted( + "USA500IDXUSD", at(kSummerWed, hours{16}, minutes{30}))); + } + + SECTION("winter (EST): 14:30-17:30 UTC") { + CHECK_FALSE(market_hours::tradePermitted( + "USA500IDXUSD", at(kWinterWed, hours{14}, minutes{29}))); + CHECK(market_hours::tradePermitted( + "USA500IDXUSD", at(kWinterWed, hours{14}, minutes{30}))); + CHECK(market_hours::tradePermitted( + "USA500IDXUSD", at(kWinterWed, hours{17}, minutes{29}))); + CHECK_FALSE(market_hours::tradePermitted( + "USA500IDXUSD", at(kWinterWed, hours{17}, minutes{30}))); + } +} + +TEST_CASE("unknown symbols never trade under the filter", "[marketHours]") { + // A time inside every session's window and clear of the weekend rules. + CHECK_FALSE(market_hours::tradePermitted( + "DOGEUSD", at(kSummerWed, hours{14}, minutes{45}))); +} + +// The Friday before and the Monday after each transition must use that day's +// own open; the transition Sunday itself is weekend-blocked, which is what +// makes date-level DST granularity exact. +TEST_CASE("DST boundary weeks switch opens Friday-to-Monday", "[marketHours]") { + SECTION("London spring forward (Sun 2026-03-29)") { + CHECK_FALSE(market_hours::tradePermitted( + "EURUSD", at(2026y / 3 / 27, hours{7}, minutes{30}))); // GMT Friday + CHECK(market_hours::tradePermitted( + "EURUSD", at(2026y / 3 / 27, hours{8}, minutes{30}))); + CHECK_FALSE(market_hours::tradePermitted( + "EURUSD", at(2026y / 3 / 29, hours{8}, minutes{30}))); // Sunday + CHECK(market_hours::tradePermitted( + "EURUSD", at(2026y / 3 / 30, hours{7}, minutes{30}))); // BST Monday + CHECK_FALSE(market_hours::tradePermitted( + "EURUSD", at(2026y / 3 / 30, hours{10}, minutes{30}))); + } + + SECTION("London fall back (Sun 2026-10-25)") { + CHECK(market_hours::tradePermitted( + "EURUSD", at(2026y / 10 / 23, hours{7}, minutes{30}))); // BST Friday + CHECK_FALSE(market_hours::tradePermitted( + "EURUSD", at(2026y / 10 / 26, hours{7}, minutes{30}))); // GMT Monday + CHECK(market_hours::tradePermitted( + "EURUSD", at(2026y / 10 / 26, hours{8}, minutes{30}))); + } + + SECTION("New York spring forward (Sun 2026-03-08)") { + CHECK_FALSE(market_hours::tradePermitted( + "USA500IDXUSD", at(2026y / 3 / 6, hours{13}, minutes{45}))); // EST Fri + CHECK(market_hours::tradePermitted( + "USA500IDXUSD", at(2026y / 3 / 6, hours{14}, minutes{45}))); + CHECK(market_hours::tradePermitted( + "USA500IDXUSD", at(2026y / 3 / 9, hours{13}, minutes{45}))); // EDT Mon + } + + SECTION("New York fall back (Sun 2026-11-01)") { + CHECK(market_hours::tradePermitted( + "USA500IDXUSD", at(2026y / 10 / 30, hours{13}, minutes{45}))); // EDT Fri + CHECK_FALSE(market_hours::tradePermitted( + "USA500IDXUSD", at(2026y / 11 / 2, hours{13}, minutes{45}))); // EST Mon + CHECK(market_hours::tradePermitted( + "USA500IDXUSD", at(2026y / 11 / 2, hours{14}, minutes{45}))); + } +} diff --git a/tests/nyOpenRangeBreakout.cpp b/tests/nyOpenRangeBreakout.cpp new file mode 100644 index 0000000..40743d6 --- /dev/null +++ b/tests/nyOpenRangeBreakout.cpp @@ -0,0 +1,415 @@ +#include + +#include +#include +#include +#include // setenv — keep the bar store off QuestDB +#include +#include +#include +#include + +#include +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import nyOpenRangeBreakoutStrategy; +import barStore; // bars::BarStore — the strategy reads bars from it +import priceData; +import trade; +import tradeManager; + +namespace { + +using std::chrono::minutes; +using std::chrono::seconds; + +// Two UTC midnights, one per DST regime: 2026-01-05 is EST (NY opens 14:30 +// UTC), 2026-06-01 is EDT (NY opens 13:30 UTC). Both are Mondays, though the +// strategy itself never reads the weekday — that gate lives in the run loop +// (market_hours::tradePermitted). +const std::chrono::system_clock::time_point kWinterMidnight = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5}; +const std::chrono::system_clock::time_point kSummerMidnight = + std::chrono::sys_days{std::chrono::year{2026} / 6 / 1}; + +// Signal timeframe: 15m bars, window derived exactly like the sweep mapper — +// ceil((RANGE_HOURS x 60 + entry window) / minutes) + 2 bars, the ctor +// minimum. +tradingDefinitions::StrategyConfig makeConfig(int rangeHours = 4, + int bufferPips = 0, + int entryWindowMinutes = 120, + int maxTradeDurationMinutes = 0, + int ohlcMinutes = 15) { + tradingDefinitions::StrategyConfig config; + config.UUID = "test-ny-open-range"; + config.TRADING_VARIABLES.STRATEGY = "NyOpenRangeBreakoutStrategy"; + config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.TRADING_SIZE = 1; + const int ohlcCount = + (rangeHours * 60 + entryWindowMinutes + ohlcMinutes - 1) / ohlcMinutes + + 2; + config.OHLC_VARIABLES = { + tradingDefinitions::OHLCVariables{.OHLC_COUNT = ohlcCount, + .OHLC_MINUTES = ohlcMinutes}, + }; + config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES = + tradingDefinitions::NyOpenRangeBreakoutVariables{ + .RANGE_HOURS = rangeHours, + .BUFFER_PIPS = bufferPips, + .ENTRY_WINDOW_MINUTES = entryWindowMinutes, + .MAX_TRADE_DURATION_MINUTES = maxTradeDurationMinutes}; + return config; +} + +PriceData tickAt(std::chrono::system_clock::time_point base, + std::chrono::system_clock::duration offset, std::int32_t ask, + std::int32_t bid, const std::string& symbol = "EURUSD") { + return PriceData(ask, bid, base + offset, symbol); +} + +// The loop owner's role in miniature: register every configured timeframe. +bars::BarStore makeStore(const tradingDefinitions::StrategyConfig& config) { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + bars::BarStore store; + for (const auto& ohlc : config.OHLC_VARIABLES) { + store.registerSeries(minutes{ohlc.OHLC_MINUTES}, ohlc.OHLC_COUNT); + } + return store; +} + +// One tick in run-loop order: the store update first, then decide, then the +// management hook — runTicks feeds the shared bars BEFORE the entry gates, +// so decide() judges the tick against bar state that already includes it. +std::optional step(NyOpenRangeBreakoutStrategy& strategy, + TradeManager& tm, bars::BarStore& store, + const PriceData& tick) { + store.update(tick); + const auto signal = strategy.decide(tick, store); + strategy.during(tick, store, tm); + return signal; +} + +// The exact OHLC a crafted 15m bar should end up with. +struct BarShape { + std::int32_t o, h, l, c; +}; + +// Four asks inside one 15m bar, fed open/high/low/close within 45 seconds of +// `offset`. Bars are FIRST-TICK anchored (ohlcBuilder rolls on the first tick +// strictly more than one duration past the bar's start), so consecutive +// feeds must sit more than 15 minutes apart to land in separate bars. Every +// feed tick is pre-open under BOTH DST regimes (all before 13:30 UTC), so +// the clock gate guarantees no signal. +void feedBar(NyOpenRangeBreakoutStrategy& strategy, TradeManager& tm, + bars::BarStore& store, std::chrono::system_clock::time_point base, + minutes offset, const BarShape& bar, + const std::string& symbol = "EURUSD") { + const std::array, 4> ticks{{ + {seconds{0}, bar.o}, + {seconds{15}, bar.h}, + {seconds{30}, bar.l}, + {seconds{45}, bar.c}, + }}; + for (const auto& [tickOffset, ask] : ticks) { + CHECK_FALSE(step(strategy, tm, store, + tickAt(base, offset + tickOffset, ask, ask - 10, symbol)) + .has_value()); + } +} + +// Canonical pre-open fixture around `base` (a UTC midnight), built for +// RANGE_HOURS = 4 in BOTH DST regimes: the range window is [10:30, 14:30) +// UTC in winter and [09:30, 13:30) under EDT, so every range bar below sits +// inside both and the fixture's tradeable extremes are identical either way. +// 09:20 pre-range bar — satisfies the coverage gate in both regimes (its +// high is the `preRangeHigh` knob so a test can plant an extreme +// there; it must stay OUT of the range) +// 10:40/11:40/12:20 — the pre-open range: high 110050, low 109950 +// 13:00 — the last range bar; the decision tick itself rolls it closed +void feedPreOpenFixture(NyOpenRangeBreakoutStrategy& strategy, + TradeManager& tm, bars::BarStore& store, + std::chrono::system_clock::time_point base, + bool withCoverageBar = true, + std::int32_t preRangeHigh = 110005) { + if (withCoverageBar) { + feedBar(strategy, tm, store, base, minutes{560}, + {110000, preRangeHigh, 109995, 110000}); + } + // Without the coverage bar the series' oldest bar starts 10:40, strictly + // after the winter range start (10:30) — partial coverage. + feedBar(strategy, tm, store, base, minutes{640}, + {110000, 110020, 109980, 110010}); + feedBar(strategy, tm, store, base, minutes{700}, + {110010, 110050, 109990, 110030}); // range high 110050 + feedBar(strategy, tm, store, base, minutes{740}, + {110030, 110040, 109950, 109990}); // range low 109950 + feedBar(strategy, tm, store, base, minutes{780}, + {109990, 110030, 109970, 110000}); // 13:00 — last range bar +} + +} // namespace + +TEST_CASE("NyOpenRangeBreakoutStrategy trades the NY break of the pre-open range", + "[nyOpenRangeBreakout]") { + NyOpenRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedPreOpenFixture(strategy, tm, store, kWinterMidnight); + + // Winter: NY opens 14:30 UTC; the window (120m) runs to 16:30. + SECTION("bid above the range high inside the window: LONG") { + CHECK(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{875}, 110071, 110061)) == + Direction::LONG); + } + + SECTION("bid exactly on the range high: no signal (strictly above)") { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{875}, 110060, 110050)) + .has_value()); + } + + SECTION("ask below the range low inside the window: SHORT") { + CHECK(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{875}, 109940, 109930)) == + Direction::SHORT); + } +} + +TEST_CASE("NyOpenRangeBreakoutStrategy only fires inside the entry window", + "[nyOpenRangeBreakout]") { + NyOpenRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedPreOpenFixture(strategy, tm, store, kWinterMidnight); + + SECTION("a breakout before the winter open (13:35) is refused") { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{815}, 110071, 110061)) + .has_value()); + } + + SECTION("a breakout after the window closes (16:35) is refused") { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{995}, 110071, 110061)) + .has_value()); + } +} + +TEST_CASE("NyOpenRangeBreakoutStrategy follows the EDT NY open", + "[nyOpenRangeBreakout]") { + NyOpenRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedPreOpenFixture(strategy, tm, store, kSummerMidnight); + + // 13:35 is pre-open in winter (see above) but inside the window under + // EDT — the same offset flipping proves the DST rule is consulted. + CHECK(step(strategy, tm, store, + tickAt(kSummerMidnight, minutes{815}, 110071, 110061)) == + Direction::LONG); +} + +TEST_CASE("NyOpenRangeBreakoutStrategy pads the range with BUFFER_PIPS", + "[nyOpenRangeBreakout]") { + // 2 pips = 20 points on EURUSD: the padded high sits at 110070. + NyOpenRangeBreakoutStrategy strategy{makeConfig(4, 2)}; + TradeManager tm; + auto store = makeStore(makeConfig(4, 2)); + feedPreOpenFixture(strategy, tm, store, kWinterMidnight); + + SECTION("a poke through the raw high but not the padding: no signal") { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{875}, 110071, 110061)) + .has_value()); + } + + SECTION("clearing the padded high: LONG") { + CHECK(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{875}, 110081, 110071)) == + Direction::LONG); + } +} + +TEST_CASE("NyOpenRangeBreakoutStrategy keeps pre-range bars out of the range", + "[nyOpenRangeBreakout]") { + NyOpenRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + // The 09:20 bar spikes to 110200 — it starts before the range window, so + // the tradeable high must stay 110050. + feedPreOpenFixture(strategy, tm, store, kWinterMidnight, true, 110200); + + CHECK(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{875}, 110071, 110061)) == + Direction::LONG); +} + +TEST_CASE("NyOpenRangeBreakoutStrategy refuses a partially covered range", + "[nyOpenRangeBreakout]") { + NyOpenRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + // No coverage bar: the series' oldest bar starts 10:40, strictly after + // the winter range start (10:30), so the pre-open window is only + // partially represented — a fragment range (a run's first day) must not + // trade. + feedPreOpenFixture(strategy, tm, store, kWinterMidnight, false); + + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{875}, 110071, 110061)) + .has_value()); +} + +// The time-cap tests drive during() directly: its exit path is independent of +// bar state (no warm-up needed), and trades are opened straight on the +// TradeManager — same harness as the SessionRangeBreakoutStrategy cap tests. +TEST_CASE("NyOpenRangeBreakoutStrategy closes trades past the max duration via during()", + "[nyOpenRangeBreakout]") { + NyOpenRangeBreakoutStrategy strategy{makeConfig(4, 0, 120, 60)}; + TradeManager tm; + + SECTION("LONG past the cap closes at the bid") { + tm.openTrade(tickAt(kWinterMidnight, seconds{0}, 110000, 109998), 1, + Direction::LONG); + strategy.during( + tickAt(kWinterMidnight, minutes{60} + seconds{1}, 110050, 110040), bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110040); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("SHORT past the cap closes at the ask") { + tm.openTrade(tickAt(kWinterMidnight, seconds{0}, 110000, 109998), 1, + Direction::SHORT); + strategy.during( + tickAt(kWinterMidnight, minutes{60} + seconds{1}, 110050, 110040), bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110050); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("exactly at the cap stays open (strictly greater)") { + tm.openTrade(tickAt(kWinterMidnight, seconds{0}, 110000, 109998), 1, + Direction::LONG); + strategy.during(tickAt(kWinterMidnight, minutes{60}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } +} + +TEST_CASE("NyOpenRangeBreakoutStrategy max duration of zero disables the exit", + "[nyOpenRangeBreakout]") { + NyOpenRangeBreakoutStrategy strategy{makeConfig(4, 0, 120, 0)}; + TradeManager tm; + + tm.openTrade(tickAt(kWinterMidnight, seconds{0}, 110000, 109998), 1, + Direction::LONG); + strategy.during(tickAt(kWinterMidnight, minutes{600}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); +} + +TEST_CASE("NyOpenRangeBreakoutStrategy rejects malformed configuration", + "[nyOpenRangeBreakout]") { + SECTION("no OHLC timeframe") { + auto config = makeConfig(); + config.OHLC_VARIABLES.clear(); + CHECK_THROWS_AS(NyOpenRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("missing NY_OPEN_RANGE_BREAKOUT_VARIABLES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES = std::nullopt; + CHECK_THROWS_AS(NyOpenRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("RANGE_HOURS below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES->RANGE_HOURS = 0; + CHECK_THROWS_AS(NyOpenRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("RANGE_HOURS reaching past the previous UTC midnight") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES->RANGE_HOURS = 14; + // Keep the window check satisfied so the depth cap is what throws. + config.OHLC_VARIABLES[0].OHLC_COUNT = 200; + CHECK_THROWS_AS(NyOpenRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("ENTRY_WINDOW_MINUTES below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES + ->ENTRY_WINDOW_MINUTES = 0; + CHECK_THROWS_AS(NyOpenRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("negative BUFFER_PIPS") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES->BUFFER_PIPS = -1; + CHECK_THROWS_AS(NyOpenRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("window one bar short of range start -> entry cutoff coverage") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_COUNT -= 1; + CHECK_THROWS_AS(NyOpenRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("OHLC_MINUTES below 1") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_MINUTES = 0; + CHECK_THROWS_AS(NyOpenRangeBreakoutStrategy{config}, std::invalid_argument); + } +} + +TEST_CASE("StrategyVariables round-trips NY_OPEN_RANGE_BREAKOUT_VARIABLES through JSON", + "[nyOpenRangeBreakout]") { + SECTION("present group survives the round-trip") { + tradingDefinitions::StrategyVariables vars; + vars.NY_OPEN_RANGE_BREAKOUT_VARIABLES = + tradingDefinitions::NyOpenRangeBreakoutVariables{ + .RANGE_HOURS = 8, + .BUFFER_PIPS = 3, + .ENTRY_WINDOW_MINUTES = 90, + .MAX_TRADE_DURATION_MINUTES = 45}; + + const nlohmann::json j = vars; + const auto back = j.get(); + + REQUIRE(back.NY_OPEN_RANGE_BREAKOUT_VARIABLES.has_value()); + CHECK(back.NY_OPEN_RANGE_BREAKOUT_VARIABLES->RANGE_HOURS == 8); + CHECK(back.NY_OPEN_RANGE_BREAKOUT_VARIABLES->BUFFER_PIPS == 3); + CHECK(back.NY_OPEN_RANGE_BREAKOUT_VARIABLES->ENTRY_WINDOW_MINUTES == 90); + CHECK(back.NY_OPEN_RANGE_BREAKOUT_VARIABLES->MAX_TRADE_DURATION_MINUTES == 45); + } + + SECTION("absent MAX_TRADE_DURATION_MINUTES parses as disabled") { + // Models a winner config persisted before the field existed: the + // WITH_DEFAULT codec must fall back to 0, not throw. + const auto vars = + nlohmann::json::parse( + R"({"NY_OPEN_RANGE_BREAKOUT_VARIABLES":{"RANGE_HOURS":4,)" + R"("BUFFER_PIPS":2,"ENTRY_WINDOW_MINUTES":60}})") + .get(); + + REQUIRE(vars.NY_OPEN_RANGE_BREAKOUT_VARIABLES.has_value()); + CHECK(vars.NY_OPEN_RANGE_BREAKOUT_VARIABLES->ENTRY_WINDOW_MINUTES == 60); + CHECK(vars.NY_OPEN_RANGE_BREAKOUT_VARIABLES->MAX_TRADE_DURATION_MINUTES == 0); + } + + SECTION("absent group serialises as null and stays absent") { + const tradingDefinitions::StrategyVariables vars; + const nlohmann::json j = vars; + + CHECK(j.at("NY_OPEN_RANGE_BREAKOUT_VARIABLES").is_null()); + CHECK_FALSE(j.get() + .NY_OPEN_RANGE_BREAKOUT_VARIABLES.has_value()); + } +} diff --git a/tests/ohlc.cpp b/tests/ohlc.cpp index 60add33..c9368cf 100644 --- a/tests/ohlc.cpp +++ b/tests/ohlc.cpp @@ -1,8 +1,12 @@ #include +#include #include #include +#include // setenv/unsetenv (POSIX) #include +#include +#include #include import ohlcBuilder; @@ -24,6 +28,17 @@ PriceData tickAt(std::chrono::system_clock::duration offset) { return PriceData(110002, 110001, t0 + offset, "EURUSD"); } +// The engine's prepopulate gate is ON by default (see ohlcBuilder), but unit +// tests must stay hermetic: any test that hands calculateOHLC an empty bar +// list and a prepopulate count would otherwise fire real QuestDB queries +// (the breakout-strategy and live-runner tests all do). Force the gate off +// for the whole binary before main; the gate tests below flip it explicitly +// and restore "0" when done. +[[maybe_unused]] const bool kPrepopulateForcedOff = [] { + setenv("OHLC_PREPOPULATE", "0", 1); + return true; +}(); + } // namespace TEST_CASE("OhlcObject defaults mirror the C# sentinels", "[ohlc]") { @@ -108,13 +123,16 @@ TEST_CASE("a tick past the duration completes the bar and opens a new one", "[oh REQUIRE(bars.size() == 2); - // Completed bar keeps its extremes; its close is overwritten with the new - // bucket's first price (ported C# behaviour, bars join up). + // Completed bar keeps its extremes AND its own bucket's last price as the + // close — the rollover tick belongs entirely to the new bar. (The old + // C#-ported close-join overwrote it with the next bucket's first price, + // which could push close outside [low, high] on a gap and disagreed with + // the prepopulate query's last(ask) convention.) CHECK(bars[0].complete); CHECK(bars[0].open == 110001); CHECK(bars[0].high == 110010); CHECK(bars[0].low == 110001); - CHECK(bars[0].close == 110020); + CHECK(bars[0].close == 110010); // New bar is seeded entirely from the rollover tick. CHECK_FALSE(bars[1].complete); @@ -147,3 +165,72 @@ TEST_CASE("a stream spanning several buckets yields one bar per bucket", "[ohlc] CHECK(bars[2].low == 110015); CHECK(bars[2].close == 110015); } + +// A zero/negative duration would roll a new bar on every tick — one bar per +// tick over months of ticks is an OOM, so the builder rejects it up front. +TEST_CASE("non-positive bar duration throws instead of exploding", "[ohlc]") { + std::vector bars; + CHECK_THROWS_AS( + ohlc::calculateOHLC(tickAt(minutes{0}), 110001, minutes{0}, bars), + std::invalid_argument); + CHECK_THROWS_AS( + ohlc::calculateOHLC(tickAt(minutes{0}), 110001, minutes{-5}, bars), + std::invalid_argument); + CHECK(bars.empty()); +} + +TEST_CASE("prepopulateQuery pins the QuestDB SQL shape", "[ohlc]") { + // Bounds are hardcoded, not recomputed with the same chrono ops: t0 is + // 2026-01-05T09:00Z = 1767603600000000us, and 5m x 24 bars -> a 16-day + // lookback (ceil(120/1440)*6 + 10) = 1382400000000us earlier. + CHECK(ohlc::prepopulateQuery("EURUSD", t0, minutes{5}, 24) == + "SELECT timestamp, open, high, low, close FROM (" + "SELECT timestamp, first(ask) AS open, max(ask) AS high, min(ask) AS low, " + "last(ask) AS close, count() AS ticks FROM 'EURUSD' " + "WHERE timestamp >= cast(1766221200000000L AS timestamp) " + "AND timestamp < cast(1767603600000000L AS timestamp) " + "SAMPLE BY 5m ALIGN TO CALENDAR" + ") WHERE ticks >= 10 ORDER BY timestamp DESC LIMIT 24"); +} + +TEST_CASE("prepopulation gate off falls back to the cold tick seed", "[ohlc]") { + // Explicit "0", not unsetenv: the gate defaults ON, so unset means on. + setenv("OHLC_PREPOPULATE", "0", 1); + std::vector bars; + + ohlc::calculateOHLC(tickAt(minutes{0}), 110001, minutes{5}, bars, 3); + + REQUIRE(bars.size() == 1); + CHECK(bars[0].date == t0); + CHECK(bars[0].open == 110001); + CHECK(bars[0].close == 110001); + CHECK_FALSE(bars[0].complete); +} + +TEST_CASE("unknown symbol yields empty without touching the DB", "[ohlc]") { + // The symbol guard precedes any connection, so this passes gate-on with no + // QuestDB running (and doubles as the SQL-injection whitelist check). + setenv("OHLC_PREPOPULATE", "1", 1); + const auto bars = ohlc::prepopulateOHLC("NOPE", t0, minutes{5}, 24); + setenv("OHLC_PREPOPULATE", "0", 1); // back to the binary's hermetic state + + CHECK(bars.empty()); +} + +// Hidden ([.]) — needs a live QuestDB with EURUSD ticks. Run explicitly: +// ./build/tests/unit_tests "[dblive]" +TEST_CASE("prepopulateOHLC fetches ascending bars from a live QuestDB", "[.][dblive]") { + setenv("OHLC_PREPOPULATE", "1", 1); + const auto bars = ohlc::prepopulateOHLC("EURUSD", std::chrono::system_clock::now(), + minutes{5}, 24); + setenv("OHLC_PREPOPULATE", "0", 1); // back to the binary's hermetic state + + REQUIRE_FALSE(bars.empty()); + CHECK(bars.size() <= 24); + CHECK(std::ranges::is_sorted(bars, {}, &OhlcObject::date)); + for (std::size_t i = 0; i < bars.size(); ++i) { + CHECK(bars[i].complete == (i + 1 < bars.size())); // only back() in progress + CHECK(bars[i].low <= bars[i].high); + CHECK(bars[i].low > 0); // scaled INT prices parsed, not garbage + } +} diff --git a/tests/ohlcBreakout.cpp b/tests/ohlcBreakout.cpp index b01626c..17df7dc 100644 --- a/tests/ohlcBreakout.cpp +++ b/tests/ohlcBreakout.cpp @@ -2,6 +2,7 @@ #include #include +#include // setenv — keep the bar store off QuestDB #include #include #include @@ -11,6 +12,7 @@ #include "shared/tradingDefinitions/strategyConfig.hpp" import ohlcBreakoutStrategy; +import barStore; // bars::BarStore — the strategy reads bars from it import priceData; import trade; import tradeManager; @@ -23,23 +25,29 @@ using std::chrono::seconds; const std::chrono::system_clock::time_point t0 = std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; -// Breakout timeframe: 3 one-minute candles; trend timeframe: 4 one-minute -// candles (EMA period = 4/2 = 2). Ticks in these tests are spaced 2 minutes -// apart so every tick rolls a fresh bar — warm-up completes after 4 ticks and -// tick 5 is the first that can signal. -tradingDefinitions::StrategyConfig makeConfig(int bufferPips) { +// Breakout timeframe: 3 one-minute candles; trend timeframe: 4 candles of +// `trendMinutes` (EMA period = 4/2 = 2). Ticks in these tests are spaced 2 +// minutes apart so every tick rolls a fresh 1m bar. The store updates BEFORE +// decide(), so the 4th tick is already warm — the fixtures are chosen so +// none of the warm-up ticks signals. +tradingDefinitions::StrategyConfig makeConfig(int bufferPips, + int maxTradeDurationMinutes = 0, + int trendMinutes = 1) { tradingDefinitions::StrategyConfig config; config.UUID = "test-ohlc-breakout"; config.TRADING_VARIABLES.STRATEGY = "OhlcBreakoutStrategy"; - config.TRADING_VARIABLES.STOP_DISTANCE_IN_PIPS = 10; - config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_PIPS = 10; + config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR = 10; config.TRADING_VARIABLES.TRADING_SIZE = 1; config.OHLC_VARIABLES = { tradingDefinitions::OHLCVariables{.OHLC_COUNT = 3, .OHLC_MINUTES = 1}, // breakout - tradingDefinitions::OHLCVariables{.OHLC_COUNT = 4, .OHLC_MINUTES = 1}, // trend + tradingDefinitions::OHLCVariables{.OHLC_COUNT = 4, + .OHLC_MINUTES = trendMinutes}, // trend }; config.STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES = - tradingDefinitions::OHLCBreakoutVariables{.BUFFER_PIPS = bufferPips}; + tradingDefinitions::OHLCBreakoutVariables{ + .BUFFER_PIPS = bufferPips, + .MAX_TRADE_DURATION_MINUTES = maxTradeDurationMinutes}; return config; } @@ -48,12 +56,25 @@ PriceData tickAt(std::chrono::system_clock::duration offset, std::int32_t ask, return PriceData(ask, bid, t0 + offset, symbol); } -// One tick in run-loop order: decide first, then during (runTicks calls -// decide before the management hook on each tick). +// The loop owner's role in miniature: one store carrying both of the config's +// timeframes. When both are 1 minute they dedup into one shared series with +// the larger window (4); each consumer reads its own tail, like production. +bars::BarStore makeStore(int trendMinutes = 1) { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + bars::BarStore store; + store.registerSeries(minutes{1}, 3); // breakout timeframe + store.registerSeries(minutes{trendMinutes}, 4); // trend timeframe + return store; +} + +// One tick in run-loop order: the store update first, then decide, then the +// management hook — runTicks feeds the shared bars BEFORE the entry gates, +// so decide() judges the tick against bar state that already includes it. std::optional step(OhlcBreakoutStrategy& strategy, TradeManager& tm, - const PriceData& tick) { - const auto signal = strategy.decide(tick); - strategy.during(tick, tm); + bars::BarStore& store, const PriceData& tick) { + store.update(tick); + const auto signal = strategy.decide(tick, store); + strategy.during(tick, store, tm); return signal; } @@ -61,11 +82,11 @@ std::optional step(OhlcBreakoutStrategy& strategy, TradeManager& tm, // them may signal. Bars are built from the ask; bid is ask minus a 2-point // spread and irrelevant until the signal tick. void warmUp(OhlcBreakoutStrategy& strategy, TradeManager& tm, - const std::vector& asks, + bars::BarStore& store, const std::vector& asks, const std::string& symbol = "EURUSD") { for (std::size_t i = 0; i < asks.size(); ++i) { const auto tick = tickAt(minutes{2 * static_cast(i)}, asks[i], asks[i] - 2, symbol); - CHECK_FALSE(step(strategy, tm, tick).has_value()); + CHECK_FALSE(step(strategy, tm, store, tick).has_value()); } } @@ -84,62 +105,93 @@ TEST_CASE("OhlcBreakoutStrategy signals LONG on a buffered breakout in an uptren "[ohlcBreakout]") { OhlcBreakoutStrategy strategy{makeConfig(2)}; // buffer = 2 pips = 20 points TradeManager tm; - warmUp(strategy, tm, kRisingAsks); + auto store = makeStore(); + warmUp(strategy, tm, store, kRisingAsks); - // Closed-candle highest high is 110020, buffer 20 -> level 110040. - const auto signal = step(strategy, tm, tickAt(minutes{8}, 110100, 110090)); + // The signal tick rolls its own (in-progress) bar first, so the closed + // breakout candles are 110020/110030: high 110030, buffer 20 -> 110050. + const auto signal = + step(strategy, tm, store, tickAt(minutes{8}, 110100, 110090)); CHECK(signal == Direction::LONG); } TEST_CASE("OhlcBreakoutStrategy respects the pip buffer", "[ohlcBreakout]") { OhlcBreakoutStrategy strategy{makeConfig(2)}; TradeManager tm; - warmUp(strategy, tm, kRisingAsks); + auto store = makeStore(); + warmUp(strategy, tm, store, kRisingAsks); + // Closed candles at the judged tick are 110020/110030: high 110030, + // buffer 20 -> buffered level 110050. SECTION("above the high but inside the buffer: no signal") { - // bid 110035 clears the 110020 high but not the 110040 buffered level. - CHECK_FALSE(step(strategy, tm, tickAt(minutes{8}, 110045, 110035)).has_value()); + // bid 110035 clears the 110030 high but not the 110050 buffered level. + CHECK_FALSE( + step(strategy, tm, store, tickAt(minutes{8}, 110045, 110035)).has_value()); } SECTION("exactly on the buffered level: no signal (strictly greater)") { - CHECK_FALSE(step(strategy, tm, tickAt(minutes{8}, 110050, 110040)).has_value()); + CHECK_FALSE( + step(strategy, tm, store, tickAt(minutes{8}, 110060, 110050)).has_value()); } SECTION("one point beyond the buffered level: LONG") { - CHECK(step(strategy, tm, tickAt(minutes{8}, 110051, 110041)) == Direction::LONG); + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110061, 110051)) == + Direction::LONG); } } +// The current tick's own close counts as trend evidence now (the store +// updates pre-decide), so blocking a counter-trend breakout requires +// genuinely separated timeframes: a steep multi-hour fall holds the 60m EMA +// far above a small local pop that clears the 1m range. TEST_CASE("OhlcBreakoutStrategy trend filter blocks counter-trend breakouts", "[ohlcBreakout]") { - OhlcBreakoutStrategy strategy{makeConfig(2)}; + OhlcBreakoutStrategy strategy{makeConfig(2, 0, 60)}; // trend = 60m bars TradeManager tm; - warmUp(strategy, tm, kFallingAsks); - - // bid 110200 is far above the closed-candle high (110080) + buffer, but - // the macro trend is down (last close 110040 < EMA ~110043) — no LONG. - CHECK_FALSE(step(strategy, tm, tickAt(minutes{8}, 110210, 110200)).has_value()); + auto store = makeStore(60); + + // Four 60m trend bars stepping down 1000 points (each tick also rolls a + // 1m breakout bar). Signals during the descent are not under test. + step(strategy, tm, store, tickAt(minutes{0}, 110000, 109998)); + step(strategy, tm, store, tickAt(minutes{61}, 109000, 108998)); + step(strategy, tm, store, tickAt(minutes{122}, 108000, 107998)); + step(strategy, tm, store, tickAt(minutes{183}, 107000, 106998)); + + // A tight local range at the bottom, inside the 4th 60m bar. + step(strategy, tm, store, tickAt(minutes{185}, 107010, 107008)); + step(strategy, tm, store, tickAt(minutes{187}, 107020, 107018)); + + // bid 107190 clears the local closed-candle high (107020) + buffer by a + // mile, but the 60m EMA (~107633) is still far overhead — no LONG (and + // nowhere near the local low, so no SHORT either). + CHECK_FALSE( + step(strategy, tm, store, tickAt(minutes{189}, 107200, 107190)).has_value()); } TEST_CASE("OhlcBreakoutStrategy signals SHORT on a buffered breakdown in a downtrend", "[ohlcBreakout]") { OhlcBreakoutStrategy strategy{makeConfig(2)}; TradeManager tm; - warmUp(strategy, tm, kFallingAsks); + auto store = makeStore(); + warmUp(strategy, tm, store, kFallingAsks); - // Closed-candle lowest low is 110060, buffer 20 -> level 110040. + // Closed candles at the judged tick are 110060/110040: lowest low + // 110040, buffer 20 -> buffered level 110020. SECTION("ask below the buffered low: SHORT") { - CHECK(step(strategy, tm, tickAt(minutes{8}, 110030, 110020)) == Direction::SHORT); + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110019, 110009)) == + Direction::SHORT); } SECTION("exactly on the buffered level: no signal (strictly less)") { - CHECK_FALSE(step(strategy, tm, tickAt(minutes{8}, 110040, 110030)).has_value()); + CHECK_FALSE( + step(strategy, tm, store, tickAt(minutes{8}, 110020, 110010)).has_value()); } } TEST_CASE("OhlcBreakoutStrategy keeps per-symbol state isolated", "[ohlcBreakout]") { OhlcBreakoutStrategy strategy{makeConfig(2)}; TradeManager tm; + auto store = makeStore(); // Interleave a rising EURUSD with a falling AUDUSD at a very different // price level. If either symbol's ticks leaked into the other's bars, the @@ -147,15 +199,19 @@ TEST_CASE("OhlcBreakoutStrategy keeps per-symbol state isolated", "[ohlcBreakout const std::vector audAsks{65100, 65080, 65060, 65040}; for (std::size_t i = 0; i < 4; ++i) { const auto offset = minutes{2 * static_cast(i)}; - CHECK_FALSE(step(strategy, tm, + CHECK_FALSE(step(strategy, tm, store, tickAt(offset, kRisingAsks[i], kRisingAsks[i] - 2)).has_value()); - CHECK_FALSE(step(strategy, tm, + CHECK_FALSE(step(strategy, tm, store, tickAt(offset + seconds{30}, audAsks[i], audAsks[i] - 2, "AUDUSD")) .has_value()); } - CHECK(step(strategy, tm, tickAt(minutes{8}, 110100, 110090)) == Direction::LONG); - CHECK(step(strategy, tm, tickAt(minutes{8} + seconds{30}, 65030, 65020, "AUDUSD")) == + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 110100, 110090)) == + Direction::LONG); + // AUDUSD closed candles at the judged tick are 65060/65040: low 65040, + // buffer 20 -> level 65020, so the ask must undercut 65020. + CHECK(step(strategy, tm, store, + tickAt(minutes{8} + seconds{30}, 65010, 65000, "AUDUSD")) == Direction::SHORT); } @@ -185,18 +241,89 @@ TEST_CASE("OhlcBreakoutStrategy rejects malformed configuration", "[ohlcBreakout } } +// The time-cap tests drive during() directly: its exit path is independent of +// bar state (no warm-up needed), and trades are opened straight on the +// TradeManager — exactly how the live book seeds them (openTrade from a +// synthetic tick stamped with the broker's openedAt). +TEST_CASE("OhlcBreakoutStrategy closes trades past the max duration via during()", + "[ohlcBreakout]") { + OhlcBreakoutStrategy strategy{makeConfig(0, 60)}; + TradeManager tm; + + SECTION("LONG past the cap closes at the bid") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::LONG); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050, 110040), bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110040); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("SHORT past the cap closes at the ask") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::SHORT); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050, 110040), bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110050); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("exactly at the cap stays open (strictly greater)") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::LONG); + strategy.during(tickAt(minutes{60}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } + + SECTION("another symbol's tick never closes it") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::LONG); + strategy.during(tickAt(minutes{120}, 65030, 65020, "AUDUSD"), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } +} + +TEST_CASE("OhlcBreakoutStrategy max duration of zero disables the exit", + "[ohlcBreakout]") { + OhlcBreakoutStrategy strategy{makeConfig(0, 0)}; + TradeManager tm; + + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::LONG); + strategy.during(tickAt(minutes{600}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); +} + TEST_CASE("StrategyVariables round-trips OHLC_BREAKOUT_VARIABLES through JSON", "[ohlcBreakout]") { SECTION("present group survives the round-trip") { tradingDefinitions::StrategyVariables vars; - vars.OHLC_BREAKOUT_VARIABLES = - tradingDefinitions::OHLCBreakoutVariables{.BUFFER_PIPS = 7}; + vars.OHLC_BREAKOUT_VARIABLES = tradingDefinitions::OHLCBreakoutVariables{ + .BUFFER_PIPS = 7, .MAX_TRADE_DURATION_MINUTES = 45}; const nlohmann::json j = vars; const auto back = j.get(); REQUIRE(back.OHLC_BREAKOUT_VARIABLES.has_value()); CHECK(back.OHLC_BREAKOUT_VARIABLES->BUFFER_PIPS == 7); + CHECK(back.OHLC_BREAKOUT_VARIABLES->MAX_TRADE_DURATION_MINUTES == 45); + } + + SECTION("absent MAX_TRADE_DURATION_MINUTES parses as disabled") { + // Models a winner config persisted before the field existed: the + // WITH_DEFAULT codec must fall back to 0 (disabled), not throw. + const auto vars = + nlohmann::json::parse( + R"({"OHLC_RSI_VARIABLES":null,)" + R"("OHLC_BREAKOUT_VARIABLES":{"BUFFER_PIPS":7}})") + .get(); + + REQUIRE(vars.OHLC_BREAKOUT_VARIABLES.has_value()); + CHECK(vars.OHLC_BREAKOUT_VARIABLES->BUFFER_PIPS == 7); + CHECK(vars.OHLC_BREAKOUT_VARIABLES->MAX_TRADE_DURATION_MINUTES == 0); } SECTION("absent group serialises as null and stays absent") { diff --git a/tests/orderChannel.cpp b/tests/orderChannel.cpp new file mode 100644 index 0000000..6885bc9 --- /dev/null +++ b/tests/orderChannel.cpp @@ -0,0 +1,625 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// OrderChannel flow tests: the C# Request() semantics around one placement +// (market lookup, size modifier, in-flight lock extension, and the outcome +// bookkeeping — Accepted -> book under IG's dealReference, Rejected -> +// release, Failed -> the 2-minute brake) plus the ClosePosition flow +// (direction flip, missing-dealId guard, the recently-closed window, and +// the remove-on-gone branch). Broker and Redis are injected seams +// (PlaceOrder/PlaceClose, Hooks, MarketLookup), so no server is involved, +// and the market table itself is stubbed — these tests pin the machinery, +// not the currently-listed markets. + +#include + +#include +#include +#include +#include +#include +#include +#include + +import igMarkets; +import liveStrategyRunner; +import marketDefinitions; +import orderChannel; +import orderRequest; +import symbolScale; +import trade; + +namespace { + +constexpr std::chrono::system_clock::time_point kTick{ + std::chrono::microseconds{1'719'360'000'000'000LL}}; + +// The mini epic is what must reach the broker; the CFD epic is a decoy the +// channel must NOT pick up. tradeSizeModifier 0.5 exercises the C# +// TradeSizeModifier path. +constexpr live::MarketDefinition kEurUsd{ + .symbol = "EURUSD", + .igMarketId = "EURUSD", + .epicCfd = "TEST.EPIC.CFD.IP", + .epicMini = "TEST.EPIC.MINI.IP", + .currency = "USD", + .polygonIdentifier = "", + .polygonScale = 0, + .type = live::MarketType::Forex, + .tradeSizeModifier = 0.5, +}; + +const live::MarketLookup kTestLookup = + [](std::string_view symbol) -> const live::MarketDefinition* { + return symbol == "EURUSD" ? &kEurUsd : nullptr; +}; + +// Records every hook invocation, in order, with the arguments the channel +// passed. All hooks report success unless the test flips the flags. +struct HookRecorder { + struct Extend { + std::string uuid; + std::string direction; + std::chrono::seconds ttl; + }; + std::vector sequence; // "cluster"/"extend"/"release"/... + std::vector extends; + std::vector> releases; // uuid, dir + std::vector> saves; // ref, payload + std::vector> adds; // uuid, ref + std::vector> receipts; // ref, payload + std::vector> removes; // uuid, ref + std::vector> clusterChecks; // sym, strat + bool saveOk = true; + bool clusterAllow = true; + + live::OrderChannel::Hooks hooks() { + return live::OrderChannel::Hooks{ + .clusterBlocked = [this](const std::string& symbol, + const std::string& strategyName) { + sequence.emplace_back("cluster"); + clusterChecks.emplace_back(symbol, strategyName); + return !clusterAllow; + }, + .extendLock = [this](const std::string& uuid, + const std::string& direction, + const std::chrono::seconds ttl) { + sequence.emplace_back("extend"); + extends.push_back({uuid, direction, ttl}); + return true; + }, + .releaseLock = [this](const std::string& uuid, + const std::string& direction) { + sequence.emplace_back("release"); + releases.emplace_back(uuid, direction); + return true; + }, + .savePosition = [this](const std::string& reference, + const std::string& payload) { + sequence.emplace_back("save"); + saves.emplace_back(reference, payload); + return saveOk; + }, + .addPosition = [this](const std::string& uuid, + const std::string& reference) { + sequence.emplace_back("add"); + adds.emplace_back(uuid, reference); + return true; + }, + .saveDealReceipt = [this](const std::string& reference, + const std::string& symbol, + const std::string& payload) { + sequence.emplace_back("receipt"); + receipts.emplace_back(reference + "#" + symbol, payload); + return true; + }, + .removePosition = [this](const std::string& uuid, + const std::string& reference) { + sequence.emplace_back("remove"); + removes.emplace_back(uuid, reference); + return true; + }, + }; + } +}; + +live::OrderRequest makeRequest(const Direction direction = Direction::LONG, + const std::string& symbol = "EURUSD") { + const auto request = live::makeOrderRequest(live::OrderIntent{ + .strategyName = "StubStrategy", + .strategyUuid = "u-eur", + .symbol = "EURUSD", // must exist in symbolScale to build + .direction = direction, + .size = 3, + .stopDistancePips = 25, + .limitDistancePips = 50, + .bid = 110000, + .ask = 110002, + .timestamp = kTick, + }); + REQUIRE(request.has_value()); + live::OrderRequest result = *request; + result.symbol = symbol; // fake-symbol tests exercise the market lookup + return result; +} + +live::CloseRequest makeCloseRequest(const std::string& dealId = "DEAL-77") { + return live::CloseRequest{ + .strategyUuid = "u-eur", + .strategyName = "StubStrategy", + .symbol = "EURUSD", + .direction = Direction::LONG, // open position was LONG (broker BUY) + .size = 1.5, + .dealId = dealId, + .dealReference = "ueur-L1719360000000", + }; +} + +const ig::PlaceClose kNoClose = [](const ig::TradeCloseObj&, + const ig::OrderContext&) { + return ig::CloseResult{.status = ig::CloseStatus::Failed, + .reason = "unexpected close in this test"}; +}; + +constexpr std::chrono::seconds kInFlightTtl{7}; +constexpr std::chrono::seconds kFailureTtl{9}; +constexpr std::chrono::seconds kClosedTtl{300}; + +} // namespace + +TEST_CASE("an accepted open books the deal under IG's reference and keeps " + "the lock", + "[orderChannel]") { + HookRecorder recorder; + std::vector placed; + std::vector contexts; + live::OrderChannel channel( + kTestLookup, + [&](const ig::TradeOpenObj& order, const ig::OrderContext& context) { + placed.push_back(order); + contexts.push_back(context); + return ig::OpenResult{.status = ig::OpenStatus::Accepted, + .dealReference = "IGREF-1"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK(channel.request(makeRequest())); + + // The cluster gate ran first, the lock was extended (in-flight TTL) + // BEFORE the broker was called, then the deal was saved (PO#), listed + // (PL#) and receipted — and never released. + REQUIRE(recorder.sequence + == std::vector{"cluster", "extend", "save", "add", + "receipt"}); + REQUIRE(recorder.extends.size() == 1); + CHECK(recorder.extends[0].uuid == "u-eur"); + CHECK(recorder.extends[0].direction == "LONG"); + CHECK(recorder.extends[0].ttl == kInFlightTtl); + + // The TradeOpenObj carries the market definition and the modified size; + // the context carries what the request gate and audit trail need. The + // epic must be the MINI contract, never the CFD one. + REQUIRE(placed.size() == 1); + CHECK(placed[0].epic == "TEST.EPIC.MINI.IP"); + CHECK(placed[0].currencyCode == "USD"); + CHECK(placed[0].direction == "BUY"); // engine LONG -> broker BUY + CHECK(placed[0].size == 1.5); // TRADING_SIZE 3 * sizeModifier 0.5 + CHECK(placed[0].expiry == "-"); + CHECK(placed[0].orderType == "MARKET"); + CHECK(placed[0].forceOpen); + CHECK_FALSE(placed[0].guaranteedStop); + CHECK(placed[0].stopDistance == 25); + CHECK(placed[0].limitDistance == 50); + CHECK(placed[0].dealReference == makeRequest().dealReference); + REQUIRE(contexts.size() == 1); + CHECK(contexts[0].strategyUuid == "u-eur"); + CHECK(contexts[0].symbol == "EURUSD"); + CHECK(contexts[0].openDirection == "BUY"); + + // Booked under the BROKER's echoed reference, not the one we minted. + REQUIRE(recorder.saves.size() == 1); + CHECK(recorder.saves[0].first == "IGREF-1"); + REQUIRE(recorder.adds.size() == 1); + CHECK(recorder.adds[0] + == std::pair{"u-eur", "IGREF-1"}); + REQUIRE(recorder.receipts.size() == 1); + CHECK(recorder.receipts[0].first == "IGREF-1#EURUSD"); +} + +TEST_CASE("an accepted open with no echoed reference books under ours", + "[orderChannel]") { + HookRecorder recorder; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{.status = ig::OpenStatus::Accepted}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + const live::OrderRequest request = makeRequest(); + CHECK(channel.request(request)); + REQUIRE(recorder.saves.size() == 1); + CHECK(recorder.saves[0].first == request.dealReference); +} + +TEST_CASE("a SHORT request trades SELL but locks under SHORT", + "[orderChannel]") { + HookRecorder recorder; + std::optional brokerDirection; + live::OrderChannel channel( + kTestLookup, + [&brokerDirection](const ig::TradeOpenObj& order, + const ig::OrderContext&) { + brokerDirection = order.direction; + return ig::OpenResult{.status = ig::OpenStatus::Accepted, + .dealReference = "IGREF-2"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK(channel.request(makeRequest(Direction::SHORT))); + CHECK(brokerDirection == "SELL"); + // The lock hooks speak the engine's direction vocabulary — the SAME + // strings the runner's gate acquired with (LOCK##). + REQUIRE(recorder.extends.size() == 1); + CHECK(recorder.extends[0].direction == "SHORT"); +} + +TEST_CASE("a rejected open releases the lock and records nothing", + "[orderChannel]") { + HookRecorder recorder; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{.status = ig::OpenStatus::Rejected, + .reason = "insufficient margin"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK_FALSE(channel.request(makeRequest())); + CHECK(recorder.sequence + == std::vector{"cluster", "extend", "release"}); + REQUIRE(recorder.releases.size() == 1); + CHECK(recorder.releases[0] + == std::pair{"u-eur", "LONG"}); + CHECK(recorder.saves.empty()); + CHECK(recorder.adds.empty()); +} + +TEST_CASE("a failed open extends the lock for the failure TTL", + "[orderChannel]") { + HookRecorder recorder; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{.status = ig::OpenStatus::Failed, + .reason = "timeout"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK_FALSE(channel.request(makeRequest())); + // In-flight extend, then the C# two-minute-style failure extend; the + // order MAY still be live at the broker so the lock must NOT be released. + CHECK(recorder.sequence + == std::vector{"cluster", "extend", "extend"}); + REQUIRE(recorder.extends.size() == 2); + CHECK(recorder.extends[1].ttl == kFailureTtl); + CHECK(recorder.releases.empty()); + CHECK(recorder.saves.empty()); +} + +TEST_CASE("a throwing broker call takes the failed path, not the process down", + "[orderChannel]") { + HookRecorder recorder; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) -> ig::OpenResult { + throw std::runtime_error("socket reset"); + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK_FALSE(channel.request(makeRequest())); + CHECK(recorder.sequence + == std::vector{"cluster", "extend", "extend"}); + REQUIRE(recorder.extends.size() == 2); + CHECK(recorder.extends[1].ttl == kFailureTtl); +} + +TEST_CASE("a symbol missing from the market definitions never reaches the " + "broker or the lock hooks", + "[orderChannel]") { + HookRecorder recorder; + int brokerCalls = 0; + live::OrderChannel channel( + kTestLookup, + [&brokerCalls](const ig::TradeOpenObj&, const ig::OrderContext&) { + ++brokerCalls; + return ig::OpenResult{.status = ig::OpenStatus::Accepted, + .dealReference = "IGREF-X"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK_FALSE(channel.request(makeRequest(Direction::LONG, "NOTLISTED"))); + CHECK(brokerCalls == 0); + // C# semantics: the signal is dropped and the gate's lock runs out its + // own TTL — no cluster check, no extend, no release. + CHECK(recorder.sequence.empty()); +} + +TEST_CASE("a blocked cluster gate drops the signal and leaves the lock alone", + "[orderChannel]") { + HookRecorder recorder; + recorder.clusterAllow = false; + int brokerCalls = 0; + live::OrderChannel channel( + kTestLookup, + [&brokerCalls](const ig::TradeOpenObj&, const ig::OrderContext&) { + ++brokerCalls; + return ig::OpenResult{.status = ig::OpenStatus::Accepted, + .dealReference = "IGREF-X"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK_FALSE(channel.request(makeRequest())); + // C# semantics: the blocked open never reaches the broker, and the + // gate's trade lock is neither extended nor released — it runs out its + // own TTL, throttling re-signals while the cluster stays busy. + CHECK(brokerCalls == 0); + CHECK(recorder.sequence == std::vector{"cluster"}); + CHECK(recorder.extends.empty()); + CHECK(recorder.releases.empty()); + CHECK(recorder.saves.empty()); +} + +TEST_CASE("the cluster gate is consulted with the symbol and strategy name", + "[orderChannel]") { + HookRecorder recorder; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{.status = ig::OpenStatus::Accepted, + .dealReference = "IGREF-1"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK(channel.request(makeRequest())); + REQUIRE(recorder.clusterChecks.size() == 1); + CHECK(recorder.clusterChecks[0] + == std::pair{"EURUSD", "StubStrategy"}); +} + +TEST_CASE("bookkeeping failure after an accepted open still reports success", + "[orderChannel]") { + HookRecorder recorder; + recorder.saveOk = false; // Redis died between the fill and the PO# write + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{.status = ig::OpenStatus::Accepted, + .dealReference = "IGREF-3"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + // The deal is LIVE at the broker whatever Redis says; the position + // producer rebuilds PL#/PO# from the broker book on its next refresh. + CHECK(channel.request(makeRequest())); + REQUIRE(recorder.adds.size() == 1); // PL# append still attempted +} + +TEST_CASE("the position payload pins the fields the tooling greps for", + "[orderChannel]") { + const live::OrderRequest request = makeRequest(); + const ig::TradeOpenObj order{ + .currencyCode = "USD", + .epic = "TEST.EPIC.MINI.IP", + .direction = "BUY", + .size = 1.5, + .stopDistance = 25, + .limitDistance = 50, + .dealReference = request.dealReference, + }; + const std::string payload = + live::buildPositionPayload(request, order, "IGREF-9", ""); + + CHECK(payload.contains(R"("dealId":"")")); // confirm never resolved + CHECK(payload.contains(R"("dealReference":"IGREF-9")")); + CHECK(payload.contains(R"("strategyId":"u-eur")")); + CHECK(payload.contains(R"("symbol":"EURUSD")")); + CHECK(payload.contains(R"("direction":"BUY")")); + CHECK(payload.contains(R"("level":110002)")); + + const std::string receipt = live::buildDealReceipt(request, "IGREF-9", ""); + CHECK(receipt.contains(R"("id":"DealId#IGREF-9")")); + CHECK(receipt.contains(R"("sort":"EURUSD")")); + CHECK(receipt.contains(R"("dealReference":"IGREF-9")")); + CHECK(receipt.contains(R"("strategyId":"u-eur")")); + CHECK(receipt.contains(R"("strategyName":"StubStrategy")")); + CHECK(receipt.contains(R"("dealId":"")")); + CHECK(receipt.contains(R"("date":"2024-06-26T00:00:00Z")")); +} + +TEST_CASE("a confirmed dealId lands in the PO# payload and the receipt", + "[orderChannel]") { + HookRecorder recorder; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{.status = ig::OpenStatus::Accepted, + .dealReference = "IGREF-4", + .dealId = "DIAAA-77"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK(channel.request(makeRequest())); + REQUIRE(recorder.saves.size() == 1); + CHECK(recorder.saves[0].second.contains(R"("dealId":"DIAAA-77")")); + REQUIRE(recorder.receipts.size() == 1); + CHECK(recorder.receipts[0].second.contains(R"("dealId":"DIAAA-77")")); +} + +TEST_CASE("a confirm-driven rejection still releases the lock", + "[orderChannel]") { + // The confirms poll surfaces REJECTED through the same OpenStatus the + // channel already handles — early lock release, nothing booked. + HookRecorder recorder; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{.status = ig::OpenStatus::Rejected, + .dealReference = "IGREF-5", + .reason = "INSUFFICIENT_FUNDS"}; + }, + kNoClose, recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK_FALSE(channel.request(makeRequest())); + CHECK(recorder.sequence + == std::vector{"cluster", "extend", "release"}); + CHECK(recorder.saves.empty()); +} + +TEST_CASE("closing a LONG position sells it back and prunes the book", + "[orderChannel]") { + HookRecorder recorder; + std::vector closes; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{}; + }, + [&closes](const ig::TradeCloseObj& close, const ig::OrderContext&) { + closes.push_back(close); + return ig::CloseResult{.status = ig::CloseStatus::Ok}; + }, + recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK(channel.closePosition(makeCloseRequest())); + + REQUIRE(closes.size() == 1); + CHECK(closes[0].direction == "SELL"); // flip of the open BUY + CHECK(closes[0].orderType == "MARKET"); + CHECK(closes[0].dealId == "DEAL-77"); + CHECK(closes[0].size == 1.5); + REQUIRE(recorder.removes.size() == 1); + CHECK(recorder.removes[0] + == std::pair{"u-eur", + "ueur-L1719360000000"}); + + // The recently-closed window: a second close of the same deal is + // suppressed without touching the broker again. + CHECK_FALSE(channel.closePosition(makeCloseRequest())); + CHECK(closes.size() == 1); +} + +TEST_CASE("a close without a dealId never reaches the broker", + "[orderChannel]") { + HookRecorder recorder; + int closeCalls = 0; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{}; + }, + [&closeCalls](const ig::TradeCloseObj&, const ig::OrderContext&) { + ++closeCalls; + return ig::CloseResult{.status = ig::CloseStatus::Ok}; + }, + recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK_FALSE(channel.closePosition(makeCloseRequest(""))); + CHECK(closeCalls == 0); + CHECK(recorder.removes.empty()); +} + +TEST_CASE("a close with no broker response drops the phantom book entry", + "[orderChannel]") { + HookRecorder recorder; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{}; + }, + [](const ig::TradeCloseObj&, const ig::OrderContext&) { + return ig::CloseResult{.status = ig::CloseStatus::Gone, + .reason = "no response"}; + }, + recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + // Reported as NOT closed, but the book entry is removed — the C# + // missing-from-IG branch. + CHECK_FALSE(channel.closePosition(makeCloseRequest())); + REQUIRE(recorder.removes.size() == 1); + + // Gone does NOT enter the recently-closed window: a retry may reach IG. + CHECK_FALSE(channel.closePosition(makeCloseRequest())); + CHECK(recorder.removes.size() == 2); +} + +TEST_CASE("a failed close keeps the book entry", "[orderChannel]") { + HookRecorder recorder; + live::OrderChannel channel( + kTestLookup, + [](const ig::TradeOpenObj&, const ig::OrderContext&) { + return ig::OpenResult{}; + }, + [](const ig::TradeCloseObj&, const ig::OrderContext&) { + return ig::CloseResult{.status = ig::CloseStatus::Failed, + .reason = "HTTP 500"}; + }, + recorder.hooks(), kInFlightTtl, kFailureTtl, kClosedTtl); + + CHECK_FALSE(channel.closePosition(makeCloseRequest())); + CHECK(recorder.removes.empty()); +} + +TEST_CASE("every priced symbol is tradable and vice versa", "[orderChannel]") { + // The market table and the symbolScale price table must cover the SAME + // symbols: a priced symbol with no market silently never trades, and a + // market with no scale can't even build an OrderRequest. + CHECK(live::kMarkets.size() == symbol_scale::kTable.size()); + for (const auto& entry : symbol_scale::kTable) { + INFO(entry.symbol); + CHECK(live::findMarket(entry.symbol) != nullptr); + } + for (const auto& market : live::kMarkets) { + INFO(market.symbol); + CHECK(symbol_scale::get(market.symbol) != symbol_scale::kUnknown); + } + CHECK(live::findMarket("NOSUCHSYM") == nullptr); + CHECK(live::findMarket("") == nullptr); +} + +TEST_CASE("the market table carries the C# production wire contract", + "[orderChannel]") { + // These values come verbatim from the C# engine's live + // MarketDescriptions — a broker wire contract, so the load-bearing + // entries are pinned (unlike strategy config, which tests leave free). + const auto* eurusd = live::findMarket("EURUSD"); + REQUIRE(eurusd != nullptr); + CHECK(eurusd->epicMini == "CS.D.EURUSD.MINI.IP"); + CHECK(eurusd->epicCfd == "CS.D.EURUSD.CFD.IP"); + CHECK(eurusd->currency == "USD"); + CHECK(eurusd->tradeSizeModifier == 0.0); // C# null... + CHECK(eurusd->sizeModifier() == 1.0); // ...means unscaled + + // The two markets the C# config scales down. XAUUSD's C# 0.5 was raised + // to 1.0 — IG rejects size 0.5 on that epic as below the market minimum. + const auto* silver = live::findMarket("XAGUSD"); + REQUIRE(silver != nullptr); + CHECK(silver->epicMini == "CS.D.CFDSILVER.CFM.IP"); + CHECK(silver->sizeModifier() == 0.2); + const auto* gold = live::findMarket("XAUUSD"); + REQUIRE(gold != nullptr); + CHECK(gold->epicMini == "CS.D.CFPGOLD.CFP.IP"); + CHECK(gold->currency == "GBP"); // yes — GBP on this account + CHECK(gold->sizeModifier() == 1.0); + + // The one market whose mini epic differs from its CFD epic — picking + // the wrong one doubles the exposure the modifier halves. + const auto* ftse = live::findMarket("GBRIDXGBP"); + REQUIRE(ftse != nullptr); + CHECK(ftse->epicMini == "IX.D.FTSE.IFM.IP"); + CHECK(ftse->epicCfd == "IX.D.FTSE.CFD.IP"); + CHECK(ftse->sizeModifier() == 0.5); +} diff --git a/tests/orderRequest.cpp b/tests/orderRequest.cpp new file mode 100644 index 0000000..b00d2c6 --- /dev/null +++ b/tests/orderRequest.cpp @@ -0,0 +1,129 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// OrderRequest construction: the C# RequestObject level math translated to +// integer points (entry side by direction, stop adverse, limit favourable, +// pips -> points via symbolScale), plus the deal-reference format IG +// constrains ([A-Za-z0-9_-], max 30 chars) — pinned here because a drifted +// reference silently breaks fill matching in the confirm stream. + +#include + +#include +#include +#include + +import liveStrategyRunner; +import orderRequest; +import trade; + +namespace { + +// 2024-06-26 00:00:00 UTC, same fixture instant as the liveRunner tests. +constexpr std::chrono::system_clock::time_point kTick{ + std::chrono::microseconds{1'719'360'000'000'000LL}}; + +live::OrderIntent makeIntent(const std::string& symbol, + const Direction direction) { + return live::OrderIntent{ + .strategyName = "StubStrategy", + .strategyUuid = "f47ac10b-58cc-4372-a567-0e02b2c3d479", + .symbol = symbol, + .direction = direction, + .size = 3, + .stopDistancePips = 25, + .limitDistancePips = 50, + .bid = 110000, + .ask = 110002, + .timestamp = kTick, + }; +} + +bool validReferenceCharset(const std::string& reference) { + return std::ranges::all_of(reference, [](const char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || c == '_' || c == '-'; + }); +} + +} // namespace + +TEST_CASE("LONG request enters at ask with stop below and limit above", + "[orderRequest]") { + const auto request = live::makeOrderRequest( + makeIntent("EURUSD", Direction::LONG)); // EURUSD: 10 points per pip + REQUIRE(request.has_value()); + + CHECK(request->pointsPerPip == 10); + CHECK(request->level == 110002); // entry at ask + CHECK(request->stopLevel == 110002 - 25 * 10); // stop adverse (below) + CHECK(request->limitLevel == 110002 + 50 * 10); // limit favourable (above) + CHECK(request->spreadPoints == 2); + CHECK(request->symbol == "EURUSD"); + CHECK(request->size == 3); + CHECK(request->stopDistancePips == 25); + CHECK(request->limitDistancePips == 50); + CHECK(request->timestamp == kTick); +} + +TEST_CASE("SHORT request mirrors: enters at bid, stop above, limit below", + "[orderRequest]") { + const auto request = + live::makeOrderRequest(makeIntent("EURUSD", Direction::SHORT)); + REQUIRE(request.has_value()); + + CHECK(request->level == 110000); // entry at bid + CHECK(request->stopLevel == 110000 + 25 * 10); // stop adverse (above) + CHECK(request->limitLevel == 110000 - 50 * 10); // limit favourable (below) + CHECK(request->spreadPoints == 2); +} + +TEST_CASE("pip distances scale by the symbol's points-per-pip", + "[orderRequest]") { + const auto request = live::makeOrderRequest( + makeIntent("XAGUSD", Direction::LONG)); // metals: 1000 points per pip + REQUIRE(request.has_value()); + CHECK(request->pointsPerPip == 1000); + CHECK(request->stopLevel == 110002 - 25 * 1000); + CHECK(request->limitLevel == 110002 + 50 * 1000); +} + +TEST_CASE("a symbol unknown to symbolScale yields no request", + "[orderRequest]") { + CHECK_FALSE( + live::makeOrderRequest(makeIntent("NOSUCHSYM", Direction::LONG)) + .has_value()); +} + +TEST_CASE("deal reference fits IG's constraints and format", "[orderRequest]") { + const std::string reference = live::makeDealReference( + "f47ac10b-58cc-4372-a567-0e02b2c3d479", Direction::LONG, kTick); + + // First 8 alphanumerics of the UUID (dashes stripped), direction letter, + // epoch millis of the decision tick. + CHECK(reference == "f47ac10b-L1719360000000"); + CHECK(reference.size() <= 30); + CHECK(validReferenceCharset(reference)); + + CHECK(live::makeDealReference("f47ac10b-58cc-4372-a567-0e02b2c3d479", + Direction::SHORT, kTick) + == "f47ac10b-S1719360000000"); + // Different decision ticks mint different references (the idempotency + // token must be unique per placement attempt). + CHECK(live::makeDealReference("f47ac10b-58cc-4372-a567-0e02b2c3d479", + Direction::LONG, + kTick + std::chrono::milliseconds{1}) + != reference); +} + +TEST_CASE("makeOrderRequest carries the minted deal reference", + "[orderRequest]") { + const auto request = + live::makeOrderRequest(makeIntent("EURUSD", Direction::LONG)); + REQUIRE(request.has_value()); + CHECK(request->dealReference == "f47ac10b-L1719360000000"); + CHECK(validReferenceCharset(request->dealReference)); +} diff --git a/tests/outcomeIndices.cpp b/tests/outcomeIndices.cpp new file mode 100644 index 0000000..3cdeeed --- /dev/null +++ b/tests/outcomeIndices.cpp @@ -0,0 +1,138 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include + +#include +#include // setenv/unsetenv — the batch label is env-overridable +#include +#include + +#include + +#include "run/reporting/outcomeIndices.hpp" +#include "run/reporting/tradingResults.hpp" +#include "shared/tradingDefinitions/config/configuration.hpp" + +import elasticClient; // resultsBaseFor — the winners/results routing split +import rollingWindow; // rolling::kFullHistory — the ladder's terminal window + +// Fixed epochs, each verified against strftime("%G-%V") itself +// (date -u -r +%G-%V): the ISO week-based year differs from the +// calendar year around New Year, and the week is zero-padded so lexical +// order stays chronological. +TEST_CASE("isoWeekLabel formats the ISO year and week", "[outcomeIndices]") { + CHECK(outcome_index::isoWeekLabel(1783771200) == "2026-28"); // 2026-07-11 + CHECK(outcome_index::isoWeekLabel(1767268800) == "2026-01"); // 2026-01-01 (zero-padded) + CHECK(outcome_index::isoWeekLabel(1735560000) == "2025-01"); // 2024-12-30 (ISO year ahead) + CHECK(outcome_index::isoWeekLabel(1798804800) == "2026-53"); // 2027-01-01 (ISO year behind) +} + +// $BACKTEST_BATCH pins the label (re-runs; load.sh exports it so its eight +// per-strategy invocations can't straddle an ISO-week boundary); without it +// the label is the current UTC week in the same YYYY-WW shape. +TEST_CASE("currentBatchLabel prefers the env override", "[outcomeIndices]") { + setenv("BACKTEST_BATCH", "2099-01", 1); + CHECK(outcome_index::currentBatchLabel() == "2099-01"); + + unsetenv("BACKTEST_BATCH"); + const std::string label = outcome_index::currentBatchLabel(); + REQUIRE(label.size() == 7); + for (std::size_t i = 0; i < label.size(); ++i) { + INFO("unexpected character at index " << i << " in " << label); + if (i == 4) { + CHECK(label[i] == '-'); + } else { + CHECK(std::isdigit(static_cast(label[i])) != 0); + } + } +} + +TEST_CASE("weeklyIndex suffixes the batch and falls back bare", "[outcomeIndices]") { + CHECK(outcome_index::weeklyIndex(outcome_index::kResultsBase, "2026-28") == + "backtesting-results-2026-28"); + // Empty batch (payloads predating the field, hand-run configs) keeps + // writing to the unsuffixed base — the legacy escape hatch. + CHECK(outcome_index::weeklyIndex(outcome_index::kResultsBase, "") == + "backtesting-results"); +} + +TEST_CASE("currentAlias derives the rolling alias name", "[outcomeIndices]") { + CHECK(outcome_index::currentAlias(outcome_index::kWinnersBase) == + "backtesting-winners-current"); + // Elasticsearch forbids an alias sharing a concrete index's name, and the + // unsuffixed fallback index is the one name the batch suffix can't + // distinguish the alias from. + for (const std::string_view base : outcome_index::kWeeklyBases) { + CHECK(outcome_index::currentAlias(base) != + outcome_index::weeklyIndex(base, "")); + } +} + +// The winners/results split: exactly the ladder's terminal full-history +// window routes to the winners index — the population live boots against — +// and every other window is a screening pass. Derived from +// rolling::kFullHistory rather than literals, so the test follows the ladder +// if it ever grows. +TEST_CASE("resultsBaseFor routes only the terminal window to the winners index", + "[outcomeIndices]") { + const rolling::Window full = rolling::kFullHistory; + CHECK(resultsBaseFor(full.lastMonths, full.offsetMonths) == + outcome_index::kWinnersBase); + CHECK(resultsBaseFor(full.lastMonths, full.offsetMonths + 3) == + outcome_index::kResultsBase); + CHECK(resultsBaseFor(full.lastMonths - 1, full.offsetMonths) == + outcome_index::kResultsBase); + CHECK(resultsBaseFor(0, 0) == outcome_index::kResultsBase); +} + +// Outcome documents carry top-level copies of the batch identity for Kibana +// filters/aggregations (the embedded config has them too) — and a pre-batch +// config keeps its exact legacy shape: absent keys, not empty strings. +TEST_CASE("outcome documents carry the batch identity when present", + "[outcomeIndices]") { + tradingDefinitions::Configuration config; + config.RUN_ID = "run"; + config.BATCH = "2099-01"; + config.EXECUTION_TS = "2099-01-01T00:00:00Z"; + + const TradeFinal finalDoc{.RUN_ID = "run", + .timestamp = "t", + .durationSeconds = 1.0, + .success = 1, + .status = "completed", + .hostname = "host", + .config = config}; + const nlohmann::json finalJson = finalDoc; + CHECK(finalJson.at("batch") == "2099-01"); + CHECK(finalJson.at("executionTimestamp") == "2099-01-01T00:00:00Z"); + CHECK(finalJson.at("config").at("BATCH") == "2099-01"); + + const TradingFailure failureDoc{.RUN_ID = "run", + .timestamp = "t", + .durationSeconds = 1.0, + .reason = "loss_limit", + .breachPnlPips = 0.0, + .lossFloorPips = 0.0, + .hostname = "host", + .config = config, + .results = {}}; + const nlohmann::json failureJson = failureDoc; + CHECK(failureJson.at("batch") == "2099-01"); + CHECK(failureJson.at("executionTimestamp") == "2099-01-01T00:00:00Z"); + + config.BATCH.clear(); + config.EXECUTION_TS.clear(); + const TradingResults legacyDoc{.RUN_ID = "run", + .timestamp = "t", + .durationSeconds = 1.0, + .hostname = "host", + .config = config, + .results = {}}; + const nlohmann::json legacyJson = legacyDoc; + CHECK_FALSE(legacyJson.contains("batch")); + CHECK_FALSE(legacyJson.contains("executionTimestamp")); +} diff --git a/tests/positionBook.cpp b/tests/positionBook.cpp new file mode 100644 index 0000000..b09993d --- /dev/null +++ b/tests/positionBook.cpp @@ -0,0 +1,161 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// The live position book codecs: the PO# payload the order channel writes +// must round-trip through the decoder the position feed reads (a drift +// silently empties every worker's book), and the record -> BookedPosition +// mapping must reverse the trade-size modifier so seeded trades reflect the +// actual broker position. Redis itself is not involved — same codec-only +// precedent as tests/positionManager.cpp. + +#include + +#include +#include + +#include "shared/redis/positionManager.hpp" + +import igMarkets; +import liveStrategyRunner; +import marketDefinitions; +import orderChannel; +import orderRequest; +import redisPositionFeed; +import trade; + +namespace { + +constexpr std::chrono::system_clock::time_point kTick{ + std::chrono::microseconds{1'719'360'000'000'000LL}}; + +live::OrderRequest makeRequest() { + const auto request = live::makeOrderRequest(live::OrderIntent{ + .strategyName = "StubStrategy", + .strategyUuid = "u-eur", + .symbol = "EURUSD", + .direction = Direction::LONG, + .size = 3, + .stopDistancePips = 25, + .limitDistancePips = 50, + .bid = 110000, + .ask = 110002, + .timestamp = kTick, + }); + REQUIRE(request.has_value()); + return *request; +} + +redis_positions::PositionRecord makeRecord() { + redis_positions::PositionRecord record; + record.dealId = "DIAAA-1"; + record.dealReference = "IGREF-1"; + record.symbol = "EURUSD"; + record.direction = "BUY"; + record.size = 1.5; + record.level = 110002; + record.openedAtMicros = 1'719'360'000'000'000LL; + return record; +} + +} // namespace + +TEST_CASE("the PO# payload round-trips through decodePositionRecord", + "[positionBook]") { + const live::OrderRequest request = makeRequest(); + const ig::TradeOpenObj order{ + .currencyCode = "USD", + .epic = "TEST.EPIC.MINI.IP", + .direction = "BUY", + .size = 1.5, + .stopDistance = 25, + .limitDistance = 50, + .dealReference = request.dealReference, + }; + const std::string payload = live::buildPositionPayload( + request, order, "IGREF-1", "DIAAA-1"); + + const auto record = redis_positions::decodePositionRecord(payload); + REQUIRE(record.has_value()); + CHECK(record->dealId == "DIAAA-1"); + CHECK(record->dealReference == "IGREF-1"); + CHECK(record->symbol == "EURUSD"); + CHECK(record->epic == "TEST.EPIC.MINI.IP"); + CHECK(record->direction == "BUY"); + CHECK(record->size == 1.5); + CHECK(record->level == 110002); + CHECK(record->stopLevel == 110002 - 250); + CHECK(record->limitLevel == 110002 + 500); + CHECK(record->strategyId == "u-eur"); + CHECK(record->strategyName == "StubStrategy"); + CHECK(record->openedAtMicros == 1'719'360'000'000'000LL); +} + +TEST_CASE("decodePositionRecord tolerates missing optionals and rejects " + "garbage", + "[positionBook]") { + // Only identity, symbol and direction are load-bearing — a + // producer-rewritten payload may drop the rest. + const auto minimal = redis_positions::decodePositionRecord( + R"({"dealReference":"R1","symbol":"EURUSD","direction":"SELL"})"); + REQUIRE(minimal.has_value()); + CHECK(minimal->dealId.empty()); + CHECK(minimal->size == 0.0); + CHECK(minimal->level == 0); + + CHECK_FALSE(redis_positions::decodePositionRecord( + R"({"symbol":"EURUSD","direction":"BUY"})") // no reference + .has_value()); + CHECK_FALSE(redis_positions::decodePositionRecord( + R"({"dealReference":"R1","direction":"BUY"})") // no symbol + .has_value()); + CHECK_FALSE(redis_positions::decodePositionRecord( + R"({"dealReference":"R1","symbol":"EURUSD"})") // no direction + .has_value()); + CHECK_FALSE(redis_positions::decodePositionRecord("not json").has_value()); + CHECK_FALSE(redis_positions::decodePositionRecord("[]").has_value()); + CHECK_FALSE(redis_positions::decodePositionRecord("").has_value()); +} + +TEST_CASE("toBookedPosition maps BUY/SELL and reverses the size modifier", + "[positionBook]") { + // A market with a 0.5 modifier: broker 1.5 units = 3 engine lots. + constexpr live::MarketDefinition halved{ + .symbol = "EURUSD", + .igMarketId = "EURUSD", + .epicCfd = "TEST.EPIC.CFD.IP", + .epicMini = "TEST.EPIC.MINI.IP", + .currency = "USD", + .polygonIdentifier = "", + .polygonScale = 0, + .type = live::MarketType::Forex, + .tradeSizeModifier = 0.5, + }; + const auto buy = live::toBookedPosition(makeRecord(), &halved); + REQUIRE(buy.has_value()); + CHECK(buy->direction == Direction::LONG); + CHECK(buy->brokerSize == 1.5); // verbatim — what a close must send + CHECK(buy->engineSize == 3); // 1.5 / 0.5 + CHECK(buy->level == 110002); + CHECK(buy->dealId == "DIAAA-1"); + CHECK(buy->dealReference == "IGREF-1"); + CHECK(buy->openedAt == kTick); + + auto sellRecord = makeRecord(); + sellRecord.direction = "SELL"; + const auto sell = live::toBookedPosition(sellRecord, &halved); + REQUIRE(sell.has_value()); + CHECK(sell->direction == Direction::SHORT); + + // Symbol dropped from the market table: size passes through unscaled. + const auto unscaled = live::toBookedPosition(makeRecord(), nullptr); + REQUIRE(unscaled.has_value()); + CHECK(unscaled->engineSize == 2); // llround(1.5 / 1.0) + + // An unaddressable direction must not enter the book. + auto badRecord = makeRecord(); + badRecord.direction = "LONG"; // engine vocabulary, not the broker's + CHECK_FALSE(live::toBookedPosition(badRecord, &halved).has_value()); +} diff --git a/tests/positionClustering.cpp b/tests/positionClustering.cpp new file mode 100644 index 0000000..f544ae9 --- /dev/null +++ b/tests/positionClustering.cpp @@ -0,0 +1,218 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// redis_clusters — the C# PositionClustering port. Pins the wire contract +// shared with the C# producer (CG#/CLUSTER_LOCK# key formats, the +// symbol -> cluster map, per-cluster limits and strictness) and the pure +// evaluateClusterGroup verdict logic, all without a Redis server — the same +// convention as the tradeLocks/positionManager tests. + +#include + +#include +#include +#include +#include +#include + +#include "shared/redis/positionClustering.hpp" + +import symbolScale; + +namespace { + +using redis_clusters::ClusterVerdict; + +std::vector groupsVector(const std::string_view symbol) { + const auto groups = redis_clusters::groupsFor(symbol); + return {groups.begin(), groups.end()}; +} + +} // namespace + +TEST_CASE("cluster keys match the C# PositionClustering formats", + "[positionClustering]") { + CHECK(redis_clusters::clusterKey("US_Index") == "CG#US_Index"); + CHECK(redis_clusters::clusterLockKey("US_Index") + == "CLUSTER_LOCK#US_Index"); +} + +TEST_CASE("every priced symbol belongs to at least one cluster", + "[positionClustering]") { + // A symbol the engine can trade but no cluster knows about would be + // blocked outright by the fail-closed gate — that must be a deliberate + // table edit, not an accident. + for (const auto& entry : symbol_scale::kTable) { + INFO("symbol_scale entry missing a cluster: " << entry.symbol); + CHECK_FALSE(redis_clusters::groupsFor(entry.symbol).empty()); + } +} + +TEST_CASE("the cluster map mirrors the C# BuildCluster memberships", + "[positionClustering]") { + CHECK(groupsVector("USA500IDXUSD") + == std::vector{"US_Index"}); + CHECK(groupsVector("AUDNZD") + == std::vector{"Commodity_FX", "Crosses", + "AUD_Pairs"}); + CHECK(groupsVector("EURNOK") + == std::vector{"Commodity_FX", "Crosses", + "EUR_Pairs", "Scandi_Pairs"}); + CHECK(groupsVector("USDSEK") + == std::vector{"USD_Forex", "Scandi_Pairs"}); + // Symbols the engine does not trade stay in the map on purpose (the C# + // producer may still cluster them); unknown symbols resolve empty. + CHECK(groupsVector("GBPAUD") + == std::vector{"Crosses", "GBP_Pairs", + "AUD_Pairs"}); + CHECK(groupsVector("NGASCMDUSD") + == std::vector{"Energy"}); + CHECK(redis_clusters::groupsFor("DOGEUSD").empty()); + CHECK(redis_clusters::groupsFor("").empty()); +} + +TEST_CASE("cluster limits and strictness mirror the C# tables", + "[positionClustering]") { + CHECK(redis_clusters::clusterLimit("US_Index") == 1); + CHECK(redis_clusters::clusterLimit("USD_Forex") == 2); + CHECK(redis_clusters::clusterLimit("Scandi_Pairs") == 1); + // The C# GroupLimits fallback: an unlisted cluster defaults to 1. + CHECK(redis_clusters::clusterLimit("NOT_A_CLUSTER") == 1); + + CHECK(redis_clusters::isStrictCluster("US_Index")); + CHECK(redis_clusters::isStrictCluster("Precious_Metals")); + CHECK_FALSE(redis_clusters::isStrictCluster("USD_Forex")); + CHECK_FALSE(redis_clusters::isStrictCluster("Scandi_Pairs")); + CHECK_FALSE(redis_clusters::isStrictCluster("NOT_A_CLUSTER")); +} + +TEST_CASE("evaluateClusterGroup applies the C# checks in order", + "[positionClustering]") { + SECTION("empty cluster allows") { + CHECK(redis_clusters::evaluateClusterGroup( + {}, "EURUSD", "OhlcBreakoutStrategy", 2, false) + == ClusterVerdict::Allowed); + } + + SECTION("under the limit with no conflicts allows") { + const std::vector members{ + "GBPUSD#RandomStrategy#IGREF-1"}; + CHECK(redis_clusters::evaluateClusterGroup( + members, "EURUSD", "OhlcBreakoutStrategy", 2, false) + == ClusterVerdict::Allowed); + } + + SECTION("a full cluster blocks, even for an unrelated strategy") { + const std::vector members{ + "GBPUSD#RandomStrategy#IGREF-1", + "USDJPY#RandomStrategy#IGREF-2"}; + CHECK(redis_clusters::evaluateClusterGroup( + members, "EURUSD", "OhlcBreakoutStrategy", 2, false) + == ClusterVerdict::GroupFull); + } + + SECTION("capacity outranks stacking (the C# check order)") { + const std::vector members{ + "USA500IDXUSD#OhlcBreakoutStrategy#IGREF-1"}; + CHECK(redis_clusters::evaluateClusterGroup( + members, "USA500IDXUSD", "OhlcBreakoutStrategy", 1, true) + == ClusterVerdict::GroupFull); + } + + SECTION("the same (symbol, strategy) already open blocks — stacking") { + const std::vector members{ + "EURUSD#OhlcBreakoutStrategy#IGREF-1"}; + CHECK(redis_clusters::evaluateClusterGroup( + members, "EURUSD", "OhlcBreakoutStrategy", 2, false) + == ClusterVerdict::SymbolStrategyStacked); + } + + SECTION("same strategy, another symbol: blocked when strict") { + // Every strict cluster ships with limit 1 today, so this branch is + // shadowed by GroupFull through the real tables — it guards the day + // a strict cluster's capacity is raised (the C# has the same + // structure). + const std::vector members{ + "USA30IDXUSD#OhlcBreakoutStrategy#IGREF-1"}; + CHECK(redis_clusters::evaluateClusterGroup( + members, "USA500IDXUSD", "OhlcBreakoutStrategy", 2, true) + == ClusterVerdict::StrategyNotDiverse); + } + + SECTION("same strategy, another symbol: allowed when not strict") { + const std::vector members{ + "GBPUSD#OhlcBreakoutStrategy#IGREF-1"}; + CHECK(redis_clusters::evaluateClusterGroup( + members, "EURUSD", "OhlcBreakoutStrategy", 2, false) + == ClusterVerdict::Allowed); + } + + SECTION("malformed members are skipped, not trusted") { + const std::vector members{"garbage-without-hashes"}; + CHECK(redis_clusters::evaluateClusterGroup( + members, "EURUSD", "OhlcBreakoutStrategy", 2, true) + == ClusterVerdict::Allowed); + } + + SECTION("a two-field member still matches (the C# parts >= 2 tolerance)") { + const std::vector members{"EURUSD#OhlcBreakoutStrategy"}; + CHECK(redis_clusters::evaluateClusterGroup( + members, "EURUSD", "OhlcBreakoutStrategy", 2, false) + == ClusterVerdict::SymbolStrategyStacked); + } +} + +TEST_CASE("clusterMemberString writes the CG# member wire format", + "[positionClustering]") { + CHECK(redis_clusters::clusterMemberString( + {.symbol = "XAUUSD", + .strategyName = "Fvg", + .dealReference = "ref-1"}) + == "XAUUSD#Fvg#ref-1"); +} + +TEST_CASE("groupMembersByCluster fans members out, dedups and drops unmapped " + "symbols", + "[positionClustering]") { + // Injected lookup — the grouping machinery is under test, not the + // current kClusterTable (same seam discipline as MarketLookup). + static constexpr std::array aaaGroups{"C1", "C2"}; + static constexpr std::array bbbGroups{"C2"}; + const redis_clusters::GroupsLookup lookup = + [](const std::string_view symbol) + -> std::span { + if (symbol == "AAA") { + return aaaGroups; + } + if (symbol == "BBB") { + return bbbGroups; + } + return {}; + }; + + const std::vector members{ + {.symbol = "AAA", .strategyName = "s1", .dealReference = "d1"}, + {.symbol = "BBB", .strategyName = "s2", .dealReference = "d2"}, + // Exact duplicate — the C# HashSet collapses it. + {.symbol = "AAA", .strategyName = "s1", .dealReference = "d1"}, + // Unmapped symbol — dropped, like the C# TryGetValue skip. + {.symbol = "ZZZ", .strategyName = "s3", .dealReference = "d3"}, + }; + + const auto clusters = redis_clusters::groupMembersByCluster(members, lookup); + REQUIRE(clusters.size() == 2); + REQUIRE(clusters.contains("C1")); + REQUIRE(clusters.contains("C2")); + CHECK(clusters.at("C1") == std::vector{"AAA#s1#d1"}); + CHECK(clusters.at("C2") + == std::vector{"AAA#s1#d1", "BBB#s2#d2"}); +} + +TEST_CASE("groupMembersByCluster of nothing is an empty map", + "[positionClustering]") { + CHECK(redis_clusters::groupMembersByCluster({}, redis_clusters::groupsFor) + .empty()); +} diff --git a/tests/positionCounterCache.cpp b/tests/positionCounterCache.cpp new file mode 100644 index 0000000..c8e3063 --- /dev/null +++ b/tests/positionCounterCache.cpp @@ -0,0 +1,76 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// PositionCountCache — the MAX_OPEN_TRADES cap's per-thread count cache. +// Clock-free by design (`now`/`freshUntil` are injected), so expiry and the +// invalidate-after-open contract are pinned here without waiting on a TTL. + +#include + +#include + +import redisPositionCounter; + +namespace { + +using Clock = std::chrono::steady_clock; + +} // namespace + +TEST_CASE("a cached count is served only while fresh", "[positionCounter]") { + live::PositionCountCache cache; + const Clock::time_point t0{}; + const auto ttl = std::chrono::seconds{15}; + + CHECK_FALSE(cache.get("u-eur", t0).has_value()); + + cache.put("u-eur", 2, t0 + ttl); + CHECK(cache.get("u-eur", t0) == 2); + CHECK(cache.get("u-eur", t0 + ttl - std::chrono::milliseconds{1}) == 2); + + // freshUntil is EXCLUSIVE: at exactly the deadline the value is stale. + CHECK_FALSE(cache.get("u-eur", t0 + ttl).has_value()); + CHECK_FALSE(cache.get("u-eur", t0 + ttl * 2).has_value()); +} + +TEST_CASE("invalidate forces the next read back to Redis", + "[positionCounter]") { + // The L4 regression: after this worker's own accepted open moved PL#, + // serving the pre-open count for the rest of the TTL let a second + // direction through MAX_OPEN_TRADES (the trade lock is per-direction + // and never serialised the two). brokerOrderSink invalidates on every + // accepted open / successful close; a fresh get() must then miss. + live::PositionCountCache cache; + const Clock::time_point t0{}; + const auto freshUntil = t0 + std::chrono::seconds{15}; + + cache.put("u-eur", 0, freshUntil); + REQUIRE(cache.get("u-eur", t0) == 0); + + cache.invalidate("u-eur"); + CHECK_FALSE(cache.get("u-eur", t0).has_value()); + + // Re-population after the forced miss behaves like any fresh entry. + cache.put("u-eur", 1, freshUntil); + CHECK(cache.get("u-eur", t0) == 1); +} + +TEST_CASE("entries are isolated per strategy uuid", "[positionCounter]") { + live::PositionCountCache cache; + const Clock::time_point t0{}; + const auto freshUntil = t0 + std::chrono::seconds{15}; + + cache.put("u-eur", 1, freshUntil); + cache.put("u-gbp", 3, freshUntil); + + cache.invalidate("u-eur"); + CHECK_FALSE(cache.get("u-eur", t0).has_value()); + CHECK(cache.get("u-gbp", t0) == 3); // untouched + + // Invalidating a uuid nobody cached is a harmless no-op. + cache.invalidate("u-unknown"); + CHECK(cache.get("u-gbp", t0) == 3); +} diff --git a/tests/positionManager.cpp b/tests/positionManager.cpp new file mode 100644 index 0000000..893671e --- /dev/null +++ b/tests/positionManager.cpp @@ -0,0 +1,156 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// The position KEY FORMATS and the PL# list encoding are a wire contract +// shared with the C# engine's PositionManager and the external position +// producer (PL# / PO# / PH#, list = JSON string +// array) — drift would silently split the position space between the +// programs, so both are pinned here without needing a Redis server. The +// Redis behaviour itself lives behind RedisOperations, same as TradeLocks. + +#include + +#include +#include + +#include "shared/redis/positionManager.hpp" + +TEST_CASE("position keys match the C# PositionManager prefixes", + "[positionManager]") { + CHECK(redis_positions::positionListKey("uuid-1") == "PL#uuid-1"); + CHECK(redis_positions::positionKey("deal-9") == "PO#deal-9"); + CHECK(redis_positions::historyPositionKey("deal-9") == "PH#deal-9"); +} + +TEST_CASE("the three key families never collide for the same id", + "[positionManager]") { + const std::string id = "abc"; + CHECK(redis_positions::positionListKey(id) != redis_positions::positionKey(id)); + CHECK(redis_positions::positionKey(id) + != redis_positions::historyPositionKey(id)); + CHECK(redis_positions::positionListKey(id) + != redis_positions::historyPositionKey(id)); +} + +TEST_CASE("position list codec round-trips the C# JSON array shape", + "[positionManager]") { + CHECK(redis_positions::encodePositionList({}) == "[]"); + CHECK(redis_positions::encodePositionList({"d1", "d2"}) + == R"(["d1","d2"])"); + + const std::vector dealIds{"deal-1", "deal-2", "deal-3"}; + const auto decoded = redis_positions::decodePositionList( + redis_positions::encodePositionList(dealIds)); + REQUIRE(decoded.has_value()); + CHECK(*decoded == dealIds); +} + +TEST_CASE("decodePositionList accepts an empty array and preserves order", + "[positionManager]") { + const auto empty = redis_positions::decodePositionList("[]"); + REQUIRE(empty.has_value()); + CHECK(empty->empty()); + + const auto ordered = redis_positions::decodePositionList(R"(["b","a"])"); + REQUIRE(ordered.has_value()); + CHECK(*ordered == std::vector{"b", "a"}); +} + +TEST_CASE("decodePositionList rejects anything but a JSON string array", + "[positionManager]") { + CHECK_FALSE(redis_positions::decodePositionList("not json").has_value()); + CHECK_FALSE(redis_positions::decodePositionList("{}").has_value()); + CHECK_FALSE(redis_positions::decodePositionList("null").has_value()); + CHECK_FALSE(redis_positions::decodePositionList("\"d1\"").has_value()); + CHECK_FALSE(redis_positions::decodePositionList("[1, 2]").has_value()); + CHECK_FALSE(redis_positions::decodePositionList(R"(["d1", 2])").has_value()); +} + +TEST_CASE("encodePositionRecord writes the orderChannel/C# PO# wire shape", + "[positionManager]") { + // Exact-string pin: the field ORDER is part of the contract (the C# + // JsonSerializer and buildPositionPayload both write it this way), so an + // alphabetising encoder would be a wire drift even with equal content. + const redis_positions::PositionRecord record{ + .dealId = "DIAAAA", + .dealReference = "ref-1", + .symbol = "XAUUSD", + .epic = "CS.D.CFPGOLD.CFP.IP", + .direction = "BUY", + .size = 1.5, + .level = 2345678, + .stopLevel = 2340000, + .limitLevel = 0, + .strategyId = "uuid-1", + .strategyName = "Fvg", + .openedAtMicros = 1720000000000000, + }; + CHECK(redis_positions::encodePositionRecord(record) + == R"({"dealId":"DIAAAA","dealReference":"ref-1","symbol":"XAUUSD",)" + R"("epic":"CS.D.CFPGOLD.CFP.IP","direction":"BUY","size":1.5,)" + R"("level":2345678,"stopLevel":2340000,"limitLevel":0,)" + R"("strategyId":"uuid-1","strategyName":"Fvg",)" + R"("openedAt":1720000000000000})"); +} + +TEST_CASE("encodePositionRecord round-trips through decodePositionRecord", + "[positionManager]") { + redis_positions::PositionRecord record; + record.dealId = "DI-1"; + record.dealReference = "ref-2"; + record.symbol = "EURUSD"; + record.epic = "CS.D.EURUSD.MINI.IP"; + record.direction = "SELL"; + record.size = 2.0; + record.level = 110001; + record.stopLevel = 110500; + record.limitLevel = 109000; + record.strategyId = "uuid-9"; + record.strategyName = "OhlcBreakout"; + record.openedAtMicros = 1234567890123456; + + const auto decoded = redis_positions::decodePositionRecord( + redis_positions::encodePositionRecord(record)); + REQUIRE(decoded.has_value()); + CHECK(decoded->dealId == record.dealId); + CHECK(decoded->dealReference == record.dealReference); + CHECK(decoded->symbol == record.symbol); + CHECK(decoded->epic == record.epic); + CHECK(decoded->direction == record.direction); + CHECK(decoded->size == record.size); + CHECK(decoded->level == record.level); + CHECK(decoded->stopLevel == record.stopLevel); + CHECK(decoded->limitLevel == record.limitLevel); + CHECK(decoded->strategyId == record.strategyId); + CHECK(decoded->strategyName == record.strategyName); + CHECK(decoded->openedAtMicros == record.openedAtMicros); +} + +TEST_CASE("decodeDealReceipt reads the buildDealReceipt/C# DealReceipt shape", + "[positionManager]") { + // The literal mirrors orderChannel::buildDealReceipt — the producer only + // consumes the strategy attribution and the broker dealId. + const auto receipt = redis_positions::decodeDealReceipt( + R"({"id":"DealId#ref-1","sort":"XAUUSD","date":"2026-07-10T09:00:00Z",)" + R"("dealReference":"ref-1","strategyId":"uuid-1","dealId":"DIAAAA",)" + R"("strategyName":"Fvg"})"); + REQUIRE(receipt.has_value()); + CHECK(receipt->strategyId == "uuid-1"); + CHECK(receipt->dealId == "DIAAAA"); + CHECK(receipt->strategyName == "Fvg"); +} + +TEST_CASE("decodeDealReceipt tolerates missing fields but rejects non-objects", + "[positionManager]") { + const auto sparse = redis_positions::decodeDealReceipt("{}"); + REQUIRE(sparse.has_value()); // the caller substitutes "Unknown" + CHECK(sparse->strategyId.empty()); + CHECK(sparse->strategyName.empty()); + + CHECK_FALSE(redis_positions::decodeDealReceipt("not json").has_value()); + CHECK_FALSE(redis_positions::decodeDealReceipt("[]").has_value()); + CHECK_FALSE(redis_positions::decodeDealReceipt("null").has_value()); +} diff --git a/tests/positionSync.cpp b/tests/positionSync.cpp new file mode 100644 index 0000000..cb00e7d --- /dev/null +++ b/tests/positionSync.cpp @@ -0,0 +1,303 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// positionSync — the pure half of the IG position producer (the C# +// igmarkets_positions port): the /positions JSON decode, IG's createdDate +// parse, the update-TTL weekend rule, and the fresh-record/update mutations. +// All exercised without Redis or HTTP, with the price scale INJECTED — the +// machinery is under test, not the current symbolScale/marketDefinitions +// tables. The one table check (findMarketByEpicMini) is self-consistency: +// every market must be reachable from its own mini epic, whatever the table +// currently lists. + +#include + +#include +#include + +#include "shared/redis/positionManager.hpp" + +import positionSync; +import marketDefinitions; + +using namespace std::chrono; + +namespace { + +std::int64_t microsAt(const sys_days day, const hours h, const minutes m, + const seconds s, const milliseconds ms) { + return duration_cast((day + h + m + s + ms).time_since_epoch()) + .count(); +} + +positions::IgPosition samplePosition() { + positions::IgPosition position; + position.epic = "CS.D.CFPGOLD.CFP.IP"; + position.dealReference = "ref-1"; + position.dealId = "DIAAAA"; + position.direction = "BUY"; + position.createdDate = "2026/07/10 09:30:00:250"; + position.dealSize = 1.5; + position.openLevel = 2345.678; + position.stopLevel = 2340.0; + position.limitLevel = 0.0; // IG null decoded as 0 + return position; +} + +} // namespace + +// ---- updateTtl: the Friday-night weekend hold ------------------------------ + +TEST_CASE("updateTtl is 10 minutes on an ordinary weekday", + "[positionSync]") { + // 2026-07-09 is a Thursday, 2026-07-11 a Saturday. + CHECK(positions::updateTtl(sys_days{2026y / 7 / 9} + 21h + 56min) + == minutes{10}); + CHECK(positions::updateTtl(sys_days{2026y / 7 / 11} + 21h + 57min) + == minutes{10}); +} + +TEST_CASE("updateTtl holds Friday 21:55:00-21:59:59 UTC over the weekend", + "[positionSync]") { + constexpr auto weekendHold = days{2} + hours{2}; + const sys_days friday{2026y / 7 / 10}; // a Friday + CHECK(positions::updateTtl(friday + 21h + 54min + 59s) == minutes{10}); + CHECK(positions::updateTtl(friday + 21h + 55min) == weekendHold); + CHECK(positions::updateTtl(friday + 21h + 59min + 59s) == weekendHold); + CHECK(positions::updateTtl(friday + 22h) == minutes{10}); + // Same minutes on other Friday hours stay ordinary. + CHECK(positions::updateTtl(friday + 20h + 57min) == minutes{10}); +} + +// ---- parseCreatedDateMicros: IG's "yyyy/MM/dd HH:mm:ss:fff" ---------------- + +TEST_CASE("parseCreatedDateMicros decodes IG's createdDate as UTC micros", + "[positionSync]") { + CHECK(positions::parseCreatedDateMicros("2026/07/10 21:55:00:123", 42) + == microsAt(sys_days{2026y / 7 / 10}, 21h, 55min, 0s, + milliseconds{123})); + CHECK(positions::parseCreatedDateMicros("1970/01/01 00:00:00:000", 42) + == 0); +} + +TEST_CASE("parseCreatedDateMicros falls back on anything malformed", + "[positionSync]") { + constexpr std::int64_t fallback = 42; + CHECK(positions::parseCreatedDateMicros("", fallback) == fallback); + CHECK(positions::parseCreatedDateMicros("not a date", fallback) + == fallback); + // Truncated (no millis). + CHECK(positions::parseCreatedDateMicros("2026/07/10 21:55:00", fallback) + == fallback); + // Wrong separators (ISO dashes). + CHECK(positions::parseCreatedDateMicros("2026-07-10 21:55:00:123", + fallback) + == fallback); + // Out-of-range fields. + CHECK(positions::parseCreatedDateMicros("2026/13/10 21:55:00:123", + fallback) + == fallback); + CHECK(positions::parseCreatedDateMicros("2026/02/30 21:55:00:123", + fallback) + == fallback); + CHECK(positions::parseCreatedDateMicros("2026/07/10 24:00:00:000", + fallback) + == fallback); + // Non-digits in a field. + CHECK(positions::parseCreatedDateMicros("2026/07/10 21:5x:00:123", + fallback) + == fallback); +} + +// ---- decodeAccountPositions: the C# AccountPositions shape ----------------- + +TEST_CASE("decodeAccountPositions reads the nested market/position fields", + "[positionSync]") { + const std::string body = R"({ + "positions": [{ + "position": { + "contractSize": 1.0, + "createdDate": "2026/07/10 09:30:00:250", + "dealId": "DIAAAA", + "dealSize": 1.5, + "dealReference": "ref-1", + "direction": "BUY", + "limitLevel": null, + "openLevel": 2345.678, + "currency": "GBP", + "controlledRisk": false, + "stopLevel": 2340.0 + }, + "market": { + "instrumentName": "Spot Gold", + "epic": "CS.D.CFPGOLD.CFP.IP", + "bid": 2345.5, + "offer": 2345.9 + } + }] + })"; + const auto decoded = positions::decodeAccountPositions(body); + REQUIRE(decoded.has_value()); + REQUIRE(decoded->size() == 1); + const positions::IgPosition& position = decoded->front(); + CHECK(position.epic == "CS.D.CFPGOLD.CFP.IP"); + CHECK(position.dealReference == "ref-1"); + CHECK(position.dealId == "DIAAAA"); + CHECK(position.direction == "BUY"); + CHECK(position.createdDate == "2026/07/10 09:30:00:250"); + CHECK(position.dealSize == 1.5); + CHECK(position.openLevel == 2345.678); + CHECK(position.stopLevel == 2340.0); + CHECK(position.limitLevel == 0.0); // null -> 0, the C# `?? 0` +} + +TEST_CASE("decodeAccountPositions tolerates missing sub-objects per entry", + "[positionSync]") { + const auto decoded = positions::decodeAccountPositions( + R"({"positions":[{"position":{"dealReference":"ref-2"}},{}]})"); + REQUIRE(decoded.has_value()); + REQUIRE(decoded->size() == 2); + CHECK(decoded->at(0).dealReference == "ref-2"); + CHECK(decoded->at(0).epic.empty()); // no market object + CHECK(decoded->at(1).dealReference.empty()); +} + +TEST_CASE("decodeAccountPositions treats a null/absent positions list as an " + "empty book", + "[positionSync]") { + const auto absent = positions::decodeAccountPositions("{}"); + REQUIRE(absent.has_value()); + CHECK(absent->empty()); + const auto null = positions::decodeAccountPositions(R"({"positions":null})"); + REQUIRE(null.has_value()); + CHECK(null->empty()); + const auto empty = + positions::decodeAccountPositions(R"({"positions":[]})"); + REQUIRE(empty.has_value()); + CHECK(empty->empty()); +} + +TEST_CASE("decodeAccountPositions rejects undecodable bodies", + "[positionSync]") { + CHECK_FALSE(positions::decodeAccountPositions("not json").has_value()); + CHECK_FALSE(positions::decodeAccountPositions("[]").has_value()); + CHECK_FALSE( + positions::decodeAccountPositions(R"({"positions":42})").has_value()); + CHECK_FALSE(positions::decodeAccountPositions(R"({"positions":["x"]})") + .has_value()); +} + +// ---- makeFreshRecord: the C# Save() ---------------------------------------- + +TEST_CASE("makeFreshRecord scales prices with the injected multiplier", + "[positionSync]") { + const auto record = positions::makeFreshRecord( + samplePosition(), "XAUUSD", "uuid-1", "Fvg", 1000, 99); + CHECK(record.dealId == "DIAAAA"); + CHECK(record.dealReference == "ref-1"); + CHECK(record.symbol == "XAUUSD"); + CHECK(record.epic == "CS.D.CFPGOLD.CFP.IP"); + CHECK(record.direction == "BUY"); + CHECK(record.size == 1.5); + CHECK(record.level == 2345678); + CHECK(record.stopLevel == 2340000); + CHECK(record.limitLevel == 0); + CHECK(record.strategyId == "uuid-1"); + CHECK(record.strategyName == "Fvg"); + CHECK(record.openedAtMicros + == microsAt(sys_days{2026y / 7 / 10}, 9h, 30min, 0s, + milliseconds{250})); +} + +TEST_CASE("makeFreshRecord rounds half away from zero, like the C# ScalePrice", + "[positionSync]") { + // 0.25 is binary-exact, so 0.25 * 10 is EXACTLY 2.5 — a true halfway + // case (a banker's-rounding encoder would write 2). Away-from-zero is + // the C# MidpointRounding the wire contract expects. + positions::IgPosition position = samplePosition(); + position.openLevel = 0.25; // * 10 = 2.5 -> 3 + position.stopLevel = -0.25; // * 10 = -2.5 -> -3 (away from zero) + position.limitLevel = 0.75; // * 10 = 7.5 -> 8 + const auto record = + positions::makeFreshRecord(position, "EURUSD", "s", "n", 10, 0); + CHECK(record.level == 3); + CHECK(record.stopLevel == -3); + CHECK(record.limitLevel == 8); +} + +TEST_CASE("makeFreshRecord writes level 0 when there is no price scale", + "[positionSync]") { + const auto record = positions::makeFreshRecord(samplePosition(), "XAUUSD", + "s", "n", 0, 0); + CHECK(record.level == 0); + CHECK(record.stopLevel == 0); + CHECK(record.limitLevel == 0); +} + +TEST_CASE("makeFreshRecord stamps openedAt with the fallback when " + "createdDate is malformed", + "[positionSync]") { + positions::IgPosition position = samplePosition(); + position.createdDate = "garbage"; + constexpr std::int64_t nowMicros = 1234567890; + const auto record = positions::makeFreshRecord(position, "XAUUSD", "s", + "n", 1000, nowMicros); + CHECK(record.openedAtMicros == nowMicros); +} + +// ---- applyBrokerUpdate: the C# Update() ------------------------------------ + +TEST_CASE("applyBrokerUpdate fills a missing strategy attribution", + "[positionSync]") { + redis_positions::PositionRecord record; + record.strategyId = ""; + positions::applyBrokerUpdate(record, samplePosition(), "uuid-1"); + CHECK(record.strategyId == "uuid-1"); + + record.strategyId = "Unknown"; + positions::applyBrokerUpdate(record, samplePosition(), "uuid-2"); + CHECK(record.strategyId == "uuid-2"); +} + +TEST_CASE("applyBrokerUpdate keeps a real strategy attribution", + "[positionSync]") { + redis_positions::PositionRecord record; + record.strategyId = "uuid-original"; + positions::applyBrokerUpdate(record, samplePosition(), "uuid-receipt"); + CHECK(record.strategyId == "uuid-original"); +} + +TEST_CASE("applyBrokerUpdate refreshes the dealId only when IG reports one", + "[positionSync]") { + redis_positions::PositionRecord record; + record.dealId = "DI-OLD"; + positions::IgPosition position = samplePosition(); + position.dealId = ""; + positions::applyBrokerUpdate(record, position, "s"); + CHECK(record.dealId == "DI-OLD"); + + position.dealId = "DI-NEW"; + positions::applyBrokerUpdate(record, position, "s"); + CHECK(record.dealId == "DI-NEW"); +} + +// ---- findMarketByEpicMini: reverse lookup self-consistency ----------------- + +TEST_CASE("every market is reachable from its own mini epic", + "[positionSync]") { + for (const live::MarketDefinition& market : live::kMarkets) { + INFO("epicMini not found: " << market.epicMini); + const live::MarketDefinition* found = + live::findMarketByEpicMini(market.epicMini); + REQUIRE(found != nullptr); + CHECK(found->epicMini == market.epicMini); + } +} + +TEST_CASE("an unknown epic finds no market", "[positionSync]") { + CHECK(live::findMarketByEpicMini("IX.D.NOTREAL.IFS.IP") == nullptr); + CHECK(live::findMarketByEpicMini("") == nullptr); +} diff --git a/tests/rangeBar.cpp b/tests/rangeBar.cpp new file mode 100644 index 0000000..b530cd2 --- /dev/null +++ b/tests/rangeBar.cpp @@ -0,0 +1,326 @@ +#include + +#include +#include +#include +#include // setenv — keep the builder off QuestDB +#include +#include +#include + +import rangeBarBuilder; +import ohlcObject; +import priceData; + +namespace { + +using std::chrono::hours; +using std::chrono::microseconds; +using std::chrono::minutes; +using std::chrono::seconds; + +// Fixed simulation start; the construct is deliberately clock-free, so only +// the ORDER of ticks matters — offsets exist to give bars distinct dates. +const std::chrono::system_clock::time_point t0 = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; + +// Range bars build from the ASK; the bid rides 2 points under and is ignored. +PriceData tickAt(std::chrono::system_clock::duration offset, std::int32_t ask, + const std::string& symbol = "EURUSD") { + return PriceData(ask, ask - 2, t0 + offset, symbol); +} + +// Hermetic: a RangeSeries' first update fires the QuestDB warm-up when the +// gate is on (it defaults ON), so force it off for the whole binary before +// main — same pattern as tests/ohlc.cpp. The gate tests below flip it +// explicitly and restore "0" when done. +[[maybe_unused]] const bool kPrepopulateForcedOff = [] { + setenv("OHLC_PREPOPULATE", "0", 1); + return true; +}(); + +// The workhorse spec: a 4-tick window at 100% makes windowRange, thresholds +// and warm-up boundaries hand-computable. +constexpr rangebar::RangeBarSpec kSpec{.atrTickWindow = 4, + .atrPercent = 100, + .count = 8}; + +} // namespace + +TEST_CASE("RangeSeries rejects non-positive spec fields", "[rangeBar]") { + using rangebar::RangeSeries; + CHECK_THROWS_AS(RangeSeries({0, 100, 8}), std::invalid_argument); + CHECK_THROWS_AS(RangeSeries({-1, 100, 8}), std::invalid_argument); + CHECK_THROWS_AS(RangeSeries({4, 0, 8}), std::invalid_argument); + CHECK_THROWS_AS(RangeSeries({4, -100, 8}), std::invalid_argument); + CHECK_THROWS_AS(RangeSeries({4, 100, 0}), std::invalid_argument); + CHECK_THROWS_AS(RangeSeries({4, 100, -8}), std::invalid_argument); +} + +TEST_CASE("no bars form until the tick window is warm", "[rangeBar]") { + rangebar::RangeSeries s(kSpec); + + // Three ticks into a 4-tick window: measuring, not yet trading. + s.update(tickAt(seconds{0}, 100000)); + s.update(tickAt(seconds{1}, 100010)); + s.update(tickAt(seconds{2}, 100005)); + CHECK_FALSE(s.warm()); + CHECK(s.bars().empty()); + + // The 4th tick fills the window; bar #1 opens on this very tick. + s.update(tickAt(seconds{3}, 100008)); + CHECK(s.warm()); + REQUIRE(s.bars().size() == 1); + CHECK_FALSE(s.bars().front().complete); +} + +TEST_CASE("rolling max and min slide and expire after exactly N ticks", + "[rangeBar]") { + rangebar::RangeSeries s(kSpec); + + s.update(tickAt(seconds{0}, 100000)); + s.update(tickAt(seconds{1}, 100200)); // the spike, tick #2 + s.update(tickAt(seconds{2}, 100010)); + s.update(tickAt(seconds{3}, 100020)); + CHECK(s.windowRange() == 200); // spike high vs tick-1 low + + // Tick #5 expires tick #1: the low rises to 100010, spike still in. + s.update(tickAt(seconds{4}, 100030)); + CHECK(s.windowRange() == 190); + + // Tick #6 = spike + window: the spike stops influencing the measure on + // exactly this tick — no clock involved, purely a count of events. + s.update(tickAt(seconds{5}, 100040)); + CHECK(s.windowRange() == 30); +} + +TEST_CASE("first bar opens on the first warm tick with its threshold locked", + "[rangeBar]") { + rangebar::RangeSeries s(kSpec); + s.update(tickAt(seconds{0}, 100000)); + s.update(tickAt(seconds{1}, 100002)); + s.update(tickAt(seconds{2}, 100001)); + s.update(tickAt(seconds{3}, 100010)); // warm; this tick IS the window max + + REQUIRE(s.bars().size() == 1); + const OhlcObject& bar = s.bars().front(); + CHECK(bar.date == t0 + seconds{3}); + CHECK(bar.open == 100010); + CHECK(bar.high == 100010); + CHECK(bar.low == 100010); + CHECK(bar.close == 100010); + CHECK_FALSE(bar.complete); + // The opening tick's own price counts: windowRange includes 100010, so + // the locked threshold is 10, not the pre-tick 2. + CHECK(s.lockedThreshold() == 10); +} + +TEST_CASE("intra-threshold ticks fold into the in-progress bar", "[rangeBar]") { + rangebar::RangeSeries s(kSpec); + s.update(tickAt(seconds{0}, 100000)); + s.update(tickAt(seconds{1}, 100002)); + s.update(tickAt(seconds{2}, 100001)); + s.update(tickAt(seconds{3}, 100010)); // bar opens, threshold 10 + + s.update(tickAt(seconds{4}, 100015)); + s.update(tickAt(seconds{5}, 100008)); + + REQUIRE(s.bars().size() == 1); + const OhlcObject& bar = s.bars().front(); + CHECK(bar.open == 100010); + CHECK(bar.high == 100015); + CHECK(bar.low == 100008); + CHECK(bar.close == 100008); + CHECK_FALSE(bar.complete); // high - low = 7 < 10 +} + +TEST_CASE("the breaching tick closes the bar and the next tick opens a new one", + "[rangeBar]") { + rangebar::RangeSeries s(kSpec); + s.update(tickAt(seconds{0}, 100000)); + s.update(tickAt(seconds{1}, 100002)); + s.update(tickAt(seconds{2}, 100001)); + s.update(tickAt(seconds{3}, 100010)); // bar opens, threshold 10 + s.update(tickAt(seconds{4}, 100014)); + + // high - low hits the threshold ON this tick: included and closed at its + // real price, immediately — not lazily when the successor opens. + s.update(tickAt(seconds{5}, 100020)); + REQUIRE(s.bars().size() == 1); + CHECK(s.bars().front().complete); + CHECK(s.bars().front().high == 100020); + CHECK(s.bars().front().low == 100010); + CHECK(s.bars().front().close == 100020); + + // The successor is seeded entirely from the NEXT tick's real price. + s.update(tickAt(seconds{6}, 100018)); + REQUIRE(s.bars().size() == 2); + CHECK(s.bars()[0].complete); + CHECK_FALSE(s.bars()[1].complete); + CHECK(s.bars()[1].date == t0 + seconds{6}); + CHECK(s.bars()[1].open == 100018); + CHECK(s.bars()[1].high == 100018); + CHECK(s.bars()[1].low == 100018); + CHECK(s.bars()[1].close == 100018); +} + +TEST_CASE("a gap tick crossing many thresholds closes exactly one bar", + "[rangeBar]") { + rangebar::RangeSeries s(kSpec); + s.update(tickAt(seconds{0}, 100000)); + s.update(tickAt(seconds{1}, 100001)); + s.update(tickAt(seconds{2}, 100002)); + s.update(tickAt(seconds{3}, 100003)); // bar opens, threshold 3 + + // ~33x the threshold in one tick: ONE bar completes, keeping the whole + // overshoot as its real range — no phantom bars fill the gap. + s.update(tickAt(seconds{4}, 100103)); + REQUIRE(s.bars().size() == 1); + CHECK(s.bars().front().complete); + CHECK(s.bars().front().close == 100103); + CHECK(s.bars().front().high - s.bars().front().low == 100); + + // And the gap has already widened the measure: the successor locks a + // threshold reflecting it (window {100002,100003,100103,100104} = 102). + s.update(tickAt(seconds{5}, 100104)); + REQUIRE(s.bars().size() == 2); + CHECK(s.lockedThreshold() == 102); +} + +TEST_CASE("the locked threshold never moves mid-bar; the next bar adapts " + "instantly", "[rangeBar]") { + rangebar::RangeSeries s(kSpec); + s.update(tickAt(seconds{0}, 100000)); + s.update(tickAt(seconds{1}, 100001)); + s.update(tickAt(seconds{2}, 100002)); + s.update(tickAt(seconds{3}, 100003)); // bar 1 opens, threshold 3 + CHECK(s.lockedThreshold() == 3); + + // The measure drifts to 4 (window {100001..100005}) but the open bar's + // target stays the locked 3. + s.update(tickAt(seconds{4}, 100005)); + CHECK(s.windowRange() == 4); + CHECK(s.lockedThreshold() == 3); + CHECK_FALSE(s.bars().front().complete); + + // The bar closes on its ORIGINAL threshold (range 3), even though the + // floating measure says 4 — that is what lock-at-open means. + s.update(tickAt(seconds{5}, 100006)); + REQUIRE(s.bars().size() == 1); + CHECK(s.bars().front().complete); + CHECK(s.bars().front().high - s.bars().front().low == 3); + + // A spike on the very next tick: the successor opens with a threshold + // that already reflects it. No time bucket to wait out — the design's + // whole point. + s.update(tickAt(seconds{6}, 100100)); + REQUIRE(s.bars().size() == 2); + CHECK(s.lockedThreshold() == 97); // window {100003,100005,100006,100100} +} + +TEST_CASE("the threshold floors at one point", "[rangeBar]") { + // 1% of a dead-flat window rounds to 0; the floor keeps it at 1 point. + rangebar::RangeSeries s({.atrTickWindow = 4, .atrPercent = 1, .count = 8}); + for (int i = 0; i < 4; ++i) { + s.update(tickAt(seconds{i}, 100000)); + } + REQUIRE(s.bars().size() == 1); + CHECK(s.lockedThreshold() == 1); + + SECTION("a dead-flat stream holds one open bar, never a bar per tick") { + for (int i = 4; i < 20; ++i) { + s.update(tickAt(seconds{i}, 100000)); + } + REQUIRE(s.bars().size() == 1); + CHECK_FALSE(s.bars().front().complete); + } + + SECTION("a one-point move meets the floored threshold") { + s.update(tickAt(seconds{4}, 100001)); + REQUIRE(s.bars().size() == 1); + CHECK(s.bars().front().complete); + } +} + +TEST_CASE("identical price sequences build identical bars regardless of tick " + "spacing", "[rangeBar]") { + const std::vector prices = {100000, 100050, 100010, 100020, + 100030, 100060, 100005, 100040}; + + // Same prices, wildly different clocks: microseconds apart vs hours + // apart (spanning days). The clock never enters the construct. + rangebar::RangeSeries fast(kSpec); + rangebar::RangeSeries slow(kSpec); + for (std::size_t i = 0; i < prices.size(); ++i) { + fast.update(tickAt(microseconds{i}, prices[i])); + slow.update(tickAt(hours{7 * i}, prices[i])); + } + + REQUIRE(fast.bars().size() == slow.bars().size()); + REQUIRE_FALSE(fast.bars().empty()); + for (std::size_t i = 0; i < fast.bars().size(); ++i) { + CHECK(fast.bars()[i].open == slow.bars()[i].open); + CHECK(fast.bars()[i].high == slow.bars()[i].high); + CHECK(fast.bars()[i].low == slow.bars()[i].low); + CHECK(fast.bars()[i].close == slow.bars()[i].close); + CHECK(fast.bars()[i].complete == slow.bars()[i].complete); + } + CHECK(fast.lockedThreshold() == slow.lockedThreshold()); +} + +TEST_CASE("prepopulateTicksQuery pins the QuestDB SQL shape", "[rangeBar]") { + // Bounds are hardcoded, not recomputed with the same arithmetic: t0 is + // 2026-01-05T09:00Z = 1767603600000000us, and the cap for {4, 100, 8} is + // 4 (exact window warm-up) + 2*8*4 (bar-formation margin) = 68. + CHECK(rangebar::prepopulateTicksQuery("EURUSD", t0, kSpec) == + "SELECT 'EURUSD' as symbol, ask, bid, timestamp FROM 'EURUSD' " + "WHERE timestamp < cast(1767603600000000L AS timestamp) " + "ORDER BY timestamp DESC LIMIT 68"); +} + +TEST_CASE("prepopulation gate off falls back to a cold start", "[rangeBar]") { + // Explicit "0", not unsetenv: the gate defaults ON, so unset means on. + setenv("OHLC_PREPOPULATE", "0", 1); + + // A 1-tick window is warm from the very first tick, so the cold start is + // visible immediately: one bar seeded from the live tick, no DB rows. + rangebar::RangeSeries s({.atrTickWindow = 1, .atrPercent = 100, .count = 4}); + s.update(tickAt(seconds{0}, 100000)); + + REQUIRE(s.bars().size() == 1); + CHECK(s.bars().front().date == t0); + CHECK(s.bars().front().open == 100000); + CHECK_FALSE(s.bars().front().complete); +} + +TEST_CASE("unknown symbol yields empty without touching the DB", "[rangeBar]") { + // The symbol guard precedes any connection, so this passes gate-on with + // no QuestDB running (and doubles as the SQL-injection whitelist check). + setenv("OHLC_PREPOPULATE", "1", 1); + const auto ticks = rangebar::prepopulateTicks("NOPE", t0, kSpec); + setenv("OHLC_PREPOPULATE", "0", 1); // back to the binary's hermetic state + + CHECK(ticks.empty()); +} + +// Hidden ([.]) — needs a live QuestDB with EURUSD ticks. Run explicitly: +// ./build/tests/unit_tests "[dblive]" +TEST_CASE("prepopulateTicks fetches ascending ticks from a live QuestDB", + "[.][dblive]") { + const rangebar::RangeBarSpec spec{.atrTickWindow = 1000, + .atrPercent = 100, + .count = 24}; + setenv("OHLC_PREPOPULATE", "1", 1); + const auto ticks = rangebar::prepopulateTicks( + "EURUSD", std::chrono::system_clock::now(), spec); + setenv("OHLC_PREPOPULATE", "0", 1); // back to the binary's hermetic state + + REQUIRE_FALSE(ticks.empty()); + CHECK(std::cmp_less_equal(ticks.size(), rangebar::prepopulateTickCap(spec))); + CHECK(std::ranges::is_sorted(ticks, {}, &PriceData::timestamp)); + for (const PriceData& tick : ticks) { + CHECK(tick.ask > 0); // scaled INT prices parsed, not garbage + CHECK(tick.bid > 0); + CHECK(tick.symbol == "EURUSD"); + } +} diff --git a/tests/rangeVelocity.cpp b/tests/rangeVelocity.cpp new file mode 100644 index 0000000..2982040 --- /dev/null +++ b/tests/rangeVelocity.cpp @@ -0,0 +1,555 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// RangeVelocityStrategy: velocity momentum on range bars. The fixture makes +// every timing semantic hand-computable: a 2-tick rolling window at 100% with +// every tick stepping exactly +/-10 points locks every bar's threshold at 10, +// so EVERY bar is exactly two ticks — an opening tick and a breach tick whose +// step sign is the bar's direction. The successor's opening tick reuses the +// breach tick's timestamp (the bar construct is clock-free; only order +// matters), so a bar's open-to-open duration equals its scripted formation +// gap, which also equals the newest bar's tick-measured duration. + +#include + +#include +#include +#include // setenv — keep the store off QuestDB +#include +#include +#include +#include + +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import rangeVelocityStrategy; +import barStore; // bars::BarStore — the strategy reads range bars from it +import priceData; +import rangeBarBuilder; // rangebar::RangeBarSpec — series registration +import trade; +import tradeManager; + +namespace { + +using std::chrono::hours; +using std::chrono::microseconds; +using std::chrono::minutes; +using std::chrono::seconds; + +const std::chrono::system_clock::time_point t0 = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; + +// Plain tick for the direct during() calls (time-cap tests). Bid rides 2 +// points under the ask, so a LONG's exit-side close price is ask - 2. +PriceData tickAt(std::chrono::system_clock::duration offset, std::int32_t ask, + const std::string& symbol = "EURUSD") { + return PriceData(ask, ask - 2, t0 + offset, symbol); +} + +// Defaults match the hand-computation recipe: K=2 run bars, M=3 baseline +// bars, ratio 100 ("at the norm" passes), E=2 exit bars, time cap off, +// RANGE_COUNT 7 >= max(K+M, E). +tradingDefinitions::StrategyConfig makeConfig(const int runBars = 2, + const int speedLookback = 3, + const int ratio = 100, + const int exitRun = 2, + const int maxDurationMinutes = 0, + const int rangeCount = 7, + const int tickWindow = 2, + const int percent = 100) { + tradingDefinitions::StrategyConfig config; + config.UUID = "range-velocity-test"; + config.TRADING_VARIABLES.STRATEGY = "RangeVelocityStrategy"; + config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR = 25; + config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR = 50; + config.TRADING_VARIABLES.TRADING_SIZE = 1; + config.RANGE_VARIABLES = {{.RANGE_ATR_TICK_WINDOW = tickWindow, + .RANGE_ATR_PERCENT = percent, + .RANGE_COUNT = rangeCount}}; + config.STRATEGY_VARIABLES.RANGE_VELOCITY_VARIABLES = + tradingDefinitions::RangeVelocityVariables{ + .RUN_BARS = runBars, + .SPEED_LOOKBACK_BARS = speedLookback, + .SPEED_RATIO_PERCENT = ratio, + .EXIT_RUN_BARS = exitRun, + .MAX_TRADE_DURATION_MINUTES = maxDurationMinutes}; + return config; +} + +bars::BarStore makeStore(const int rangeCount = 7) { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + bars::BarStore store; + store.registerRangeSeries(rangebar::RangeBarSpec{ + .atrTickWindow = 2, .atrPercent = 100, .count = rangeCount}); + return store; +} + +// One tick in run-loop order: shared bars first, then decide, then the +// management hook — mirroring runTicks exactly. +std::optional step(RangeVelocityStrategy& strategy, TradeManager& tm, + bars::BarStore& store, const PriceData& tick) { + store.update(tick); + const auto signal = strategy.decide(tick, store); + strategy.during(tick, store, tm); + return signal; +} + +// Per-symbol scripted price stream. `lastBreach` is kept so tests can open +// trades at exactly the tick a signal fired on, the way runTicks would. +struct SymbolStream { + std::string symbol = "EURUSD"; + std::int32_t price = 100000; + std::chrono::system_clock::time_point clock = t0; + bool warmed = false; + PriceData lastBreach{}; +}; + +// Emits one complete range bar of direction `dir` (+1 up, -1 down) taking +// `formation` of tick time, and returns the BREACH tick's decide() result. +// The opening tick can never signal (the just-closed gate is down), which is +// asserted inline. The very first call emits one extra warming tick — a +// 2-tick window needs one predecessor before bar #1 can open. +std::optional emitBar(RangeVelocityStrategy& strategy, + TradeManager& tm, bars::BarStore& store, + SymbolStream& s, const int dir, + const std::chrono::microseconds formation) { + if (!s.warmed) { + CHECK_FALSE(step(strategy, tm, store, + PriceData(s.price, s.price - 2, s.clock, s.symbol)) + .has_value()); + s.warmed = true; + } + s.price += 10 * dir; // opening tick: timestamp of the previous breach + CHECK_FALSE(step(strategy, tm, store, + PriceData(s.price, s.price - 2, s.clock, s.symbol)) + .has_value()); + s.price += 10 * dir; // breach tick: the bar's direction and duration + s.clock += formation; + s.lastBreach = PriceData(s.price, s.price - 2, s.clock, s.symbol); + return step(strategy, tm, store, s.lastBreach); +} + +} // namespace + +TEST_CASE("RangeVelocityStrategy rejects malformed configuration", + "[rangeVelocity]") { + SECTION("empty RANGE_VARIABLES") { + auto config = makeConfig(); + config.RANGE_VARIABLES.clear(); + CHECK_THROWS_AS(RangeVelocityStrategy{config}, std::invalid_argument); + } + SECTION("zero-sentinel fields in RANGE_VARIABLES[0]") { + CHECK_THROWS_AS( + RangeVelocityStrategy{makeConfig(2, 3, 100, 2, 0, 7, 0, 100)}, + std::invalid_argument); + CHECK_THROWS_AS( + RangeVelocityStrategy{makeConfig(2, 3, 100, 2, 0, 7, 2, 0)}, + std::invalid_argument); + CHECK_THROWS_AS( + RangeVelocityStrategy{makeConfig(2, 3, 100, 2, 0, 0)}, + std::invalid_argument); + } + SECTION("missing RANGE_VELOCITY_VARIABLES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.RANGE_VELOCITY_VARIABLES = std::nullopt; + CHECK_THROWS_AS(RangeVelocityStrategy{config}, std::invalid_argument); + } + SECTION("non-positive strategy knobs") { + CHECK_THROWS_AS(RangeVelocityStrategy{makeConfig(0)}, + std::invalid_argument); + CHECK_THROWS_AS(RangeVelocityStrategy{makeConfig(2, 0)}, + std::invalid_argument); + CHECK_THROWS_AS(RangeVelocityStrategy{makeConfig(2, 3, 0)}, + std::invalid_argument); + CHECK_THROWS_AS(RangeVelocityStrategy{makeConfig(2, 3, 100, 0)}, + std::invalid_argument); + CHECK_THROWS_AS(RangeVelocityStrategy{makeConfig(2, 3, 100, 2, -1)}, + std::invalid_argument); + } + SECTION("RANGE_COUNT below the provable window bound") { + // K + M dominates: 2 + 3 = 5, count 4 is one short. + CHECK_THROWS_AS(RangeVelocityStrategy{makeConfig(2, 3, 100, 2, 0, 4)}, + std::invalid_argument); + // EXIT_RUN_BARS dominates: max(1 + 1, 7) = 7, count 6 is one short. + CHECK_THROWS_AS(RangeVelocityStrategy{makeConfig(1, 1, 100, 7, 0, 6)}, + std::invalid_argument); + // Exactly at the bound constructs. + CHECK_NOTHROW(RangeVelocityStrategy{makeConfig(2, 3, 100, 2, 0, 5)}); + } +} + +TEST_CASE("silent until RUN_BARS + SPEED_LOOKBACK_BARS bars have closed", + "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + + // Four fast up bars: run and speed would qualify, but the baseline + // cannot exist yet — every breach stays silent while n < K + M = 5. + for (int i = 0; i < 4; ++i) { + CHECK_FALSE(emitBar(strategy, tm, store, s, +1, seconds{5}).has_value()); + } + // The fifth closed bar is the first legal evaluation — and this stream + // qualifies (baseline median 5s, run at 5s, ratio 100). + CHECK(emitBar(strategy, tm, store, s, +1, seconds{5}) == Direction::LONG); +} + +TEST_CASE("LONG on K consecutive fast up bars after a slow baseline", + "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + + // Baseline: three 60s bars, directions deliberately mixed — only the + // run bars' directions matter. + CHECK_FALSE(emitBar(strategy, tm, store, s, +1, seconds{60}).has_value()); + CHECK_FALSE(emitBar(strategy, tm, store, s, -1, seconds{60}).has_value()); + CHECK_FALSE(emitBar(strategy, tm, store, s, +1, seconds{60}).has_value()); + // First run bar: fast, but n = 4 < 5 — still warming. + CHECK_FALSE(emitBar(strategy, tm, store, s, +1, seconds{5}).has_value()); + // Second run bar: 12x faster than the 60s norm — LONG on this breach. + CHECK(emitBar(strategy, tm, store, s, +1, seconds{5}) == Direction::LONG); +} + +TEST_CASE("SHORT mirror on fast down bars", "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + + CHECK_FALSE(emitBar(strategy, tm, store, s, -1, seconds{60}).has_value()); + CHECK_FALSE(emitBar(strategy, tm, store, s, +1, seconds{60}).has_value()); + CHECK_FALSE(emitBar(strategy, tm, store, s, -1, seconds{60}).has_value()); + CHECK_FALSE(emitBar(strategy, tm, store, s, -1, seconds{5}).has_value()); + CHECK(emitBar(strategy, tm, store, s, -1, seconds{5}) == Direction::SHORT); +} + +TEST_CASE("fires exactly once per bar close, never on stale state", + "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, +1, seconds{60}); + } + emitBar(strategy, tm, store, s, +1, seconds{5}); + REQUIRE(emitBar(strategy, tm, store, s, +1, seconds{5}) == Direction::LONG); + + // The successor's opening tick lands on the same just-closed series — + // but the in-progress bar is back on top, so the gate is down again. + s.price += 10; + const PriceData openingTick(s.price, s.price - 2, s.clock, s.symbol); + CHECK_FALSE(step(strategy, tm, store, openingTick).has_value()); + + // Its breach re-evaluates freshly (the run extended): a NEW signal, one + // per bar close — never a replay of the old one mid-bar. + s.price += 10; + s.clock += seconds{5}; + const PriceData breachTick(s.price, s.price - 2, s.clock, s.symbol); + CHECK(step(strategy, tm, store, breachTick) == Direction::LONG); +} + +TEST_CASE("a slower-than-allowed bar anywhere in the run blocks entry", + "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, +1, seconds{60}); + } + + SECTION("the newest run bar is too slow") { + emitBar(strategy, tm, store, s, +1, seconds{5}); + CHECK_FALSE( + emitBar(strategy, tm, store, s, +1, seconds{61}).has_value()); + } + SECTION("the earlier run bar is too slow") { + emitBar(strategy, tm, store, s, +1, seconds{61}); + CHECK_FALSE( + emitBar(strategy, tm, store, s, +1, seconds{5}).has_value()); + } +} + +TEST_CASE("an against-direction bar inside the run blocks entry", + "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, +1, seconds{60}); + } + + // Both run bars are fast, but they disagree on direction. + emitBar(strategy, tm, store, s, -1, seconds{5}); + CHECK_FALSE(emitBar(strategy, tm, store, s, +1, seconds{5}).has_value()); +} + +TEST_CASE("the speed-ratio boundary admits exactly <=", "[rangeVelocity]") { + // Ratio 50 over a 60s median allows exactly 30s per run bar. + TradeManager tm; + + SECTION("exactly at the allowance enters") { + RangeVelocityStrategy strategy{makeConfig(2, 3, 50)}; + auto store = makeStore(); + SymbolStream s; + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, +1, seconds{60}); + } + emitBar(strategy, tm, store, s, +1, seconds{30}); + CHECK(emitBar(strategy, tm, store, s, +1, seconds{30}) == + Direction::LONG); + } + SECTION("one microsecond over does not") { + RangeVelocityStrategy strategy{makeConfig(2, 3, 50)}; + auto store = makeStore(); + SymbolStream s; + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, +1, seconds{60}); + } + emitBar(strategy, tm, store, s, +1, seconds{30}); + CHECK_FALSE(emitBar(strategy, tm, store, s, +1, + seconds{30} + microseconds{1}) + .has_value()); + } +} + +TEST_CASE("session gaps: the median absorbs a baseline gap, a run gap " + "refuses entry", "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + + SECTION("a weekend bar in the baseline leaves the median at the norm") { + emitBar(strategy, tm, store, s, +1, seconds{60}); + emitBar(strategy, tm, store, s, -1, hours{72}); // the gap bar + emitBar(strategy, tm, store, s, +1, seconds{60}); + emitBar(strategy, tm, store, s, +1, seconds{5}); + // Median of {60s, 72h, 60s} is 60s — the gap never poisons it. + CHECK(emitBar(strategy, tm, store, s, +1, seconds{5}) == + Direction::LONG); + } + SECTION("a run bar spanning the gap fails the speed test") { + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, +1, seconds{60}); + } + emitBar(strategy, tm, store, s, +1, seconds{5}); + // The first bars after a session gap can never chase it. + CHECK_FALSE( + emitBar(strategy, tm, store, s, +1, hours{72}).has_value()); + } +} + +TEST_CASE("during() closes on an opposite run of E closed bars — speed " + "never enters exits", "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + + SECTION("LONG closed at the bid on the Eth against-bar's breach") { + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, +1, seconds{60}); + } + emitBar(strategy, tm, store, s, +1, seconds{5}); + REQUIRE(emitBar(strategy, tm, store, s, +1, seconds{5}) == + Direction::LONG); + tm.openTrade(s.lastBreach, 1, Direction::LONG); + + // First against-bar: the last two closed are [up, down] — open. + // Deliberately SLOW bars: exits carry no speed filter. + emitBar(strategy, tm, store, s, -1, seconds{600}); + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + + // Second against-bar completes the opposite run: closed at the bid. + emitBar(strategy, tm, store, s, -1, seconds{600}); + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == s.lastBreach.bid); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + SECTION("SHORT mirror closes at the ask") { + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, -1, seconds{60}); + } + emitBar(strategy, tm, store, s, -1, seconds{5}); + REQUIRE(emitBar(strategy, tm, store, s, -1, seconds{5}) == + Direction::SHORT); + tm.openTrade(s.lastBreach, 1, Direction::SHORT); + + emitBar(strategy, tm, store, s, +1, seconds{600}); + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + emitBar(strategy, tm, store, s, +1, seconds{600}); + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == s.lastBreach.ask); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } +} + +TEST_CASE("the opposite run must be unbroken", "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, +1, seconds{60}); + } + emitBar(strategy, tm, store, s, +1, seconds{5}); + REQUIRE(emitBar(strategy, tm, store, s, +1, seconds{5}) == Direction::LONG); + tm.openTrade(s.lastBreach, 1, Direction::LONG); + + // down, up, down: never two consecutive against-bars — stays open the + // whole way (the intervening opening ticks can't close either: the + // just-closed gate is down on them). + emitBar(strategy, tm, store, s, -1, seconds{60}); + emitBar(strategy, tm, store, s, +1, seconds{60}); + emitBar(strategy, tm, store, s, -1, seconds{60}); + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); +} + +// The time-cap tests drive during() directly: its exit path is independent +// of bar state (an empty store just skips the opposite-run leg), and trades +// are opened straight on the TradeManager — exactly how the live book seeds +// them. +TEST_CASE("RangeVelocityStrategy closes trades past the max duration via " + "during()", "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig(2, 3, 100, 2, 60)}; + TradeManager tm; + + SECTION("LONG past the cap closes at the bid") { + tm.openTrade(tickAt(seconds{0}, 110000), 1, Direction::LONG); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050), + bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110048); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + SECTION("SHORT past the cap closes at the ask") { + tm.openTrade(tickAt(seconds{0}, 110000), 1, Direction::SHORT); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050), + bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110050); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + SECTION("exactly at the cap stays open (strictly greater)") { + tm.openTrade(tickAt(seconds{0}, 110000), 1, Direction::LONG); + strategy.during(tickAt(minutes{60}, 110050), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } + SECTION("another symbol's tick never closes it") { + tm.openTrade(tickAt(seconds{0}, 110000), 1, Direction::LONG); + strategy.during(tickAt(minutes{120}, 65030, "AUDUSD"), + bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } +} + +TEST_CASE("RangeVelocityStrategy max duration of zero disables the time cap", + "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + + tm.openTrade(tickAt(seconds{0}, 110000), 1, Direction::LONG); + strategy.during(tickAt(minutes{600}, 110050), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); +} + +TEST_CASE("re-signals on the next fast close after a stop-out mid-run", + "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream s; + + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, s, +1, seconds{60}); + } + emitBar(strategy, tm, store, s, +1, seconds{5}); + REQUIRE(emitBar(strategy, tm, store, s, +1, seconds{5}) == Direction::LONG); + tm.openTrade(s.lastBreach, 1, Direction::LONG); + + // Simulated stop-out: the broker/SL closed it, not the strategy. + tm.closeTrade("EURUSD", s.lastBreach.bid, s.lastBreach); + REQUIRE_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + + // The run is still alive and fast — the next bar close re-signals + // (one-trade-per-symbol gating is the run loop's job, not decide()'s). + CHECK(emitBar(strategy, tm, store, s, +1, seconds{5}) == Direction::LONG); +} + +TEST_CASE("per-symbol streams signal independently", "[rangeVelocity]") { + RangeVelocityStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(); + SymbolStream eur; + SymbolStream aud{.symbol = "AUDUSD", .price = 65000}; + + // Interleave the two streams; each keeps its own bar history. + for (int i = 0; i < 3; ++i) { + emitBar(strategy, tm, store, eur, +1, seconds{60}); + emitBar(strategy, tm, store, aud, -1, seconds{60}); + } + emitBar(strategy, tm, store, eur, +1, seconds{5}); + emitBar(strategy, tm, store, aud, -1, seconds{5}); + CHECK(emitBar(strategy, tm, store, eur, +1, seconds{5}) == Direction::LONG); + CHECK(emitBar(strategy, tm, store, aud, -1, seconds{5}) == + Direction::SHORT); +} + +TEST_CASE("StrategyVariables round-trips RANGE_VELOCITY_VARIABLES through " + "JSON", "[rangeVelocity]") { + SECTION("a present group survives the round trip") { + tradingDefinitions::StrategyVariables vars; + vars.RANGE_VELOCITY_VARIABLES = tradingDefinitions::RangeVelocityVariables{ + .RUN_BARS = 3, + .SPEED_LOOKBACK_BARS = 32, + .SPEED_RATIO_PERCENT = 60, + .EXIT_RUN_BARS = 2, + .MAX_TRADE_DURATION_MINUTES = 240}; + + const nlohmann::json j = vars; + const auto parsed = j.get(); + REQUIRE(parsed.RANGE_VELOCITY_VARIABLES.has_value()); + CHECK(parsed.RANGE_VELOCITY_VARIABLES->RUN_BARS == 3); + CHECK(parsed.RANGE_VELOCITY_VARIABLES->SPEED_LOOKBACK_BARS == 32); + CHECK(parsed.RANGE_VELOCITY_VARIABLES->SPEED_RATIO_PERCENT == 60); + CHECK(parsed.RANGE_VELOCITY_VARIABLES->EXIT_RUN_BARS == 2); + CHECK(parsed.RANGE_VELOCITY_VARIABLES->MAX_TRADE_DURATION_MINUTES == 240); + } + SECTION("an absent field parses with the zero default (WITH_DEFAULT)") { + const nlohmann::json j = {{"RUN_BARS", 3}, + {"SPEED_LOOKBACK_BARS", 32}, + {"SPEED_RATIO_PERCENT", 60}, + {"EXIT_RUN_BARS", 2}}; + const auto rv = j.get(); + CHECK(rv.RUN_BARS == 3); + CHECK(rv.MAX_TRADE_DURATION_MINUTES == 0); + } + SECTION("an absent group serialises as null and parses back absent") { + const nlohmann::json j = tradingDefinitions::StrategyVariables{}; + CHECK(j.at("RANGE_VELOCITY_VARIABLES").is_null()); + const auto parsed = j.get(); + CHECK_FALSE(parsed.RANGE_VELOCITY_VARIABLES.has_value()); + } +} diff --git a/tests/rollingWindow.cpp b/tests/rollingWindow.cpp new file mode 100644 index 0000000..f030651 --- /dev/null +++ b/tests/rollingWindow.cpp @@ -0,0 +1,149 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include + +#include +#include + +#include + +#include "shared/tradingDefinitions/config/configuration.hpp" +#include "shared/utilities/queueKeys.hpp" + +import rollingWindow; // rolling::nextWindow, rolling::nextRunConfiguration, + // rolling::queueKeyFor + +using namespace boost::decimal::literals; + +// The ladder: each completed 3-month slice advances to the slice before it, +// until 9 months of history are covered, then one final full-history run. +TEST_CASE("nextWindow walks the 3-month slices back through 9 months", "[rollingWindow]") { + const auto second = rolling::nextWindow(3, 0); + REQUIRE(second.has_value()); + CHECK(second->lastMonths == 3); + CHECK(second->offsetMonths == 3); + + const auto third = rolling::nextWindow(3, 3); + REQUIRE(third.has_value()); + CHECK(third->lastMonths == 3); + CHECK(third->offsetMonths == 6); +} + +// The last slice reaches the full 9 months back, so it advances to the +// terminal run over the whole 9 months of data. +TEST_CASE("nextWindow ends the slices with the full-history run", "[rollingWindow]") { + const auto full = rolling::nextWindow(3, 6); + REQUIRE(full.has_value()); + CHECK(full->lastMonths == 9); + CHECK(full->offsetMonths == 0); +} + +// The full-history run is terminal — no more runs are queued after it. +TEST_CASE("nextWindow stops after the full-history run", "[rollingWindow]") { + CHECK_FALSE(rolling::nextWindow(9, 0).has_value()); +} + +// Only windows exactly on the ladder chain: a hand-queued one-off sweep +// (whatever its window) must never start spawning follow-on runs. +TEST_CASE("nextWindow ignores off-ladder windows", "[rollingWindow]") { + CHECK_FALSE(rolling::nextWindow(6, 0).has_value()); + CHECK_FALSE(rolling::nextWindow(12, 0).has_value()); + CHECK_FALSE(rolling::nextWindow(3, 1).has_value()); + CHECK_FALSE(rolling::nextWindow(9, 6).has_value()); + CHECK_FALSE(rolling::nextWindow(0, 0).has_value()); +} + +// The advancing strategy must be judged under identical rules in every +// window, so the descriptor carries the finished run's symbols, batch +// identity and every risk cap — only RUN_ID and the window itself change. +TEST_CASE("nextRunConfiguration carries symbols and risk caps to the next window", "[rollingWindow]") { + tradingDefinitions::Configuration finished; + finished.RUN_ID = "origin-run"; + finished.SYMBOLS = "EURUSD,AUDUSD"; + finished.BATCH = "2099-01"; // must chain — the terminal {9,0} run's docs + finished.EXECUTION_TS = "2099-01-01T00:00:00Z"; // name the weekly index + finished.LAST_MONTHS = 3; + finished.OFFSET_MONTHS = 0; + finished.STARTING_BALANCE = "25000"_dd; + finished.MAX_LOSS_PERCENT = "7"_dd; + finished.MAX_OPEN_TRADES = 2; + finished.MAX_TRADES_PER_MINUTE = 30; + finished.REPORT_FAILURES = false; + finished.PEAK_HOURS_ONLY = true; // non-default so a dropped copy fails + finished.ENTRY_SLIPPAGE_TENTH_PIPS = 3; // ditto — the stress must chain + + const auto next = + rolling::nextRunConfiguration(finished, {3, 3}, "next-run"); + + CHECK(next.RUN_ID == std::string("next-run")); + CHECK(next.SYMBOLS == finished.SYMBOLS); + CHECK(next.BATCH == finished.BATCH); + CHECK(next.EXECUTION_TS == finished.EXECUTION_TS); + CHECK(next.LAST_MONTHS == 3); + CHECK(next.OFFSET_MONTHS == 3); + CHECK(next.STARTING_BALANCE == finished.STARTING_BALANCE); + CHECK(next.MAX_LOSS_PERCENT == finished.MAX_LOSS_PERCENT); + CHECK(next.MAX_OPEN_TRADES == finished.MAX_OPEN_TRADES); + CHECK(next.MAX_TRADES_PER_MINUTE == finished.MAX_TRADES_PER_MINUTE); + CHECK(next.REPORT_FAILURES == finished.REPORT_FAILURES); + CHECK(next.PEAK_HOURS_ONLY == finished.PEAK_HOURS_ONLY); + CHECK(next.ENTRY_SLIPPAGE_TENTH_PIPS == finished.ENTRY_SLIPPAGE_TENTH_PIPS); +} + +// Every rung of the ladder must itself be reachable from the sweep's seed +// window (3, 0) — a table typo that orphans a rung would silently truncate +// the chain, so walk it end to end. +TEST_CASE("the ladder chains from the seed window to the full-history run", "[rollingWindow]") { + int last = 3, offset = 0; + int steps = 0; + while (const auto next = rolling::nextWindow(last, offset)) { + last = next->lastMonths; + offset = next->offsetMonths; + ++steps; + REQUIRE(steps <= 10); // a cycle in the table must fail, not hang + } + CHECK(last == 9); + CHECK(offset == 0); + CHECK(steps == 3); +} + +// Workers drain queue_keys::RUN_QUEUES in order, so wave ordering holds only +// if each chained window lands exactly one queue deeper than the window that +// spawned it. Walk the ladder and check the mapping rung by rung; by the end +// every chain queue must have been used (an unreachable queue would mean the +// ladder and the queue list drifted apart). +TEST_CASE("each chained window lands one chain queue deeper", "[rollingWindow]") { + int last = 3, offset = 0; + std::size_t depth = 0; + while (const auto next = rolling::nextWindow(last, offset)) { + ++depth; + REQUIRE(depth < queue_keys::RUN_QUEUES.size()); + CHECK(rolling::queueKeyFor(*next) == queue_keys::RUN_QUEUES[depth]); + last = next->lastMonths; + offset = next->offsetMonths; + } + CHECK(depth == queue_keys::RUN_QUEUES.size() - 1); +} + +// Only ladder rungs belong on the chain queues: the seed window and any +// hand-queued one-off go on the shared RUN queue, top priority alongside +// fresh grid sweeps. +TEST_CASE("seed and off-ladder windows map to the shared RUN queue", "[rollingWindow]") { + CHECK(rolling::queueKeyFor({3, 0}) == queue_keys::RUN); + CHECK(rolling::queueKeyFor({6, 0}) == queue_keys::RUN); + CHECK(rolling::queueKeyFor({12, 0}) == queue_keys::RUN); + CHECK(rolling::queueKeyFor({3, 1}) == queue_keys::RUN); +} + +// The priority scan peeks each queue independently and LREMs the one a run +// was found on — two rungs sharing a key would break both the ordering and +// the retire. +TEST_CASE("the run queues are distinct keys", "[rollingWindow]") { + const std::set unique(queue_keys::RUN_QUEUES.begin(), + queue_keys::RUN_QUEUES.end()); + CHECK(unique.size() == queue_keys::RUN_QUEUES.size()); +} diff --git a/tests/sessionRangeBreakout.cpp b/tests/sessionRangeBreakout.cpp new file mode 100644 index 0000000..159eaa8 --- /dev/null +++ b/tests/sessionRangeBreakout.cpp @@ -0,0 +1,392 @@ +#include + +#include +#include +#include +#include // setenv — keep the bar store off QuestDB +#include +#include +#include +#include + +#include +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import sessionRangeBreakoutStrategy; +import barStore; // bars::BarStore — the strategy reads bars from it +import priceData; +import trade; +import tradeManager; + +namespace { + +using std::chrono::minutes; +using std::chrono::seconds; + +// Two UTC midnights, one per DST regime: 2026-01-05 is GMT (London opens +// 08:00), 2026-06-01 is BST (London opens 07:00). Both are Mondays, though +// the strategy itself never reads the weekday — that gate lives in the run +// loop (market_hours::tradePermitted). +const std::chrono::system_clock::time_point kWinterMidnight = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5}; +const std::chrono::system_clock::time_point kSummerMidnight = + std::chrono::sys_days{std::chrono::year{2026} / 6 / 1}; + +// Signal timeframe: 15m bars, window derived exactly like the sweep mapper — +// ceil((480 + entry window) / minutes) + 2 bars, the ctor minimum. +tradingDefinitions::StrategyConfig makeConfig(int bufferPips = 0, + int entryWindowMinutes = 120, + int maxTradeDurationMinutes = 0, + int ohlcMinutes = 15) { + tradingDefinitions::StrategyConfig config; + config.UUID = "test-session-range"; + config.TRADING_VARIABLES.STRATEGY = "SessionRangeBreakoutStrategy"; + config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.TRADING_SIZE = 1; + const int ohlcCount = + (480 + entryWindowMinutes + ohlcMinutes - 1) / ohlcMinutes + 2; + config.OHLC_VARIABLES = { + tradingDefinitions::OHLCVariables{.OHLC_COUNT = ohlcCount, + .OHLC_MINUTES = ohlcMinutes}, + }; + config.STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES = + tradingDefinitions::SessionRangeBreakoutVariables{ + .BUFFER_PIPS = bufferPips, + .ENTRY_WINDOW_MINUTES = entryWindowMinutes, + .MAX_TRADE_DURATION_MINUTES = maxTradeDurationMinutes}; + return config; +} + +PriceData tickAt(std::chrono::system_clock::time_point base, + std::chrono::system_clock::duration offset, std::int32_t ask, + std::int32_t bid, const std::string& symbol = "EURUSD") { + return PriceData(ask, bid, base + offset, symbol); +} + +// The loop owner's role in miniature: register every configured timeframe. +bars::BarStore makeStore(const tradingDefinitions::StrategyConfig& config) { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + bars::BarStore store; + for (const auto& ohlc : config.OHLC_VARIABLES) { + store.registerSeries(minutes{ohlc.OHLC_MINUTES}, ohlc.OHLC_COUNT); + } + return store; +} + +// One tick in run-loop order: the store update first, then decide, then the +// management hook — runTicks feeds the shared bars BEFORE the entry gates, +// so decide() judges the tick against bar state that already includes it. +std::optional step(SessionRangeBreakoutStrategy& strategy, + TradeManager& tm, bars::BarStore& store, + const PriceData& tick) { + store.update(tick); + const auto signal = strategy.decide(tick, store); + strategy.during(tick, store, tm); + return signal; +} + +// The exact OHLC a crafted 15m bar should end up with. +struct BarShape { + std::int32_t o, h, l, c; +}; + +// Four asks inside one 15m bar, fed open/high/low/close within 45 seconds of +// `offset`. Bars are FIRST-TICK anchored (ohlcBuilder rolls on the first tick +// strictly more than one duration past the bar's start), so consecutive +// feeds must sit more than 15 minutes apart to land in separate bars. Every +// feed tick is pre-open, so the clock gate guarantees no signal. +void feedBar(SessionRangeBreakoutStrategy& strategy, TradeManager& tm, + bars::BarStore& store, std::chrono::system_clock::time_point base, + minutes offset, const BarShape& bar, + const std::string& symbol = "EURUSD") { + const std::array, 4> ticks{{ + {seconds{0}, bar.o}, + {seconds{15}, bar.h}, + {seconds{30}, bar.l}, + {seconds{45}, bar.c}, + }}; + for (const auto& [tickOffset, ask] : ticks) { + CHECK_FALSE(step(strategy, tm, store, + tickAt(base, offset + tickOffset, ask, ask - 10, symbol)) + .has_value()); + } +} + +// Canonical session fixture around `base` (a UTC midnight): +// 23:40 (prev day) flat bar — satisfies the pre-midnight coverage gate +// 00:00/02:00/04:00/05:40 — the Asian session: high 110050, low 109950 +// 06:30 — a post-Asia bar that must stay OUT of the range (its high is the +// `postAsiaHigh` knob so a test can plant an extreme there); it +// also rolls the 05:40 bar closed before the entry window opens. +void feedSessionFixture(SessionRangeBreakoutStrategy& strategy, + TradeManager& tm, bars::BarStore& store, + std::chrono::system_clock::time_point base, + bool withPreMidnightBar = true, + std::int32_t postAsiaHigh = 110020) { + if (withPreMidnightBar) { + feedBar(strategy, tm, store, base, minutes{-20}, + {110000, 110005, 109995, 110000}); + } + // Without the pre-midnight bar the first Asian bar starts at 00:20, so + // the series' oldest bar sits strictly after midnight — partial coverage. + feedBar(strategy, tm, store, base, withPreMidnightBar ? minutes{0} : minutes{20}, + {110000, 110020, 109980, 110010}); + feedBar(strategy, tm, store, base, minutes{120}, + {110010, 110050, 109990, 110030}); // session high 110050 + feedBar(strategy, tm, store, base, minutes{240}, + {110030, 110040, 109950, 109990}); // session low 109950 + feedBar(strategy, tm, store, base, minutes{340}, + {109990, 110030, 109970, 110000}); + feedBar(strategy, tm, store, base, minutes{390}, + {110000, postAsiaHigh, 109980, 110000}); // 06:30 — not Asia +} + +} // namespace + +TEST_CASE("SessionRangeBreakoutStrategy trades the London break of the Asian range", + "[sessionRangeBreakout]") { + SessionRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedSessionFixture(strategy, tm, store, kWinterMidnight); + + // Winter: London opens 08:00 UTC; the window (120m) runs to 10:00. + SECTION("bid above the Asian high inside the window: LONG") { + CHECK(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{485}, 110071, 110061)) == + Direction::LONG); + } + + SECTION("bid exactly on the Asian high: no signal (strictly above)") { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{485}, 110060, 110050)) + .has_value()); + } + + SECTION("ask below the Asian low inside the window: SHORT") { + CHECK(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{485}, 109940, 109930)) == + Direction::SHORT); + } +} + +TEST_CASE("SessionRangeBreakoutStrategy only fires inside the entry window", + "[sessionRangeBreakout]") { + SessionRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedSessionFixture(strategy, tm, store, kWinterMidnight); + + SECTION("a breakout before the winter open (07:05) is refused") { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{425}, 110071, 110061)) + .has_value()); + } + + SECTION("a breakout after the window closes (10:05) is refused") { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{605}, 110071, 110061)) + .has_value()); + } +} + +TEST_CASE("SessionRangeBreakoutStrategy follows the BST London open", + "[sessionRangeBreakout]") { + SessionRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedSessionFixture(strategy, tm, store, kSummerMidnight); + + // 07:05 is pre-open in winter (see above) but inside the window under + // BST — the same offset flipping proves the DST rule is consulted. + CHECK(step(strategy, tm, store, + tickAt(kSummerMidnight, minutes{425}, 110071, 110061)) == + Direction::LONG); +} + +TEST_CASE("SessionRangeBreakoutStrategy pads the range with BUFFER_PIPS", + "[sessionRangeBreakout]") { + // 2 pips = 20 points on EURUSD: the padded high sits at 110070. + SessionRangeBreakoutStrategy strategy{makeConfig(2)}; + TradeManager tm; + auto store = makeStore(makeConfig(2)); + feedSessionFixture(strategy, tm, store, kWinterMidnight); + + SECTION("a poke through the raw high but not the padding: no signal") { + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{485}, 110071, 110061)) + .has_value()); + } + + SECTION("clearing the padded high: LONG") { + CHECK(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{485}, 110081, 110071)) == + Direction::LONG); + } +} + +TEST_CASE("SessionRangeBreakoutStrategy keeps post-session bars out of the range", + "[sessionRangeBreakout]") { + SessionRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + // The 06:30 bar spikes to 110200 — outside the Asian window, so the + // tradeable high must stay 110050. + feedSessionFixture(strategy, tm, store, kWinterMidnight, true, 110200); + + CHECK(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{485}, 110071, 110061)) == + Direction::LONG); +} + +TEST_CASE("SessionRangeBreakoutStrategy refuses a partially covered session", + "[sessionRangeBreakout]") { + SessionRangeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + // No pre-midnight bar: the series' oldest bar starts 00:20, so today's + // Asian session is only partially represented — a fragment range (a + // run's first day) must not trade. + feedSessionFixture(strategy, tm, store, kWinterMidnight, false); + + CHECK_FALSE(step(strategy, tm, store, + tickAt(kWinterMidnight, minutes{485}, 110071, 110061)) + .has_value()); +} + +// The time-cap tests drive during() directly: its exit path is independent of +// bar state (no warm-up needed), and trades are opened straight on the +// TradeManager — same harness as the OhlcBreakoutStrategy cap tests. +TEST_CASE("SessionRangeBreakoutStrategy closes trades past the max duration via during()", + "[sessionRangeBreakout]") { + SessionRangeBreakoutStrategy strategy{makeConfig(0, 120, 60)}; + TradeManager tm; + + SECTION("LONG past the cap closes at the bid") { + tm.openTrade(tickAt(kWinterMidnight, seconds{0}, 110000, 109998), 1, + Direction::LONG); + strategy.during( + tickAt(kWinterMidnight, minutes{60} + seconds{1}, 110050, 110040), bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110040); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("SHORT past the cap closes at the ask") { + tm.openTrade(tickAt(kWinterMidnight, seconds{0}, 110000, 109998), 1, + Direction::SHORT); + strategy.during( + tickAt(kWinterMidnight, minutes{60} + seconds{1}, 110050, 110040), bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110050); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("exactly at the cap stays open (strictly greater)") { + tm.openTrade(tickAt(kWinterMidnight, seconds{0}, 110000, 109998), 1, + Direction::LONG); + strategy.during(tickAt(kWinterMidnight, minutes{60}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } +} + +TEST_CASE("SessionRangeBreakoutStrategy max duration of zero disables the exit", + "[sessionRangeBreakout]") { + SessionRangeBreakoutStrategy strategy{makeConfig(0, 120, 0)}; + TradeManager tm; + + tm.openTrade(tickAt(kWinterMidnight, seconds{0}, 110000, 109998), 1, + Direction::LONG); + strategy.during(tickAt(kWinterMidnight, minutes{600}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); +} + +TEST_CASE("SessionRangeBreakoutStrategy rejects malformed configuration", + "[sessionRangeBreakout]") { + SECTION("no OHLC timeframe") { + auto config = makeConfig(); + config.OHLC_VARIABLES.clear(); + CHECK_THROWS_AS(SessionRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("missing SESSION_RANGE_BREAKOUT_VARIABLES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES = std::nullopt; + CHECK_THROWS_AS(SessionRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("ENTRY_WINDOW_MINUTES below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES + ->ENTRY_WINDOW_MINUTES = 0; + CHECK_THROWS_AS(SessionRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("negative BUFFER_PIPS") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES->BUFFER_PIPS = -1; + CHECK_THROWS_AS(SessionRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("window one bar short of midnight -> entry cutoff coverage") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_COUNT -= 1; + CHECK_THROWS_AS(SessionRangeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("OHLC_MINUTES below 1") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_MINUTES = 0; + CHECK_THROWS_AS(SessionRangeBreakoutStrategy{config}, std::invalid_argument); + } +} + +TEST_CASE("StrategyVariables round-trips SESSION_RANGE_BREAKOUT_VARIABLES through JSON", + "[sessionRangeBreakout]") { + SECTION("present group survives the round-trip") { + tradingDefinitions::StrategyVariables vars; + vars.SESSION_RANGE_BREAKOUT_VARIABLES = + tradingDefinitions::SessionRangeBreakoutVariables{ + .BUFFER_PIPS = 7, + .ENTRY_WINDOW_MINUTES = 90, + .MAX_TRADE_DURATION_MINUTES = 45}; + + const nlohmann::json j = vars; + const auto back = j.get(); + + REQUIRE(back.SESSION_RANGE_BREAKOUT_VARIABLES.has_value()); + CHECK(back.SESSION_RANGE_BREAKOUT_VARIABLES->BUFFER_PIPS == 7); + CHECK(back.SESSION_RANGE_BREAKOUT_VARIABLES->ENTRY_WINDOW_MINUTES == 90); + CHECK(back.SESSION_RANGE_BREAKOUT_VARIABLES->MAX_TRADE_DURATION_MINUTES == 45); + } + + SECTION("absent MAX_TRADE_DURATION_MINUTES parses as disabled") { + // Models a winner config persisted before the field existed: the + // WITH_DEFAULT codec must fall back to 0, not throw. + const auto vars = + nlohmann::json::parse( + R"({"SESSION_RANGE_BREAKOUT_VARIABLES":{"BUFFER_PIPS":2,)" + R"("ENTRY_WINDOW_MINUTES":60}})") + .get(); + + REQUIRE(vars.SESSION_RANGE_BREAKOUT_VARIABLES.has_value()); + CHECK(vars.SESSION_RANGE_BREAKOUT_VARIABLES->ENTRY_WINDOW_MINUTES == 60); + CHECK(vars.SESSION_RANGE_BREAKOUT_VARIABLES->MAX_TRADE_DURATION_MINUTES == 0); + } + + SECTION("absent group serialises as null and stays absent") { + const tradingDefinitions::StrategyVariables vars; + const nlohmann::json j = vars; + + CHECK(j.at("SESSION_RANGE_BREAKOUT_VARIABLES").is_null()); + CHECK_FALSE(j.get() + .SESSION_RANGE_BREAKOUT_VARIABLES.has_value()); + } +} diff --git a/tests/sqlManager.cpp b/tests/sqlManager.cpp new file mode 100644 index 0000000..07c941a --- /dev/null +++ b/tests/sqlManager.cpp @@ -0,0 +1,74 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include + +#include +#include +#include +#include + +import databaseConnection; +import sqlManager; + +namespace { + +const std::chrono::system_clock::time_point kLower{ + std::chrono::sys_days{std::chrono::year{2025} / 10 / 1}}; +const std::chrono::system_clock::time_point kUpper{ + std::chrono::sys_days{std::chrono::year{2026} / 1 / 1}}; + +} // namespace + +TEST_CASE("buildMonthBoundariesQuery asks QuestDB for every boundary in one statement", "[sqlManager]") { + // One statement means one now() evaluation, so all boundaries share a + // single snapshot instant — the property the superset slicing relies on. + CHECK(SqlManager::buildMonthBoundariesQuery(2) == + "SELECT now(), dateadd('M', -1, now()), dateadd('M', -2, now())"); + CHECK_THROWS_AS(SqlManager::buildMonthBoundariesQuery(0), std::invalid_argument); +} + +TEST_CASE("formatTimestamp round-trips exactly through fastParseTimestamp", "[sqlManager]") { + const auto tp = kLower + std::chrono::hours{14} + std::chrono::minutes{3} + + std::chrono::seconds{12} + std::chrono::microseconds{123456}; + const std::string formatted = SqlManager::formatTimestamp(tp); + CHECK(formatted == "2025-10-01T14:03:12.123456Z"); + + DateCache cache; + CHECK(fastParseTimestamp(formatted.c_str(), cache) == tp); + + // Zero fractions still print all six digits and still round-trip. + const std::string midnight = SqlManager::formatTimestamp(kLower); + CHECK(midnight == "2025-10-01T00:00:00.000000Z"); + CHECK(fastParseTimestamp(midnight.c_str(), cache) == kLower); +} + +TEST_CASE("buildPriceDataBetweenQuery uses literal bounds with half-open semantics", "[sqlManager]") { + CHECK(SqlManager::buildPriceDataBetweenQuery({"EURUSD"}, kLower, kUpper) == + "SELECT 'EURUSD' as symbol, ask, bid, timestamp FROM 'EURUSD' " + "WHERE timestamp >= cast('2025-10-01T00:00:00.000000Z' as timestamp) " + "AND timestamp < cast('2026-01-01T00:00:00.000000Z' as timestamp) " + "ORDER BY timestamp"); +} + +TEST_CASE("multi-symbol tick queries union and get a deterministic tie-break", "[sqlManager]") { + const std::string query = + SqlManager::buildPriceDataBetweenQuery({"EURUSD", "USDJPY"}, kLower, kUpper); + CHECK(query.find("UNION ALL SELECT 'USDJPY'") != std::string::npos); + // Equal-timestamp ticks across symbols must come back in one defined + // order; single-symbol queries skip the tie-break so QuestDB can elide + // the sort on the designated timestamp column. + CHECK(query.ends_with(" ORDER BY timestamp, symbol")); + CHECK(SqlManager::buildPriceDataBetweenQuery({"EURUSD"}, kLower, kUpper) + .ends_with(" ORDER BY timestamp")); +} + +TEST_CASE("loadPriceDataBetween rejects unknown symbols before touching the network", "[sqlManager]") { + const DatabaseConnection db("test"); + CHECK_THROWS_AS( + SqlManager::loadPriceDataBetween(db, {"EURUSD'; DROP TABLE x;--"}, kLower, kUpper), + std::invalid_argument); +} diff --git a/tests/squeezeBreakout.cpp b/tests/squeezeBreakout.cpp new file mode 100644 index 0000000..969b4cd --- /dev/null +++ b/tests/squeezeBreakout.cpp @@ -0,0 +1,442 @@ +#include + +#include +#include +#include +#include +#include // setenv — keep the bar store off QuestDB +#include +#include +#include +#include + +#include +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import squeezeBreakoutStrategy; +import barStore; // bars::BarStore — the strategy reads bars from it +import priceData; +import trade; +import tradeManager; + +namespace { + +using std::chrono::minutes; +using std::chrono::seconds; + +const std::chrono::system_clock::time_point t0 = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; + +// Signal timeframe: 1m bars, window derived at the ctor minimum +// VALID_BARS + max(1, NR_LOOKBACK - 1) + 1 (like the sweep mapper). Trend +// timeframe: 4 bars of 1m (EMA period 2) — the store dedups both consumers +// into one shared series and each reads its own tail, like production. +tradingDefinitions::StrategyConfig makeConfig(int nrLookback = 0, + int validBars = 1, + int bufferPips = 0, + int maxTradeDurationMinutes = 0) { + tradingDefinitions::StrategyConfig config; + config.UUID = "test-squeeze"; + config.TRADING_VARIABLES.STRATEGY = "SqueezeBreakoutStrategy"; + config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR = 10; + config.TRADING_VARIABLES.TRADING_SIZE = 1; + config.OHLC_VARIABLES = { + tradingDefinitions::OHLCVariables{ + .OHLC_COUNT = validBars + std::max(1, nrLookback - 1) + 1, + .OHLC_MINUTES = 1}, // signal timeframe + tradingDefinitions::OHLCVariables{.OHLC_COUNT = 4, + .OHLC_MINUTES = 1}, // trend + }; + config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES = + tradingDefinitions::SqueezeBreakoutVariables{ + .NR_LOOKBACK = nrLookback, + .VALID_BARS = validBars, + .BUFFER_PIPS = bufferPips, + .MAX_TRADE_DURATION_MINUTES = maxTradeDurationMinutes}; + return config; +} + +PriceData tickAt(std::chrono::system_clock::duration offset, std::int32_t ask, + std::int32_t bid, const std::string& symbol = "EURUSD") { + return PriceData(ask, bid, t0 + offset, symbol); +} + +// The loop owner's role in miniature: register every configured timeframe +// (same-duration entries dedup into one shared series with the larger window; +// each consumer reads its own tail, like production). +bars::BarStore makeStore(const tradingDefinitions::StrategyConfig& config) { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + bars::BarStore store; + for (const auto& ohlc : config.OHLC_VARIABLES) { + store.registerSeries(minutes{ohlc.OHLC_MINUTES}, ohlc.OHLC_COUNT); + } + return store; +} + +// One tick in run-loop order: the store update first, then decide, then the +// management hook — runTicks feeds the shared bars BEFORE the entry gates, +// so decide() judges the tick against bar state that already includes it. +std::optional step(SqueezeBreakoutStrategy& strategy, + TradeManager& tm, bars::BarStore& store, + const PriceData& tick) { + store.update(tick); + const auto signal = strategy.decide(tick, store); + strategy.during(tick, store, tm); + return signal; +} + +// The exact OHLC a crafted 1m bar should end up with. +struct BarShape { + std::int32_t o, h, l, c; +}; + +// Four asks inside one 1m bar, fed open/high/low/close: the first tick sets +// the open, later ones only extend the extremes and overwrite the close, so +// the bar lands on exactly this shape. Bars sit 2 minutes apart (slot = bar +// index) so the NEXT bar's first tick rolls this one. Bars are built from the +// ask; the bid trails 10 points and is irrelevant until a SHORT signal tick. +// None of the feed ticks may signal (the trend window is one bar short of +// full until the decision tick arrives). +void feedBar(SqueezeBreakoutStrategy& strategy, TradeManager& tm, + bars::BarStore& store, int slot, const BarShape& bar, + const std::string& symbol = "EURUSD") { + const auto barStart = minutes{2 * slot}; + const std::array, 4> ticks{{ + {seconds{0}, bar.o}, + {seconds{15}, bar.h}, + {seconds{30}, bar.l}, + {seconds{45}, bar.c}, + }}; + for (const auto& [offset, ask] : ticks) { + CHECK_FALSE(step(strategy, tm, store, + tickAt(barStart + offset, ask, ask - 10, symbol)) + .has_value()); + } +} + +// Canonical inside-bar fixture: a flat warm bar (its level picks the trend +// regime), a wide mother bar, then an inside bar whose range [109945, 109955] +// sits inside the mother's [109900, 109960]. The decision tick at minutes{6} +// rolls the inside bar closed and probes its levels. +constexpr BarShape kWarmLow{109940, 109940, 109940, 109940}; // uptrend +constexpr BarShape kWarmHigh{110100, 110100, 110100, 110100}; // downtrend +constexpr BarShape kMotherBar{109940, 109960, 109900, 109950}; +constexpr BarShape kInsideBar{109950, 109955, 109945, 109950}; + +} // namespace + +TEST_CASE("SqueezeBreakoutStrategy trades the break of an inside bar", + "[squeezeBreakout]") { + SqueezeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedBar(strategy, tm, store, 0, kWarmLow); + feedBar(strategy, tm, store, 1, kMotherBar); + feedBar(strategy, tm, store, 2, kInsideBar); + + SECTION("bid above the inside bar's high in an uptrend: LONG") { + CHECK(step(strategy, tm, store, tickAt(minutes{6}, 109975, 109965)) == + Direction::LONG); + } + + SECTION("bid exactly on the inside bar's high: no signal (strictly above)") { + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{6}, 109965, 109955)) + .has_value()); + } +} + +TEST_CASE("SqueezeBreakoutStrategy shorts the downside break in a downtrend", + "[squeezeBreakout]") { + SqueezeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + // The high warm bar keeps the trend EMA overhead, so the downside break + // reads as trend-aligned. + feedBar(strategy, tm, store, 0, kWarmHigh); + feedBar(strategy, tm, store, 1, {109960, 110000, 109940, 109950}); + feedBar(strategy, tm, store, 2, kInsideBar); + + CHECK(step(strategy, tm, store, tickAt(minutes{6}, 109935, 109925)) == + Direction::SHORT); +} + +TEST_CASE("SqueezeBreakoutStrategy trend filter blocks a counter-trend break", + "[squeezeBreakout]") { + SqueezeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + // Same inside-bar setup as the LONG case, but the warm bar sits far + // OVERHEAD: the upside break clears the pattern's high yet stays below + // the trailing EMA, so the macro filter reads downtrend and refuses it. + feedBar(strategy, tm, store, 0, kWarmHigh); + feedBar(strategy, tm, store, 1, kMotherBar); + feedBar(strategy, tm, store, 2, kInsideBar); + + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{6}, 109970, 109960)) + .has_value()); +} + +TEST_CASE("SqueezeBreakoutStrategy ignores a bar that is not inside its mother", + "[squeezeBreakout]") { + SqueezeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + auto store = makeStore(makeConfig()); + feedBar(strategy, tm, store, 0, kWarmLow); + feedBar(strategy, tm, store, 1, kMotherBar); + // High 109965 pokes above the mother's 109960 — no contraction. + feedBar(strategy, tm, store, 2, {109950, 109965, 109945, 109950}); + + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{6}, 109980, 109970)) + .has_value()); +} + +TEST_CASE("SqueezeBreakoutStrategy honours the pattern validity window", + "[squeezeBreakout]") { + // A fourth bar follows the inside bar without being a pattern itself: its + // high pokes ONE point above the inside bar's, which breaks the + // contraction without breaching the pattern's levels mid-feed (while this + // bar is in progress the inside bar is still the newest closed pattern, + // so any feed tick beyond its high would legitimately signal). Whether + // the aged inside bar still supplies levels afterwards is exactly + // VALID_BARS. + const BarShape pokeAfter{109950, 109956, 109946, 109950}; + + SECTION("VALID_BARS = 1: the aged pattern no longer trades") { + SqueezeBreakoutStrategy strategy{makeConfig(0, 1)}; + TradeManager tm; + auto store = makeStore(makeConfig(0, 1)); + feedBar(strategy, tm, store, 0, kWarmLow); + feedBar(strategy, tm, store, 1, kMotherBar); + feedBar(strategy, tm, store, 2, kInsideBar); + feedBar(strategy, tm, store, 3, pokeAfter); + + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{8}, 109975, 109965)) + .has_value()); + } + + SECTION("VALID_BARS = 3: the pattern one bar back still trades") { + SqueezeBreakoutStrategy strategy{makeConfig(0, 3)}; + TradeManager tm; + auto store = makeStore(makeConfig(0, 3)); + feedBar(strategy, tm, store, 0, kWarmLow); + feedBar(strategy, tm, store, 1, kMotherBar); + feedBar(strategy, tm, store, 2, kInsideBar); + feedBar(strategy, tm, store, 3, pokeAfter); + + CHECK(step(strategy, tm, store, tickAt(minutes{8}, 109975, 109965)) == + Direction::LONG); + } +} + +TEST_CASE("SqueezeBreakoutStrategy NR mode requires the strictly narrowest range", + "[squeezeBreakout]") { + SECTION("narrowest of the last 3: LONG on the break") { + SqueezeBreakoutStrategy strategy{makeConfig(3)}; + TradeManager tm; + auto store = makeStore(makeConfig(3)); + feedBar(strategy, tm, store, 0, {109950, 110010, 109950, 109980}); // range 60 + feedBar(strategy, tm, store, 1, {109980, 110000, 109960, 109980}); // range 40 + feedBar(strategy, tm, store, 2, {109980, 109990, 109970, 109980}); // range 20 + + CHECK(step(strategy, tm, store, tickAt(minutes{6}, 110011, 110001)) == + Direction::LONG); + } + + SECTION("wider than an earlier bar: no pattern, no signal") { + SqueezeBreakoutStrategy strategy{makeConfig(3)}; + TradeManager tm; + auto store = makeStore(makeConfig(3)); + feedBar(strategy, tm, store, 0, {109950, 110010, 109950, 109980}); // range 60 + feedBar(strategy, tm, store, 1, {109980, 110000, 109960, 109980}); // range 40 + feedBar(strategy, tm, store, 2, {109980, 110003, 109958, 109980}); // range 45 + + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{6}, 110014, 110004)) + .has_value()); + } +} + +TEST_CASE("SqueezeBreakoutStrategy pads the levels with BUFFER_PIPS", + "[squeezeBreakout]") { + // 2 pips = 20 points on EURUSD: the padded inside-bar high sits at 109975. + SqueezeBreakoutStrategy strategy{makeConfig(0, 1, 2)}; + TradeManager tm; + auto store = makeStore(makeConfig(0, 1, 2)); + feedBar(strategy, tm, store, 0, kWarmLow); + feedBar(strategy, tm, store, 1, kMotherBar); + feedBar(strategy, tm, store, 2, kInsideBar); + + SECTION("a poke through the raw high but not the padding: no signal") { + CHECK_FALSE(step(strategy, tm, store, tickAt(minutes{6}, 109985, 109975)) + .has_value()); + } + + SECTION("clearing the padded high: LONG") { + CHECK(step(strategy, tm, store, tickAt(minutes{6}, 109986, 109976)) == + Direction::LONG); + } +} + +// The time-cap tests drive during() directly: its exit path is independent of +// bar state (no warm-up needed), and trades are opened straight on the +// TradeManager — same harness as the NyOpenRangeBreakoutStrategy cap tests. +TEST_CASE("SqueezeBreakoutStrategy closes trades past the max duration via during()", + "[squeezeBreakout]") { + SqueezeBreakoutStrategy strategy{makeConfig(0, 1, 0, 60)}; + TradeManager tm; + + SECTION("LONG past the cap closes at the bid") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::LONG); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050, 110040), + bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110040); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("SHORT past the cap closes at the ask") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::SHORT); + strategy.during(tickAt(minutes{60} + seconds{1}, 110050, 110040), + bars::BarStore{}, tm); + + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().closePrice == 110050); + CHECK_FALSE(tm.hasActiveTradeForSymbol("EURUSD")); + } + + SECTION("exactly at the cap stays open (strictly greater)") { + tm.openTrade(tickAt(seconds{0}, 110000, 109998), 1, Direction::LONG); + strategy.during(tickAt(minutes{60}, 110050, 110040), bars::BarStore{}, + tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); + } +} + +// With the cap at 0 (the pre-cap winner-config default) exits stay owned by +// Operations via SL/TP — during() must not touch open positions. +TEST_CASE("SqueezeBreakoutStrategy max duration of zero disables the exit", + "[squeezeBreakout]") { + SqueezeBreakoutStrategy strategy{makeConfig()}; + TradeManager tm; + + tm.openTrade(tickAt(seconds{0}, 110000, 109990), 1, Direction::LONG); + strategy.during(tickAt(minutes{600}, 110050, 110040), bars::BarStore{}, tm); + + CHECK(tm.hasActiveTradeForSymbol("EURUSD")); + CHECK(tm.getClosedTrades().empty()); +} + +TEST_CASE("SqueezeBreakoutStrategy rejects malformed configuration", + "[squeezeBreakout]") { + SECTION("fewer than two OHLC timeframes") { + auto config = makeConfig(); + config.OHLC_VARIABLES.resize(1); + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("missing SQUEEZE_BREAKOUT_VARIABLES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES = std::nullopt; + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("NR_LOOKBACK of 1 (degenerate — every bar matches)") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES->NR_LOOKBACK = 1; + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("negative NR_LOOKBACK") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES->NR_LOOKBACK = -1; + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("VALID_BARS below 1") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES->VALID_BARS = 0; + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("negative BUFFER_PIPS") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES->BUFFER_PIPS = -1; + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("negative MAX_TRADE_DURATION_MINUTES") { + auto config = makeConfig(); + config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES + ->MAX_TRADE_DURATION_MINUTES = -1; + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("signal window one bar short of the scan") { + auto config = makeConfig(); + config.OHLC_VARIABLES[0].OHLC_COUNT -= 1; + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("trend window below 2") { + auto config = makeConfig(); + config.OHLC_VARIABLES[1].OHLC_COUNT = 1; + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("OHLC_MINUTES below 1") { + auto config = makeConfig(); + config.OHLC_VARIABLES[1].OHLC_MINUTES = 0; + CHECK_THROWS_AS(SqueezeBreakoutStrategy{config}, std::invalid_argument); + } +} + +TEST_CASE("StrategyVariables round-trips SQUEEZE_BREAKOUT_VARIABLES through JSON", + "[squeezeBreakout]") { + SECTION("present group survives the round-trip") { + tradingDefinitions::StrategyVariables vars; + vars.SQUEEZE_BREAKOUT_VARIABLES = + tradingDefinitions::SqueezeBreakoutVariables{ + .NR_LOOKBACK = 7, + .VALID_BARS = 3, + .BUFFER_PIPS = 2, + .MAX_TRADE_DURATION_MINUTES = 45}; + + const nlohmann::json j = vars; + const auto back = j.get(); + + REQUIRE(back.SQUEEZE_BREAKOUT_VARIABLES.has_value()); + CHECK(back.SQUEEZE_BREAKOUT_VARIABLES->NR_LOOKBACK == 7); + CHECK(back.SQUEEZE_BREAKOUT_VARIABLES->VALID_BARS == 3); + CHECK(back.SQUEEZE_BREAKOUT_VARIABLES->BUFFER_PIPS == 2); + CHECK(back.SQUEEZE_BREAKOUT_VARIABLES->MAX_TRADE_DURATION_MINUTES == 45); + } + + SECTION("absent NR_LOOKBACK parses as inside-bar mode") { + // Models a winner config persisted before the field existed: the + // WITH_DEFAULT codec must fall back to 0 (a VALID mode), not throw. + const auto vars = + nlohmann::json::parse( + R"({"SQUEEZE_BREAKOUT_VARIABLES":{"VALID_BARS":2,)" + R"("BUFFER_PIPS":1}})") + .get(); + + REQUIRE(vars.SQUEEZE_BREAKOUT_VARIABLES.has_value()); + CHECK(vars.SQUEEZE_BREAKOUT_VARIABLES->NR_LOOKBACK == 0); + CHECK(vars.SQUEEZE_BREAKOUT_VARIABLES->VALID_BARS == 2); + // Same doctrine for the cap: absent = 0 = disabled, the pre-cap + // behaviour those persisted winners were scored with. + CHECK(vars.SQUEEZE_BREAKOUT_VARIABLES->MAX_TRADE_DURATION_MINUTES == 0); + } + + SECTION("absent group serialises as null and stays absent") { + const tradingDefinitions::StrategyVariables vars; + const nlohmann::json j = vars; + + CHECK(j.at("SQUEEZE_BREAKOUT_VARIABLES").is_null()); + CHECK_FALSE(j.get() + .SQUEEZE_BREAKOUT_VARIABLES.has_value()); + } +} diff --git a/tests/sweep.cpp b/tests/sweep.cpp index 5ef5f23..709de4f 100644 --- a/tests/sweep.cpp +++ b/tests/sweep.cpp @@ -9,6 +9,7 @@ #include #include #include +#include // setenv/unsetenv — the slippage stress is env-driven at load #include #include #include @@ -30,11 +31,33 @@ import parameterGenerator; // sweep::ParameterGenerator, sweep::Combination import randomStrategySweep; // sweep::buildRandomStrategySweep import ohlcBreakoutStrategySweep; // sweep::buildOhlcBreakoutStrategySweep import makeOhlcBreakoutStrategy; // sweep::makeOhlcBreakoutStrategy +import fvgStrategySweep; // sweep::buildFvgStrategySweep +import makeFvgStrategy; // sweep::makeFvgStrategy +import keltnerFadeStrategySweep; // sweep::buildKeltnerFadeStrategySweep +import makeKeltnerFadeStrategy; // sweep::makeKeltnerFadeStrategy +import sessionRangeBreakoutStrategySweep; // sweep::buildSessionRangeBreakoutStrategySweep +import makeSessionRangeBreakoutStrategy; // sweep::makeSessionRangeBreakoutStrategy +import squeezeBreakoutStrategySweep; // sweep::buildSqueezeBreakoutStrategySweep +import makeSqueezeBreakoutStrategy; // sweep::makeSqueezeBreakoutStrategy +import nyOpenRangeBreakoutStrategySweep; // sweep::buildNyOpenRangeBreakoutStrategySweep +import makeNyOpenRangeBreakoutStrategy; // sweep::makeNyOpenRangeBreakoutStrategy +import liquiditySweepReversalStrategySweep; // sweep::buildLiquiditySweepReversalStrategySweep +import makeLiquiditySweepReversalStrategy; // sweep::makeLiquiditySweepReversalStrategy +import rangeVelocityStrategySweep; // sweep::buildRangeVelocityStrategySweep +import makeRangeVelocityStrategy; // sweep::makeRangeVelocityStrategy import runConfigurationBuilder; // sweep::makeRunConfiguration, sweep::resolveSymbolGroups import symbolGroups; // sweep::cleanSymbols, sweep::allSymbolsKnown import randomStrategy; // RandomStrategy import ohlcBreakoutStrategy; // OhlcBreakoutStrategy +import fvgStrategy; // FvgStrategy +import keltnerFadeStrategy; // KeltnerFadeStrategy +import sessionRangeBreakoutStrategy; // SessionRangeBreakoutStrategy +import squeezeBreakoutStrategy; // SqueezeBreakoutStrategy +import nyOpenRangeBreakoutStrategy; // NyOpenRangeBreakoutStrategy +import liquiditySweepReversalStrategy; // LiquiditySweepReversalStrategy +import rangeVelocityStrategy; // RangeVelocityStrategy import tradeManager; // TradeManager +import barStore; // bars::BarStore — decide() interface import priceData; // PriceData import trade; // Direction @@ -44,7 +67,7 @@ using namespace boost::decimal::literals; TEST_CASE("buildRandomStrategySweep registers exactly STOP and LIMIT", "[sweep]") { const auto generator = sweep::buildRandomStrategySweep(); - INFO("RandomStrategy sweeps exactly STOP and LIMIT pip distances"); + INFO("RandomStrategy sweeps exactly the STOP and LIMIT ATR multipliers"); CHECK(generator.parameterCount() == 2); } @@ -58,14 +81,14 @@ TEST_CASE("buildRandomStrategySweep generates the full cartesian product", "[swe std::set stops, limits; for (const auto& combo : combinations) { - stops.insert(combo.get("STOP_DISTANCE_IN_PIPS")); - limits.insert(combo.get("LIMIT_DISTANCE_IN_PIPS")); + stops.insert(combo.get("STOP_DISTANCE_IN_ATR")); + limits.insert(combo.get("LIMIT_DISTANCE_IN_ATR")); } CHECK(combinations.size() > 0); CHECK(combinations.size() == stops.size() * limits.size()); for (const auto& combo : combinations) { - CHECK(combo.has("STOP_DISTANCE_IN_PIPS")); - CHECK(combo.has("LIMIT_DISTANCE_IN_PIPS")); + CHECK(combo.has("STOP_DISTANCE_IN_ATR")); + CHECK(combo.has("LIMIT_DISTANCE_IN_ATR")); // The OHLC ranges are commented out in the builder; loadCommand's // makeStrategy relies on has() returning false so the fields default // to 0 instead of throwing in get(). @@ -81,8 +104,8 @@ TEST_CASE("buildRandomStrategySweep covers every stop/limit pair once", "[sweep] std::set stops, limits; std::set> pairs; for (const auto& combo : combinations) { - const double s = combo.get("STOP_DISTANCE_IN_PIPS"); - const double l = combo.get("LIMIT_DISTANCE_IN_PIPS"); + const double s = combo.get("STOP_DISTANCE_IN_ATR"); + const double l = combo.get("LIMIT_DISTANCE_IN_ATR"); stops.insert(s); limits.insert(l); pairs.emplace(s, l); @@ -107,42 +130,134 @@ TEST_CASE("buildRandomStrategySweep expansion order is deterministic", "[sweep]" const auto b = sweep::buildRandomStrategySweep().generateAllCombinations(); REQUIRE(a.size() == b.size()); for (std::size_t i = 0; i < a.size() && i < b.size(); ++i) { - CHECK(a[i].get("STOP_DISTANCE_IN_PIPS") == b[i].get("STOP_DISTANCE_IN_PIPS")); - CHECK(a[i].get("LIMIT_DISTANCE_IN_PIPS") == b[i].get("LIMIT_DISTANCE_IN_PIPS")); + CHECK(a[i].get("STOP_DISTANCE_IN_ATR") == b[i].get("STOP_DISTANCE_IN_ATR")); + CHECK(a[i].get("LIMIT_DISTANCE_IN_ATR") == b[i].get("LIMIT_DISTANCE_IN_ATR")); } } TEST_CASE("makeRunConfiguration carries the run id", "[sweep]") { - const auto config = sweep::makeRunConfiguration("test-run-id", "EURUSD"); + const auto config = + sweep::makeRunConfiguration("test-run-id", "EURUSD", sweep::BatchStamp{}); CHECK(config.RUN_ID == std::string("test-run-id")); } +// The batch stamp is frozen at load time and must land verbatim on the +// descriptor — config.BATCH names the weekly Elasticsearch indices and +// EXECUTION_TS becomes the documents' executionTimestamp. An empty stamp is +// the legacy escape hatch (unsuffixed indices, no metadata), so both shapes +// are pinned. +TEST_CASE("makeRunConfiguration stamps the batch identity", "[sweep]") { + unsetenv("ENTRY_SLIPPAGE_TENTH_PIPS"); + const sweep::BatchStamp stamp{"2099-01", "2099-01-01T00:00:00Z"}; + const auto config = sweep::makeRunConfiguration("id", "EURUSD", stamp); + CHECK(config.BATCH == stamp.label); + CHECK(config.EXECUTION_TS == stamp.executionTs); + + const auto legacy = + sweep::makeRunConfiguration("id", "EURUSD", sweep::BatchStamp{}); + CHECK(legacy.BATCH.empty()); + CHECK(legacy.EXECUTION_TS.empty()); +} + +// currentBatchStamp is what the load command freezes: the label comes from +// outcome_index::currentBatchLabel ($BACKTEST_BATCH override first), the +// timestamp from the wall clock — machinery only, no calendar assertions. +TEST_CASE("currentBatchStamp freezes the label and a timestamp", "[sweep]") { + setenv("BACKTEST_BATCH", "2099-01", 1); + const auto stamp = sweep::currentBatchStamp(); + CHECK(stamp.label == "2099-01"); + CHECK_FALSE(stamp.executionTs.empty()); + unsetenv("BACKTEST_BATCH"); +} + TEST_CASE("makeRunConfiguration sets the run descriptor values", "[sweep]") { - const auto config = sweep::makeRunConfiguration("test-run-id", "EURUSD"); + unsetenv("ENTRY_SLIPPAGE_TENTH_PIPS"); // isolate from the test process env + const auto config = + sweep::makeRunConfiguration("test-run-id", "EURUSD", sweep::BatchStamp{}); CHECK(config.SYMBOLS == std::string("EURUSD")); - CHECK(config.LAST_MONTHS == 6); + CHECK(config.LAST_MONTHS == 3); + CHECK(config.OFFSET_MONTHS == 0); CHECK(config.STARTING_BALANCE == tradingDefinitions::DEFAULT_STARTING_BALANCE); CHECK(config.MAX_LOSS_PERCENT == "5"_dd); CHECK(config.MAX_OPEN_TRADES == 1); - CHECK(config.REPORT_FAILURES); + CHECK(config.PEAK_HOURS_ONLY == true); // new runs trade peak hours only + CHECK(config.ENTRY_SLIPPAGE_TENTH_PIPS == 0); // stress off unless exported +} + +// The slippage stress is frozen into the descriptor AT LOAD TIME from the +// environment, so every queued run (and its Elasticsearch results doc) +// records the stress it ran under. Garbage must fail the load loudly — a +// stress sweep that silently ran unstressed is worse than no sweep. +TEST_CASE("makeRunConfiguration freezes the slippage stress from the env", + "[sweep]") { + const sweep::BatchStamp noBatch{}; + setenv("ENTRY_SLIPPAGE_TENTH_PIPS", "3", 1); + CHECK(sweep::makeRunConfiguration("id", "EURUSD", noBatch) + .ENTRY_SLIPPAGE_TENTH_PIPS == 3); + + setenv("ENTRY_SLIPPAGE_TENTH_PIPS", "0.3", 1); // pips, not tenths + CHECK_THROWS_AS(sweep::makeRunConfiguration("id", "EURUSD", noBatch), + std::invalid_argument); + + setenv("ENTRY_SLIPPAGE_TENTH_PIPS", "-1", 1); + CHECK_THROWS_AS(sweep::makeRunConfiguration("id", "EURUSD", noBatch), + std::invalid_argument); + + setenv("ENTRY_SLIPPAGE_TENTH_PIPS", "lots", 1); + CHECK_THROWS_AS(sweep::makeRunConfiguration("id", "EURUSD", noBatch), + std::invalid_argument); + + unsetenv("ENTRY_SLIPPAGE_TENTH_PIPS"); + CHECK(sweep::makeRunConfiguration("id", "EURUSD", noBatch) + .ENTRY_SLIPPAGE_TENTH_PIPS == 0); } // The descriptor travels through Redis as JSON (loadCommand dumps it, the // runner parses it back), so the round-trip must preserve every field — // including the decimal ones, which move as strings via decimal_json.hpp. TEST_CASE("makeRunConfiguration survives a JSON round-trip", "[sweep]") { - const auto original = sweep::makeRunConfiguration("round-trip-id", "EURUSD"); + auto original = sweep::makeRunConfiguration( + "round-trip-id", "EURUSD", + sweep::BatchStamp{"2099-01", "2099-01-01T00:00:00Z"}); + original.ENTRY_SLIPPAGE_TENTH_PIPS = 3; // non-default so a dropped + // serializer line fails below const nlohmann::json j = original; const auto restored = j.get(); CHECK(restored.RUN_ID == original.RUN_ID); CHECK(restored.SYMBOLS == original.SYMBOLS); + CHECK(restored.BATCH == original.BATCH); + CHECK(restored.EXECUTION_TS == original.EXECUTION_TS); CHECK(restored.LAST_MONTHS == original.LAST_MONTHS); + CHECK(restored.OFFSET_MONTHS == original.OFFSET_MONTHS); CHECK(restored.STARTING_BALANCE == original.STARTING_BALANCE); CHECK(restored.MAX_LOSS_PERCENT == original.MAX_LOSS_PERCENT); CHECK(restored.MAX_OPEN_TRADES == original.MAX_OPEN_TRADES); + CHECK(restored.MAX_TRADES_PER_MINUTE == original.MAX_TRADES_PER_MINUTE); CHECK(restored.REPORT_FAILURES == original.REPORT_FAILURES); + // Meaningful because the builder sets true (non-default): a forgotten + // from_json line would restore false and fail here. + CHECK(restored.PEAK_HOURS_ONLY == original.PEAK_HOURS_ONLY); + CHECK(restored.ENTRY_SLIPPAGE_TENTH_PIPS + == original.ENTRY_SLIPPAGE_TENTH_PIPS); +} + +// Optional fields fall back to the struct defaults when absent, so queue +// payloads written before a field existed still parse — MAX_TRADES_PER_MINUTE +// lands on its default (60, the runaway-strategy brake) and OFFSET_MONTHS on 0 +// (window ends at the present day) rather than throwing. +TEST_CASE("RunConfiguration parses payloads predating the optional fields", "[sweep]") { + const nlohmann::json j{ + {"RUN_ID", "legacy"}, {"SYMBOLS", "EURUSD"}, {"LAST_MONTHS", 1}}; + const auto restored = j.get(); + CHECK(restored.MAX_TRADES_PER_MINUTE == 60); + CHECK(restored.OFFSET_MONTHS == 0); + CHECK(restored.PEAK_HOURS_ONLY == false); + CHECK(restored.ENTRY_SLIPPAGE_TENTH_PIPS == 0); + // Pre-batch payloads route to the unsuffixed legacy indices. + CHECK(restored.BATCH.empty()); + CHECK(restored.EXECUTION_TS.empty()); } // cleanSymbols normalises one symbol group for the run side, which splits @@ -185,23 +300,16 @@ TEST_CASE("resolveSymbolGroups prefers the sweep override", "[sweep]") { } } -// The shipped kSymbolGroupsOverride narrows the sweep to EURUSD — widening it -// back to the full kSymbolGroups set is a deliberate source edit in -// randomStrategySweep (empty the override), never a silent default. -TEST_CASE("buildRandomStrategySweep narrows the symbols to EURUSD", "[sweep]") { - CHECK(sweep::buildRandomStrategySweep().symbolGroups() == - std::vector{"EURUSD"}); -} - // RandomStrategy ignores everything in its config, so default-constructed // StrategyConfig is enough — selectStrategy only needs the name, and these tests // construct the class directly. TEST_CASE("RandomStrategy always returns a signal", "[sweep]") { RandomStrategy strategy{tradingDefinitions::StrategyConfig{}}; PriceData tick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); + const bars::BarStore noBars; // RandomStrategy never reads it for (int i = 0; i < 100; ++i) { - const auto signal = strategy.decide(tick); + const auto signal = strategy.decide(tick, noBars); REQUIRE(signal.has_value()); CHECK((*signal == Direction::LONG || *signal == Direction::SHORT)); } @@ -212,11 +320,12 @@ TEST_CASE("RandomStrategy always returns a signal", "[sweep]") { TEST_CASE("RandomStrategy produces both directions", "[sweep]") { RandomStrategy strategy{tradingDefinitions::StrategyConfig{}}; PriceData tick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); + const bars::BarStore noBars; // RandomStrategy never reads it bool sawLong = false; bool sawShort = false; for (int i = 0; i < 1000 && !(sawLong && sawShort); ++i) { - const auto signal = strategy.decide(tick); + const auto signal = strategy.decide(tick, noBars); sawLong = sawLong || (signal == Direction::LONG); sawShort = sawShort || (signal == Direction::SHORT); } @@ -233,7 +342,7 @@ TEST_CASE("RandomStrategy::during leaves open trades alone", "[sweep]") { PriceData tick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); manager.openTrade(tick, 1, Direction::LONG); - strategy.during(tick, manager); + strategy.during(tick, bars::BarStore{}, manager); CHECK(manager.getActiveTrades().size() == 1); CHECK(manager.getClosedTrades().size() == 0); @@ -296,10 +405,10 @@ TEST_CASE("buildStrategyChunk keys each payload by run id and its UUID", "[sweep queue_keys::strategyPayloadKey(runId, config.UUID)); const auto combo = generator.combinationAt(begin + offset); - CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_PIPS == - static_cast(combo.get("STOP_DISTANCE_IN_PIPS"))); - CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_PIPS == - static_cast(combo.get("LIMIT_DISTANCE_IN_PIPS"))); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == + static_cast(combo.get("STOP_DISTANCE_IN_ATR"))); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == + static_cast(combo.get("LIMIT_DISTANCE_IN_ATR"))); } // An empty range (how the stream terminates) yields an empty chunk. @@ -310,11 +419,75 @@ TEST_CASE("buildStrategyChunk keys each payload by run id and its UUID", "[sweep // Names the OhlcBreakout sweep registers and makeOhlcBreakoutStrategy reads // back; the two must stay in lockstep or the mapper throws at load time. namespace { -const std::array kBreakoutParameterNames = { +const std::array kBreakoutParameterNames = { "BREAKOUT_OHLC_MINUTES", "BREAKOUT_OHLC_COUNT", "TREND_OHLC_MINUTES", "TREND_OHLC_COUNT", - "BUFFER_PIPS", "STOP_DISTANCE_IN_PIPS", - "LIMIT_DISTANCE_IN_PIPS"}; + "BUFFER_PIPS", "MAX_TRADE_DURATION_MINUTES", + "STOP_DISTANCE_IN_ATR", "LIMIT_DISTANCE_IN_ATR"}; + +// Same lockstep contract for the FVG sweep and makeFvgStrategy. No OHLC +// counts: the mapper derives them from LOOKBACK_BARS / HTF_SMA_PERIOD. +const std::array kFvgParameterNames = { + "FVG_OHLC_MINUTES", "HTF_OHLC_MINUTES", + "LOOKBACK_BARS", "MIN_GAP_PIPS", + "HTF_SMA_PERIOD", "MIN_GAP_AGE_BARS", + "MAX_TRADE_DURATION_MINUTES", + "STOP_DISTANCE_IN_ATR", "LIMIT_DISTANCE_IN_ATR"}; + +// Same lockstep contract for the KeltnerFade sweep and +// makeKeltnerFadeStrategy. No OHLC count: the mapper derives it from +// BAND_SMA_PERIOD. +const std::array kKeltnerFadeParameterNames = { + "OHLC_MINUTES", "BAND_SMA_PERIOD", "BAND_ATR_MULT_TENTHS", + "MAX_TRADE_DURATION_MINUTES", + "STOP_DISTANCE_IN_ATR", "LIMIT_DISTANCE_IN_ATR"}; + +// Same lockstep contract for the SessionRangeBreakout sweep and +// makeSessionRangeBreakoutStrategy. No OHLC count: the mapper derives it from +// OHLC_MINUTES / ENTRY_WINDOW_MINUTES. +const std::array kSessionRangeBreakoutParameterNames = { + "OHLC_MINUTES", "BUFFER_PIPS", + "ENTRY_WINDOW_MINUTES", "MAX_TRADE_DURATION_MINUTES", + "STOP_DISTANCE_IN_ATR", "LIMIT_DISTANCE_IN_ATR"}; + +// Same lockstep contract for the SqueezeBreakout sweep and +// makeSqueezeBreakoutStrategy. No signal OHLC count: the mapper derives it +// from VALID_BARS / NR_LOOKBACK. +const std::array kSqueezeBreakoutParameterNames = { + "OHLC_MINUTES", "NR_LOOKBACK", + "VALID_BARS", "BUFFER_PIPS", + "MAX_TRADE_DURATION_MINUTES", + "TREND_OHLC_MINUTES", "TREND_OHLC_COUNT", + "STOP_DISTANCE_IN_ATR", "LIMIT_DISTANCE_IN_ATR"}; + +// Same lockstep contract for the NyOpenRangeBreakout sweep and +// makeNyOpenRangeBreakoutStrategy. No OHLC count: the mapper derives it from +// RANGE_HOURS / OHLC_MINUTES / ENTRY_WINDOW_MINUTES. +const std::array kNyOpenRangeBreakoutParameterNames = { + "OHLC_MINUTES", "RANGE_HOURS", + "BUFFER_PIPS", "ENTRY_WINDOW_MINUTES", + "MAX_TRADE_DURATION_MINUTES", + "STOP_DISTANCE_IN_ATR", "LIMIT_DISTANCE_IN_ATR"}; + +// Same lockstep contract for the LiquiditySweepReversal sweep and +// makeLiquiditySweepReversalStrategy. No OHLC count: the mapper derives it +// from LOOKBACK_BARS / PIVOT_BARS / VALID_BARS. +const std::array kLiquiditySweepReversalParameterNames = { + "OHLC_MINUTES", "PIVOT_BARS", + "LOOKBACK_BARS", "MIN_SWEEP_PIPS", + "DISPLACEMENT_ATR_TENTHS", "VALID_BARS", + "MAX_TRADE_DURATION_MINUTES", + "STOP_DISTANCE_IN_ATR", "LIMIT_DISTANCE_IN_ATR"}; + +// Same lockstep contract for the RangeVelocity sweep and +// makeRangeVelocityStrategy. No RANGE_COUNT: the mapper derives it from +// RUN_BARS / SPEED_LOOKBACK_BARS / EXIT_RUN_BARS. +const std::array kRangeVelocityParameterNames = { + "RANGE_ATR_TICK_WINDOW", "RANGE_ATR_PERCENT", + "RUN_BARS", "SPEED_LOOKBACK_BARS", + "SPEED_RATIO_PERCENT", "EXIT_RUN_BARS", + "MAX_TRADE_DURATION_MINUTES", + "STOP_DISTANCE_IN_ATR", "LIMIT_DISTANCE_IN_ATR"}; // Stride of each parameter's axis in expansion order (later-registered ranges // vary fastest): stepping combinationAt's index by strides[k] walks parameter @@ -329,7 +502,7 @@ std::vector axisStrides( } } -TEST_CASE("buildOhlcBreakoutStrategySweep registers all seven parameters", "[sweep]") { +TEST_CASE("buildOhlcBreakoutStrategySweep registers every mapped parameter", "[sweep]") { const auto generator = sweep::buildOhlcBreakoutStrategySweep(); CHECK(generator.parameterCount() == kBreakoutParameterNames.size()); @@ -382,14 +555,6 @@ TEST_CASE("buildOhlcBreakoutStrategySweep generates the full cartesian product", CHECK(uniqueCombinations.size() == sampled); } -// The breakout sweep narrows itself to EURUSD while the strategy is being -// validated (kOhlcBreakoutSymbolGroupsOverride) — widening it is a deliberate -// source edit, never a silent default. -TEST_CASE("buildOhlcBreakoutStrategySweep narrows the symbols to EURUSD", "[sweep]") { - const auto generator = sweep::buildOhlcBreakoutStrategySweep(); - CHECK(generator.symbolGroups() == std::vector{"EURUSD"}); -} - TEST_CASE("makeOhlcBreakoutStrategy maps a combination onto the config", "[sweep]") { sweep::Combination combo; combo.set("BREAKOUT_OHLC_MINUTES", 15); @@ -397,15 +562,16 @@ TEST_CASE("makeOhlcBreakoutStrategy maps a combination onto the config", "[sweep combo.set("TREND_OHLC_MINUTES", 60); combo.set("TREND_OHLC_COUNT", 50); combo.set("BUFFER_PIPS", 5); - combo.set("STOP_DISTANCE_IN_PIPS", 40); - combo.set("LIMIT_DISTANCE_IN_PIPS", 60); + combo.set("MAX_TRADE_DURATION_MINUTES", 45); + combo.set("STOP_DISTANCE_IN_ATR", 40); + combo.set("LIMIT_DISTANCE_IN_ATR", 60); const auto config = sweep::makeOhlcBreakoutStrategy(combo); // Must match the dispatch string in run/operations.cppm. CHECK(config.TRADING_VARIABLES.STRATEGY == "OhlcBreakoutStrategy"); - CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_PIPS == 40); - CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_PIPS == 60); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == 40); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == 60); CHECK(config.TRADING_VARIABLES.TRADING_SIZE == 1); CHECK_FALSE(config.UUID.empty()); @@ -418,6 +584,8 @@ TEST_CASE("makeOhlcBreakoutStrategy maps a combination onto the config", "[sweep REQUIRE(config.STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES.has_value()); CHECK(config.STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES->BUFFER_PIPS == 5); + CHECK(config.STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 45); } // End-to-end contract between the builder, the mapper and the strategy's @@ -455,3 +623,877 @@ TEST_CASE("makeOhlcBreakoutStrategy accepts every swept parameter value", "[swee CHECK_NOTHROW(OhlcBreakoutStrategy{config}); } } + +TEST_CASE("buildFvgStrategySweep registers every mapped parameter", "[sweep]") { + const auto generator = sweep::buildFvgStrategySweep(); + CHECK(generator.parameterCount() == kFvgParameterNames.size()); + + // Every combination carries every registered range by construction, so + // probing the two ends of the grid proves the names are registered. + REQUIRE(generator.combinationCount() > 0); + for (const auto& combo : {generator.combinationAt(0), + generator.combinationAt(generator.combinationCount() - 1)}) { + for (const auto& name : kFvgParameterNames) { + INFO("Combination is missing " << name); + CHECK(combo.has(name)); + } + } +} + +// Config-agnostic (same idea as the breakout product test): walking each +// parameter's axis must yield that range's declared number of DISTINCT +// values, their product must equal combinationCount(), and a strided sample +// of full combinations must contain no duplicates. +TEST_CASE("buildFvgStrategySweep generates the full cartesian product", "[sweep]") { + const auto generator = sweep::buildFvgStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + std::size_t product = 1; + for (std::size_t k = 0; k < counts.size(); ++k) { + std::set axisValues; + for (std::size_t j = 0; j < counts[k].second; ++j) { + axisValues.insert(generator.combinationAt(j * strides[k]).get(counts[k].first)); + } + INFO(counts[k].first << " expands to duplicate values"); + CHECK(axisValues.size() == counts[k].second); + product *= axisValues.size(); + } + CHECK(product == total); + + constexpr std::size_t kSamples = 1000; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + std::set> uniqueCombinations; + std::size_t sampled = 0; + for (std::size_t i = 0; i < total; i += sampleStride, ++sampled) { + uniqueCombinations.insert(generator.combinationAt(i).values()); + } + CHECK(uniqueCombinations.size() == sampled); +} + +TEST_CASE("makeFvgStrategy maps a combination onto the config", "[sweep]") { + sweep::Combination combo; + combo.set("FVG_OHLC_MINUTES", 15); + combo.set("HTF_OHLC_MINUTES", 60); + combo.set("LOOKBACK_BARS", 20); + combo.set("MIN_GAP_PIPS", 5); + combo.set("HTF_SMA_PERIOD", 30); + combo.set("MIN_GAP_AGE_BARS", 2); + combo.set("MAX_TRADE_DURATION_MINUTES", 90); + combo.set("STOP_DISTANCE_IN_ATR", 40); + combo.set("LIMIT_DISTANCE_IN_ATR", 60); + + const auto config = sweep::makeFvgStrategy(combo); + + // Must match the dispatch string in strategyFactory. + CHECK(config.TRADING_VARIABLES.STRATEGY == "FvgStrategy"); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == 40); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == 60); + CHECK(config.TRADING_VARIABLES.TRADING_SIZE == 1); + CHECK_FALSE(config.UUID.empty()); + + // Positional contract: [0] FVG timeframe, [1] HTF trend timeframe, with + // the window counts DERIVED at the ctor minimums — the derivation is the + // mapper's contract, so it is pinned here. + REQUIRE(config.OHLC_VARIABLES.size() == 2); + CHECK(config.OHLC_VARIABLES[0].OHLC_COUNT == 20 + 3); + CHECK(config.OHLC_VARIABLES[0].OHLC_MINUTES == 15); + CHECK(config.OHLC_VARIABLES[1].OHLC_COUNT == 30 + 2); + CHECK(config.OHLC_VARIABLES[1].OHLC_MINUTES == 60); + + REQUIRE(config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES.has_value()); + CHECK(config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES->LOOKBACK_BARS == 20); + CHECK(config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES->MIN_GAP_PIPS == 5); + CHECK(config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES->HTF_SMA_PERIOD == 30); + CHECK(config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES->MIN_GAP_AGE_BARS == 2); + CHECK(config.STRATEGY_VARIABLES.FVG_STRATEGY_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 90); +} + +// End-to-end contract between the builder, the mapper and the strategy's +// fail-fast ctor: every combination the sweep queues must produce a config an +// FvgStrategy accepts — the derived OHLC counts must satisfy the ctor +// minimums for every swept LOOKBACK_BARS / HTF_SMA_PERIOD value. Sweeping +// each parameter's axis exercises every distinct value; a strided sample of +// full combinations guards the mapper against cross-field surprises. +TEST_CASE("makeFvgStrategy accepts every swept parameter value", "[sweep]") { + const auto generator = sweep::buildFvgStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + for (std::size_t k = 0; k < counts.size(); ++k) { + for (std::size_t j = 0; j < counts[k].second; ++j) { + const auto combo = generator.combinationAt(j * strides[k]); + INFO(counts[k].first << " = " << combo.get(counts[k].first) + << " produced a config the ctor rejects"); + const auto config = sweep::makeFvgStrategy(combo); + CHECK_NOTHROW(FvgStrategy{config}); + } + } + + constexpr std::size_t kSamples = 500; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + for (std::size_t i = 0; i < total; i += sampleStride) { + const auto config = sweep::makeFvgStrategy(generator.combinationAt(i)); + CHECK_NOTHROW(FvgStrategy{config}); + } +} + +TEST_CASE("buildKeltnerFadeStrategySweep registers every mapped parameter", "[sweep]") { + const auto generator = sweep::buildKeltnerFadeStrategySweep(); + CHECK(generator.parameterCount() == kKeltnerFadeParameterNames.size()); + + // Every combination carries every registered range by construction, so + // probing the two ends of the grid proves the names are registered. + REQUIRE(generator.combinationCount() > 0); + for (const auto& combo : {generator.combinationAt(0), + generator.combinationAt(generator.combinationCount() - 1)}) { + for (const auto& name : kKeltnerFadeParameterNames) { + INFO("Combination is missing " << name); + CHECK(combo.has(name)); + } + } +} + +// Config-agnostic (same idea as the breakout product test): walking each +// parameter's axis must yield that range's declared number of DISTINCT +// values, their product must equal combinationCount(), and a strided sample +// of full combinations must contain no duplicates. +TEST_CASE("buildKeltnerFadeStrategySweep generates the full cartesian product", "[sweep]") { + const auto generator = sweep::buildKeltnerFadeStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + std::size_t product = 1; + for (std::size_t k = 0; k < counts.size(); ++k) { + std::set axisValues; + for (std::size_t j = 0; j < counts[k].second; ++j) { + axisValues.insert(generator.combinationAt(j * strides[k]).get(counts[k].first)); + } + INFO(counts[k].first << " expands to duplicate values"); + CHECK(axisValues.size() == counts[k].second); + product *= axisValues.size(); + } + CHECK(product == total); + + constexpr std::size_t kSamples = 1000; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + std::set> uniqueCombinations; + std::size_t sampled = 0; + for (std::size_t i = 0; i < total; i += sampleStride, ++sampled) { + uniqueCombinations.insert(generator.combinationAt(i).values()); + } + CHECK(uniqueCombinations.size() == sampled); +} + +TEST_CASE("makeKeltnerFadeStrategy maps a combination onto the config", "[sweep]") { + sweep::Combination combo; + combo.set("OHLC_MINUTES", 15); + combo.set("BAND_SMA_PERIOD", 20); + combo.set("BAND_ATR_MULT_TENTHS", 25); + combo.set("MAX_TRADE_DURATION_MINUTES", 90); + combo.set("STOP_DISTANCE_IN_ATR", 40); + combo.set("LIMIT_DISTANCE_IN_ATR", 60); + + const auto config = sweep::makeKeltnerFadeStrategy(combo); + + // Must match the dispatch string in strategyFactory. + CHECK(config.TRADING_VARIABLES.STRATEGY == "KeltnerFadeStrategy"); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == 40); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == 60); + CHECK(config.TRADING_VARIABLES.TRADING_SIZE == 1); + CHECK_FALSE(config.UUID.empty()); + + // Positional contract: [0] is the signal timeframe, with the window count + // DERIVED at the ctor minimum — the derivation is the mapper's contract, + // so it is pinned here. + REQUIRE(config.OHLC_VARIABLES.size() == 1); + CHECK(config.OHLC_VARIABLES[0].OHLC_COUNT == 20 + 2); + CHECK(config.OHLC_VARIABLES[0].OHLC_MINUTES == 15); + + REQUIRE(config.STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES.has_value()); + CHECK(config.STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES->BAND_SMA_PERIOD == 20); + CHECK(config.STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES->BAND_ATR_MULT_TENTHS == 25); + CHECK(config.STRATEGY_VARIABLES.KELTNER_FADE_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 90); +} + +// End-to-end contract between the builder, the mapper and the strategy's +// fail-fast ctor: every combination the sweep queues must produce a config a +// KeltnerFadeStrategy accepts — the derived OHLC count must satisfy the ctor +// minimum for every swept BAND_SMA_PERIOD value. Sweeping each parameter's +// axis exercises every distinct value; a strided sample of full combinations +// guards the mapper against cross-field surprises. +TEST_CASE("makeKeltnerFadeStrategy accepts every swept parameter value", "[sweep]") { + const auto generator = sweep::buildKeltnerFadeStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + for (std::size_t k = 0; k < counts.size(); ++k) { + for (std::size_t j = 0; j < counts[k].second; ++j) { + const auto combo = generator.combinationAt(j * strides[k]); + INFO(counts[k].first << " = " << combo.get(counts[k].first) + << " produced a config the ctor rejects"); + const auto config = sweep::makeKeltnerFadeStrategy(combo); + CHECK_NOTHROW(KeltnerFadeStrategy{config}); + } + } + + constexpr std::size_t kSamples = 500; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + for (std::size_t i = 0; i < total; i += sampleStride) { + const auto config = + sweep::makeKeltnerFadeStrategy(generator.combinationAt(i)); + CHECK_NOTHROW(KeltnerFadeStrategy{config}); + } +} + +TEST_CASE("buildSessionRangeBreakoutStrategySweep registers every mapped parameter", + "[sweep]") { + const auto generator = sweep::buildSessionRangeBreakoutStrategySweep(); + CHECK(generator.parameterCount() == kSessionRangeBreakoutParameterNames.size()); + + // Every combination carries every registered range by construction, so + // probing the two ends of the grid proves the names are registered. + REQUIRE(generator.combinationCount() > 0); + for (const auto& combo : {generator.combinationAt(0), + generator.combinationAt(generator.combinationCount() - 1)}) { + for (const auto& name : kSessionRangeBreakoutParameterNames) { + INFO("Combination is missing " << name); + CHECK(combo.has(name)); + } + } +} + +// Config-agnostic (same idea as the breakout product test): walking each +// parameter's axis must yield that range's declared number of DISTINCT +// values, their product must equal combinationCount(), and a strided sample +// of full combinations must contain no duplicates. +TEST_CASE("buildSessionRangeBreakoutStrategySweep generates the full cartesian product", + "[sweep]") { + const auto generator = sweep::buildSessionRangeBreakoutStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + std::size_t product = 1; + for (std::size_t k = 0; k < counts.size(); ++k) { + std::set axisValues; + for (std::size_t j = 0; j < counts[k].second; ++j) { + axisValues.insert(generator.combinationAt(j * strides[k]).get(counts[k].first)); + } + INFO(counts[k].first << " expands to duplicate values"); + CHECK(axisValues.size() == counts[k].second); + product *= axisValues.size(); + } + CHECK(product == total); + + constexpr std::size_t kSamples = 1000; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + std::set> uniqueCombinations; + std::size_t sampled = 0; + for (std::size_t i = 0; i < total; i += sampleStride, ++sampled) { + uniqueCombinations.insert(generator.combinationAt(i).values()); + } + CHECK(uniqueCombinations.size() == sampled); +} + +TEST_CASE("makeSessionRangeBreakoutStrategy maps a combination onto the config", + "[sweep]") { + sweep::Combination combo; + combo.set("OHLC_MINUTES", 15); + combo.set("BUFFER_PIPS", 5); + combo.set("ENTRY_WINDOW_MINUTES", 120); + combo.set("MAX_TRADE_DURATION_MINUTES", 240); + combo.set("STOP_DISTANCE_IN_ATR", 40); + combo.set("LIMIT_DISTANCE_IN_ATR", 60); + + const auto config = sweep::makeSessionRangeBreakoutStrategy(combo); + + // Must match the dispatch string in strategyFactory. + CHECK(config.TRADING_VARIABLES.STRATEGY == "SessionRangeBreakoutStrategy"); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == 40); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == 60); + CHECK(config.TRADING_VARIABLES.TRADING_SIZE == 1); + CHECK_FALSE(config.UUID.empty()); + + // Positional contract: [0] is the signal timeframe, with the window count + // DERIVED to span midnight -> entry cutoff (ceil((480 + window)/minutes) + // + 2) — the derivation is the mapper's contract, so it is pinned here. + REQUIRE(config.OHLC_VARIABLES.size() == 1); + CHECK(config.OHLC_VARIABLES[0].OHLC_COUNT == (480 + 120 + 14) / 15 + 2); + CHECK(config.OHLC_VARIABLES[0].OHLC_MINUTES == 15); + + REQUIRE(config.STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES.has_value()); + CHECK(config.STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES->BUFFER_PIPS == 5); + CHECK(config.STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES + ->ENTRY_WINDOW_MINUTES == 120); + CHECK(config.STRATEGY_VARIABLES.SESSION_RANGE_BREAKOUT_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 240); +} + +// End-to-end contract between the builder, the mapper and the strategy's +// fail-fast ctor: every combination the sweep queues must produce a config a +// SessionRangeBreakoutStrategy accepts — the derived OHLC count must satisfy +// the ctor's midnight -> entry-cutoff span for every swept OHLC_MINUTES / +// ENTRY_WINDOW_MINUTES pair. Sweeping each parameter's axis exercises every +// distinct value; a strided sample of full combinations guards the mapper +// against cross-field surprises. +TEST_CASE("makeSessionRangeBreakoutStrategy accepts every swept parameter value", + "[sweep]") { + const auto generator = sweep::buildSessionRangeBreakoutStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + for (std::size_t k = 0; k < counts.size(); ++k) { + for (std::size_t j = 0; j < counts[k].second; ++j) { + const auto combo = generator.combinationAt(j * strides[k]); + INFO(counts[k].first << " = " << combo.get(counts[k].first) + << " produced a config the ctor rejects"); + const auto config = sweep::makeSessionRangeBreakoutStrategy(combo); + CHECK_NOTHROW(SessionRangeBreakoutStrategy{config}); + } + } + + constexpr std::size_t kSamples = 500; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + for (std::size_t i = 0; i < total; i += sampleStride) { + const auto config = + sweep::makeSessionRangeBreakoutStrategy(generator.combinationAt(i)); + CHECK_NOTHROW(SessionRangeBreakoutStrategy{config}); + } +} + +TEST_CASE("buildSqueezeBreakoutStrategySweep registers every mapped parameter", + "[sweep]") { + const auto generator = sweep::buildSqueezeBreakoutStrategySweep(); + CHECK(generator.parameterCount() == kSqueezeBreakoutParameterNames.size()); + + // Every combination carries every registered range by construction, so + // probing the two ends of the grid proves the names are registered. + REQUIRE(generator.combinationCount() > 0); + for (const auto& combo : {generator.combinationAt(0), + generator.combinationAt(generator.combinationCount() - 1)}) { + for (const auto& name : kSqueezeBreakoutParameterNames) { + INFO("Combination is missing " << name); + CHECK(combo.has(name)); + } + } +} + +// Config-agnostic (same idea as the breakout product test): walking each +// parameter's axis must yield that range's declared number of DISTINCT +// values, their product must equal combinationCount(), and a strided sample +// of full combinations must contain no duplicates. +TEST_CASE("buildSqueezeBreakoutStrategySweep generates the full cartesian product", + "[sweep]") { + const auto generator = sweep::buildSqueezeBreakoutStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + std::size_t product = 1; + for (std::size_t k = 0; k < counts.size(); ++k) { + std::set axisValues; + for (std::size_t j = 0; j < counts[k].second; ++j) { + axisValues.insert(generator.combinationAt(j * strides[k]).get(counts[k].first)); + } + INFO(counts[k].first << " expands to duplicate values"); + CHECK(axisValues.size() == counts[k].second); + product *= axisValues.size(); + } + CHECK(product == total); + + constexpr std::size_t kSamples = 1000; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + std::set> uniqueCombinations; + std::size_t sampled = 0; + for (std::size_t i = 0; i < total; i += sampleStride, ++sampled) { + uniqueCombinations.insert(generator.combinationAt(i).values()); + } + CHECK(uniqueCombinations.size() == sampled); +} + +TEST_CASE("makeSqueezeBreakoutStrategy maps a combination onto the config", + "[sweep]") { + sweep::Combination combo; + combo.set("OHLC_MINUTES", 30); + combo.set("NR_LOOKBACK", 7); + combo.set("VALID_BARS", 3); + combo.set("BUFFER_PIPS", 2); + combo.set("MAX_TRADE_DURATION_MINUTES", 90); + combo.set("TREND_OHLC_MINUTES", 60); + combo.set("TREND_OHLC_COUNT", 40); + combo.set("STOP_DISTANCE_IN_ATR", 40); + combo.set("LIMIT_DISTANCE_IN_ATR", 60); + + const auto config = sweep::makeSqueezeBreakoutStrategy(combo); + + // Must match the dispatch string in strategyFactory. + CHECK(config.TRADING_VARIABLES.STRATEGY == "SqueezeBreakoutStrategy"); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == 40); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == 60); + CHECK(config.TRADING_VARIABLES.TRADING_SIZE == 1); + CHECK_FALSE(config.UUID.empty()); + + // Positional contract: [0] signal timeframe with the window count DERIVED + // at the ctor minimum (VALID_BARS + max(1, NR_LOOKBACK - 1) + 1), [1] the + // trend timeframe — the derivation is the mapper's contract, so it is + // pinned here. + REQUIRE(config.OHLC_VARIABLES.size() == 2); + CHECK(config.OHLC_VARIABLES[0].OHLC_COUNT == 3 + 6 + 1); + CHECK(config.OHLC_VARIABLES[0].OHLC_MINUTES == 30); + CHECK(config.OHLC_VARIABLES[1].OHLC_COUNT == 40); + CHECK(config.OHLC_VARIABLES[1].OHLC_MINUTES == 60); + + REQUIRE(config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES.has_value()); + CHECK(config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES->NR_LOOKBACK == 7); + CHECK(config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES->VALID_BARS == 3); + CHECK(config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES->BUFFER_PIPS == 2); + CHECK(config.STRATEGY_VARIABLES.SQUEEZE_BREAKOUT_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 90); +} + +// End-to-end contract between the builder, the mapper and the strategy's +// fail-fast ctor: every combination the sweep queues must produce a config a +// SqueezeBreakoutStrategy accepts — the derived signal count must satisfy the +// ctor minimum for every swept NR_LOOKBACK / VALID_BARS pair (including the +// inside-bar 0 mode, never the rejected NR-1). Sweeping each parameter's axis +// exercises every distinct value; a strided sample of full combinations +// guards the mapper against cross-field surprises. +TEST_CASE("makeSqueezeBreakoutStrategy accepts every swept parameter value", + "[sweep]") { + const auto generator = sweep::buildSqueezeBreakoutStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + for (std::size_t k = 0; k < counts.size(); ++k) { + for (std::size_t j = 0; j < counts[k].second; ++j) { + const auto combo = generator.combinationAt(j * strides[k]); + INFO(counts[k].first << " = " << combo.get(counts[k].first) + << " produced a config the ctor rejects"); + const auto config = sweep::makeSqueezeBreakoutStrategy(combo); + CHECK_NOTHROW(SqueezeBreakoutStrategy{config}); + } + } + + constexpr std::size_t kSamples = 500; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + for (std::size_t i = 0; i < total; i += sampleStride) { + const auto config = + sweep::makeSqueezeBreakoutStrategy(generator.combinationAt(i)); + CHECK_NOTHROW(SqueezeBreakoutStrategy{config}); + } +} + +TEST_CASE("buildNyOpenRangeBreakoutStrategySweep registers every mapped parameter", + "[sweep]") { + const auto generator = sweep::buildNyOpenRangeBreakoutStrategySweep(); + CHECK(generator.parameterCount() == kNyOpenRangeBreakoutParameterNames.size()); + + // Every combination carries every registered range by construction, so + // probing the two ends of the grid proves the names are registered. + REQUIRE(generator.combinationCount() > 0); + for (const auto& combo : {generator.combinationAt(0), + generator.combinationAt(generator.combinationCount() - 1)}) { + for (const auto& name : kNyOpenRangeBreakoutParameterNames) { + INFO("Combination is missing " << name); + CHECK(combo.has(name)); + } + } +} + +// Config-agnostic (same idea as the breakout product test): walking each +// parameter's axis must yield that range's declared number of DISTINCT +// values, their product must equal combinationCount(), and a strided sample +// of full combinations must contain no duplicates. +TEST_CASE("buildNyOpenRangeBreakoutStrategySweep generates the full cartesian product", + "[sweep]") { + const auto generator = sweep::buildNyOpenRangeBreakoutStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + std::size_t product = 1; + for (std::size_t k = 0; k < counts.size(); ++k) { + std::set axisValues; + for (std::size_t j = 0; j < counts[k].second; ++j) { + axisValues.insert(generator.combinationAt(j * strides[k]).get(counts[k].first)); + } + INFO(counts[k].first << " expands to duplicate values"); + CHECK(axisValues.size() == counts[k].second); + product *= axisValues.size(); + } + CHECK(product == total); + + constexpr std::size_t kSamples = 1000; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + std::set> uniqueCombinations; + std::size_t sampled = 0; + for (std::size_t i = 0; i < total; i += sampleStride, ++sampled) { + uniqueCombinations.insert(generator.combinationAt(i).values()); + } + CHECK(uniqueCombinations.size() == sampled); +} + +TEST_CASE("makeNyOpenRangeBreakoutStrategy maps a combination onto the config", + "[sweep]") { + sweep::Combination combo; + combo.set("OHLC_MINUTES", 15); + combo.set("RANGE_HOURS", 8); + combo.set("BUFFER_PIPS", 4); + combo.set("ENTRY_WINDOW_MINUTES", 120); + combo.set("MAX_TRADE_DURATION_MINUTES", 240); + combo.set("STOP_DISTANCE_IN_ATR", 40); + combo.set("LIMIT_DISTANCE_IN_ATR", 60); + + const auto config = sweep::makeNyOpenRangeBreakoutStrategy(combo); + + // Must match the dispatch string in strategyFactory. + CHECK(config.TRADING_VARIABLES.STRATEGY == "NyOpenRangeBreakoutStrategy"); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == 40); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == 60); + CHECK(config.TRADING_VARIABLES.TRADING_SIZE == 1); + CHECK_FALSE(config.UUID.empty()); + + // Positional contract: [0] is the signal timeframe, with the window count + // DERIVED to span range start -> entry cutoff (ceil((RANGE_HOURS x 60 + + // window)/minutes) + 2) — the derivation is the mapper's contract, so it + // is pinned here. + REQUIRE(config.OHLC_VARIABLES.size() == 1); + CHECK(config.OHLC_VARIABLES[0].OHLC_COUNT == (8 * 60 + 120 + 14) / 15 + 2); + CHECK(config.OHLC_VARIABLES[0].OHLC_MINUTES == 15); + + REQUIRE(config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES.has_value()); + CHECK(config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES->RANGE_HOURS == 8); + CHECK(config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES->BUFFER_PIPS == 4); + CHECK(config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES + ->ENTRY_WINDOW_MINUTES == 120); + CHECK(config.STRATEGY_VARIABLES.NY_OPEN_RANGE_BREAKOUT_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 240); +} + +// End-to-end contract between the builder, the mapper and the strategy's +// fail-fast ctor: every combination the sweep queues must produce a config a +// NyOpenRangeBreakoutStrategy accepts — the derived OHLC count must satisfy +// the ctor's range-start -> entry-cutoff span for every swept RANGE_HOURS / +// OHLC_MINUTES / ENTRY_WINDOW_MINUTES triple (and every RANGE_HOURS must +// clear the previous-midnight cap). Sweeping each parameter's axis exercises +// every distinct value; a strided sample of full combinations guards the +// mapper against cross-field surprises. +TEST_CASE("makeNyOpenRangeBreakoutStrategy accepts every swept parameter value", + "[sweep]") { + const auto generator = sweep::buildNyOpenRangeBreakoutStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + for (std::size_t k = 0; k < counts.size(); ++k) { + for (std::size_t j = 0; j < counts[k].second; ++j) { + const auto combo = generator.combinationAt(j * strides[k]); + INFO(counts[k].first << " = " << combo.get(counts[k].first) + << " produced a config the ctor rejects"); + const auto config = sweep::makeNyOpenRangeBreakoutStrategy(combo); + CHECK_NOTHROW(NyOpenRangeBreakoutStrategy{config}); + } + } + + constexpr std::size_t kSamples = 500; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + for (std::size_t i = 0; i < total; i += sampleStride) { + const auto config = + sweep::makeNyOpenRangeBreakoutStrategy(generator.combinationAt(i)); + CHECK_NOTHROW(NyOpenRangeBreakoutStrategy{config}); + } +} + +TEST_CASE("buildLiquiditySweepReversalStrategySweep registers every mapped parameter", + "[sweep]") { + const auto generator = sweep::buildLiquiditySweepReversalStrategySweep(); + CHECK(generator.parameterCount() == kLiquiditySweepReversalParameterNames.size()); + + // Every combination carries every registered range by construction, so + // probing the two ends of the grid proves the names are registered. + REQUIRE(generator.combinationCount() > 0); + for (const auto& combo : {generator.combinationAt(0), + generator.combinationAt(generator.combinationCount() - 1)}) { + for (const auto& name : kLiquiditySweepReversalParameterNames) { + INFO("Combination is missing " << name); + CHECK(combo.has(name)); + } + } +} + +// Config-agnostic (same idea as the breakout product test): walking each +// parameter's axis must yield that range's declared number of DISTINCT +// values, their product must equal combinationCount(), and a strided sample +// of full combinations must contain no duplicates. +TEST_CASE("buildLiquiditySweepReversalStrategySweep generates the full cartesian product", + "[sweep]") { + const auto generator = sweep::buildLiquiditySweepReversalStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + std::size_t product = 1; + for (std::size_t k = 0; k < counts.size(); ++k) { + std::set axisValues; + for (std::size_t j = 0; j < counts[k].second; ++j) { + axisValues.insert(generator.combinationAt(j * strides[k]).get(counts[k].first)); + } + INFO(counts[k].first << " expands to duplicate values"); + CHECK(axisValues.size() == counts[k].second); + product *= axisValues.size(); + } + CHECK(product == total); + + constexpr std::size_t kSamples = 1000; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + std::set> uniqueCombinations; + std::size_t sampled = 0; + for (std::size_t i = 0; i < total; i += sampleStride, ++sampled) { + uniqueCombinations.insert(generator.combinationAt(i).values()); + } + CHECK(uniqueCombinations.size() == sampled); +} + +TEST_CASE("makeLiquiditySweepReversalStrategy maps a combination onto the config", + "[sweep]") { + sweep::Combination combo; + combo.set("OHLC_MINUTES", 30); + combo.set("PIVOT_BARS", 3); + combo.set("LOOKBACK_BARS", 48); + combo.set("MIN_SWEEP_PIPS", 5); + combo.set("DISPLACEMENT_ATR_TENTHS", 10); + combo.set("VALID_BARS", 4); + combo.set("MAX_TRADE_DURATION_MINUTES", 90); + combo.set("STOP_DISTANCE_IN_ATR", 40); + combo.set("LIMIT_DISTANCE_IN_ATR", 60); + + const auto config = sweep::makeLiquiditySweepReversalStrategy(combo); + + // Must match the dispatch string in strategyFactory. + CHECK(config.TRADING_VARIABLES.STRATEGY == "LiquiditySweepReversalStrategy"); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == 40); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == 60); + CHECK(config.TRADING_VARIABLES.TRADING_SIZE == 1); + CHECK_FALSE(config.UUID.empty()); + + // Positional contract: [0] is the signal timeframe, with the window count + // DERIVED at the ctor minimum (max(LOOKBACK_BARS + PIVOT_BARS + 1, + // VALID_BARS + 11)) — the derivation is the mapper's contract, so it is + // pinned here. + REQUIRE(config.OHLC_VARIABLES.size() == 1); + CHECK(config.OHLC_VARIABLES[0].OHLC_COUNT == 48 + 3 + 1); + CHECK(config.OHLC_VARIABLES[0].OHLC_MINUTES == 30); + + REQUIRE(config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES.has_value()); + CHECK(config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES->PIVOT_BARS == 3); + CHECK(config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->LOOKBACK_BARS == 48); + CHECK(config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->MIN_SWEEP_PIPS == 5); + CHECK(config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->DISPLACEMENT_ATR_TENTHS == 10); + CHECK(config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->VALID_BARS == 4); + CHECK(config.STRATEGY_VARIABLES.LIQUIDITY_SWEEP_REVERSAL_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 90); +} + +// End-to-end contract between the builder, the mapper and the strategy's +// fail-fast ctor: every combination the sweep queues must produce a config a +// LiquiditySweepReversalStrategy accepts — the derived OHLC count must +// satisfy the ctor minimum for every swept PIVOT_BARS / LOOKBACK_BARS / +// VALID_BARS triple. Sweeping each parameter's axis exercises every distinct +// value; a strided sample of full combinations guards the mapper against +// cross-field surprises. +TEST_CASE("makeLiquiditySweepReversalStrategy accepts every swept parameter value", + "[sweep]") { + const auto generator = sweep::buildLiquiditySweepReversalStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + for (std::size_t k = 0; k < counts.size(); ++k) { + for (std::size_t j = 0; j < counts[k].second; ++j) { + const auto combo = generator.combinationAt(j * strides[k]); + INFO(counts[k].first << " = " << combo.get(counts[k].first) + << " produced a config the ctor rejects"); + const auto config = sweep::makeLiquiditySweepReversalStrategy(combo); + CHECK_NOTHROW(LiquiditySweepReversalStrategy{config}); + } + } + + constexpr std::size_t kSamples = 500; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + for (std::size_t i = 0; i < total; i += sampleStride) { + const auto config = + sweep::makeLiquiditySweepReversalStrategy(generator.combinationAt(i)); + CHECK_NOTHROW(LiquiditySweepReversalStrategy{config}); + } +} + +TEST_CASE("buildRangeVelocityStrategySweep registers every mapped parameter", + "[sweep]") { + const auto generator = sweep::buildRangeVelocityStrategySweep(); + CHECK(generator.parameterCount() == kRangeVelocityParameterNames.size()); + + // Every combination carries every registered range by construction, so + // probing the two ends of the grid proves the names are registered. + REQUIRE(generator.combinationCount() > 0); + for (const auto& combo : {generator.combinationAt(0), + generator.combinationAt(generator.combinationCount() - 1)}) { + for (const auto& name : kRangeVelocityParameterNames) { + INFO("Combination is missing " << name); + CHECK(combo.has(name)); + } + } +} + +// Config-agnostic (same idea as the breakout product test): walking each +// parameter's axis must yield that range's declared number of DISTINCT +// values, their product must equal combinationCount(), and a strided sample +// of full combinations must contain no duplicates. +TEST_CASE("buildRangeVelocityStrategySweep generates the full cartesian product", + "[sweep]") { + const auto generator = sweep::buildRangeVelocityStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + std::size_t product = 1; + for (std::size_t k = 0; k < counts.size(); ++k) { + std::set axisValues; + for (std::size_t j = 0; j < counts[k].second; ++j) { + axisValues.insert(generator.combinationAt(j * strides[k]).get(counts[k].first)); + } + INFO(counts[k].first << " expands to duplicate values"); + CHECK(axisValues.size() == counts[k].second); + product *= axisValues.size(); + } + CHECK(product == total); + + constexpr std::size_t kSamples = 1000; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + std::set> uniqueCombinations; + std::size_t sampled = 0; + for (std::size_t i = 0; i < total; i += sampleStride, ++sampled) { + uniqueCombinations.insert(generator.combinationAt(i).values()); + } + CHECK(uniqueCombinations.size() == sampled); +} + +TEST_CASE("makeRangeVelocityStrategy maps a combination onto the config", + "[sweep]") { + sweep::Combination combo; + combo.set("RANGE_ATR_TICK_WINDOW", 5000); + combo.set("RANGE_ATR_PERCENT", 40); + combo.set("RUN_BARS", 3); + combo.set("SPEED_LOOKBACK_BARS", 32); + combo.set("SPEED_RATIO_PERCENT", 60); + combo.set("EXIT_RUN_BARS", 2); + combo.set("MAX_TRADE_DURATION_MINUTES", 240); + combo.set("STOP_DISTANCE_IN_ATR", 2); + combo.set("LIMIT_DISTANCE_IN_ATR", 5); + + const auto config = sweep::makeRangeVelocityStrategy(combo); + + // Must match the dispatch string in strategyFactory. + CHECK(config.TRADING_VARIABLES.STRATEGY == "RangeVelocityStrategy"); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_ATR == 2); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_ATR == 5); + CHECK(config.TRADING_VARIABLES.TRADING_SIZE == 1); + CHECK_FALSE(config.UUID.empty()); + + // Deliberately no OHLC series — the ATR entry gate falls back to its + // default timeframe; the strategy trades range bars only. + CHECK(config.OHLC_VARIABLES.empty()); + + // Positional contract: [0] is THE range series, with the window count + // DERIVED at the ctor minimum + margin (max(RUN_BARS + + // SPEED_LOOKBACK_BARS, EXIT_RUN_BARS) + 2) — the derivation is the + // mapper's contract, so it is pinned here. + REQUIRE(config.RANGE_VARIABLES.size() == 1); + CHECK(config.RANGE_VARIABLES[0].RANGE_ATR_TICK_WINDOW == 5000); + CHECK(config.RANGE_VARIABLES[0].RANGE_ATR_PERCENT == 40); + CHECK(config.RANGE_VARIABLES[0].RANGE_COUNT == 3 + 32 + 2); + + REQUIRE(config.STRATEGY_VARIABLES.RANGE_VELOCITY_VARIABLES.has_value()); + CHECK(config.STRATEGY_VARIABLES.RANGE_VELOCITY_VARIABLES->RUN_BARS == 3); + CHECK(config.STRATEGY_VARIABLES.RANGE_VELOCITY_VARIABLES + ->SPEED_LOOKBACK_BARS == 32); + CHECK(config.STRATEGY_VARIABLES.RANGE_VELOCITY_VARIABLES + ->SPEED_RATIO_PERCENT == 60); + CHECK(config.STRATEGY_VARIABLES.RANGE_VELOCITY_VARIABLES + ->EXIT_RUN_BARS == 2); + CHECK(config.STRATEGY_VARIABLES.RANGE_VELOCITY_VARIABLES + ->MAX_TRADE_DURATION_MINUTES == 240); +} + +// End-to-end contract between the builder, the mapper and the strategy's +// fail-fast ctor: every combination the sweep queues must produce a config a +// RangeVelocityStrategy accepts — the derived RANGE_COUNT must satisfy the +// ctor minimum for every swept RUN_BARS / SPEED_LOOKBACK_BARS / +// EXIT_RUN_BARS triple. Sweeping each parameter's axis exercises every +// distinct value; a strided sample of full combinations guards the mapper +// against cross-field surprises. +TEST_CASE("makeRangeVelocityStrategy accepts every swept parameter value", + "[sweep]") { + const auto generator = sweep::buildRangeVelocityStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + for (std::size_t k = 0; k < counts.size(); ++k) { + for (std::size_t j = 0; j < counts[k].second; ++j) { + const auto combo = generator.combinationAt(j * strides[k]); + INFO(counts[k].first << " = " << combo.get(counts[k].first) + << " produced a config the ctor rejects"); + const auto config = sweep::makeRangeVelocityStrategy(combo); + CHECK_NOTHROW(RangeVelocityStrategy{config}); + } + } + + constexpr std::size_t kSamples = 500; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + for (std::size_t i = 0; i < total; i += sampleStride) { + const auto config = + sweep::makeRangeVelocityStrategy(generator.combinationAt(i)); + CHECK_NOTHROW(RangeVelocityStrategy{config}); + } +} diff --git a/tests/swingPivots.cpp b/tests/swingPivots.cpp new file mode 100644 index 0000000..0f99771 --- /dev/null +++ b/tests/swingPivots.cpp @@ -0,0 +1,94 @@ +#include + +#include +#include +#include + +import swingPivots; +import ohlcObject; + +namespace { + +// The predicates only read highs/lows; open/close mirror the midpoint and +// date is irrelevant. +OhlcObject candle(std::int32_t high, std::int32_t low) { + const std::int32_t mid = (high + low) / 2; + return OhlcObject{.open = mid, .close = mid, .high = high, .low = low}; +} + +} // namespace + +TEST_CASE("swing_pivots detects strict fractal extremes", "[swingPivots]") { + // Index 2 is a 2-wing swing high (110070 beats both wings) and index 2's + // low is NOT a swing low (109990 is beaten by index 4's 109950). + const std::vector bars{ + candle(110020, 109980), + candle(110040, 110000), + candle(110070, 109990), + candle(110030, 109970), + candle(110010, 109950), + }; + + SECTION("swing high: strictly above both wings") { + CHECK(swing_pivots::isSwingHighAt(bars, 2, 2)); + CHECK(swing_pivots::isSwingHighAt(bars, 2, 1)); + } + + SECTION("non-extremes are not pivots") { + CHECK_FALSE(swing_pivots::isSwingHighAt(bars, 1, 1)); + CHECK_FALSE(swing_pivots::isSwingLowAt(bars, 2, 2)); + } + + SECTION("swing low: strictly below both wings") { + const std::vector lows{ + candle(110020, 109980), + candle(110040, 110000), + candle(110010, 109940), + candle(110030, 109970), + candle(110010, 109990), + }; + CHECK(swing_pivots::isSwingLowAt(lows, 2, 2)); + CHECK_FALSE(swing_pivots::isSwingHighAt(lows, 2, 2)); + } +} + +TEST_CASE("swing_pivots ties disqualify", "[swingPivots]") { + // An equalled extreme is no fresh extreme (double top): index 2 matches + // index 0's high exactly, so neither is a pivot at wing 2. + const std::vector bars{ + candle(110070, 109980), + candle(110040, 110000), + candle(110070, 109990), + candle(110030, 109970), + candle(110010, 109960), + }; + CHECK_FALSE(swing_pivots::isSwingHighAt(bars, 2, 2)); + + // At wing 1 the tie sits outside the window, so index 2 qualifies again — + // the wing width bounds what an extreme is compared against. + CHECK(swing_pivots::isSwingHighAt(bars, 2, 1)); +} + +TEST_CASE("swing_pivots wing width changes the verdict", "[swingPivots]") { + // Index 2 beats its immediate neighbours but not the edge bars: a pivot + // at wing 1, not at wing 2. + const std::vector bars{ + candle(110100, 109980), + candle(110040, 110000), + candle(110070, 109990), + candle(110030, 109970), + candle(110090, 109960), + }; + CHECK(swing_pivots::isSwingHighAt(bars, 2, 1)); + CHECK_FALSE(swing_pivots::isSwingHighAt(bars, 2, 2)); +} + +TEST_CASE("swing_pivots rejects a wingless pivot", "[swingPivots]") { + const std::vector bars{ + candle(110020, 109980), + candle(110040, 110000), + candle(110030, 109990), + }; + CHECK_THROWS_AS(swing_pivots::isSwingHighAt(bars, 1, 0), std::invalid_argument); + CHECK_THROWS_AS(swing_pivots::isSwingLowAt(bars, 1, -1), std::invalid_argument); +} diff --git a/tests/tickCache.cpp b/tests/tickCache.cpp new file mode 100644 index 0000000..fe26b40 --- /dev/null +++ b/tests/tickCache.cpp @@ -0,0 +1,268 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +import priceData; +import tickCache; + +using std::chrono::system_clock; +using tick_cache::Superset; +using tick_cache::TickCache; + +namespace { + +// Deliberately small numbers unrelated to the production ladder: these tests +// pin the slicing/caching machinery, never the currently-configured windows. +constexpr int kMonths = 4; + +const system_clock::time_point kSnapshot{ + std::chrono::sys_days{std::chrono::year{2026} / 5 / 1}}; + +// boundaries[m] = m "months" before the snapshot. The cache and slicer only +// ever binary-search against these instants, so a fixed 30-day month keeps the +// fixture simple without loosening the tests. +std::vector makeBoundaries(const int months) { + std::vector boundaries(months + 1); + for (int m = 0; m <= months; ++m) { + boundaries[m] = kSnapshot - std::chrono::days{30 * m}; + } + return boundaries; +} + +PriceData tickAt(const system_clock::time_point ts) { + return PriceData(100, 99, ts, "EURUSD"); +} + +// A superset whose ticks sit at known offsets from the boundaries: one tick +// exactly ON each boundary below the snapshot, and one strictly inside each +// month, so inclusive/exclusive edges are directly observable. +std::shared_ptr makeSuperset(const int months) { + auto boundaries = makeBoundaries(months); + std::vector ticks; + for (int m = months; m >= 1; --m) { + ticks.push_back(tickAt(boundaries[m])); // on boundary + ticks.push_back(tickAt(boundaries[m] + std::chrono::days{10})); // inside month + } + ticks.push_back(tickAt(kSnapshot - std::chrono::microseconds{1})); // just before T + return std::make_shared(Superset{std::move(ticks), std::move(boundaries)}); +} + +} // namespace + +TEST_CASE("sliceForWindow honours inclusive-lower / exclusive-upper bounds", "[tickCache]") { + const auto superset = makeSuperset(kMonths); + const auto& ticks = superset->ticks; + const auto& boundaries = superset->boundaries; + + SECTION("a shifted window includes its lower boundary tick and excludes its upper") { + const auto slice = tick_cache::sliceForWindow(superset, 2, 2); + REQUIRE(slice.count > 0); + // First tick in [boundaries[4], boundaries[2]) is the one exactly on + // boundaries[4]; the tick exactly on boundaries[2] belongs to the NEXT + // window up, so the slice stops just before it. + CHECK(ticks[slice.begin].timestamp == boundaries[4]); + CHECK(ticks[slice.begin + slice.count - 1].timestamp < + boundaries[2]); + CHECK(ticks[slice.begin + slice.count].timestamp == boundaries[2]); + } + + SECTION("offset 0 leaves the upper end open to the superset's last tick") { + const auto slice = tick_cache::sliceForWindow(superset, 2, 0); + REQUIRE(slice.count > 0); + CHECK(ticks[slice.begin].timestamp == boundaries[2]); + CHECK(slice.begin + slice.count == ticks.size()); + } + + SECTION("the full-depth window is the whole buffer") { + const auto slice = tick_cache::sliceForWindow(superset, kMonths, 0); + CHECK(slice.begin == 0); + CHECK(slice.count == ticks.size()); + } + + SECTION("a window with no ticks is empty, not an error") { + auto boundaries2 = makeBoundaries(2); + // Only one tick, in the most recent month: the (1,1) window is empty. + std::vector sparse{tickAt(boundaries2[1] + std::chrono::days{1})}; + const auto sparseSet = std::make_shared( + Superset{std::move(sparse), std::move(boundaries2)}); + const auto slice = tick_cache::sliceForWindow(sparseSet, 1, 1); + CHECK(slice.count == 0); + } + + SECTION("windows deeper than the superset and degenerate windows throw") { + CHECK_THROWS_AS(tick_cache::sliceForWindow(superset, kMonths, 1), std::logic_error); + CHECK_THROWS_AS(tick_cache::sliceForWindow(superset, 0, 0), std::logic_error); + CHECK_THROWS_AS(tick_cache::sliceForWindow(superset, 1, -1), std::logic_error); + } + + SECTION("the slice keeps the superset alive on its own") { + auto slice = tick_cache::sliceForWindow(superset, 2, 0); + const auto* raw = slice.superset.get(); + CHECK(raw == superset.get()); + CHECK(slice.superset.use_count() >= 2); + } +} + +namespace { + +// Test harness around TickCache: counting loaders and a hand-cranked clock, so +// hits/misses/TTL are observable without QuestDB or sleeping. +struct CacheFixture { + int supersetLoads = 0; + int windowLoads = 0; + int beforeLoads = 0; + int lastMonthsRequested = 0; + int sequence = 0; // increments on every hook/loader call + int beforeLoadSeq = -1; + int loaderSeq = -1; + bool throwOnLoad = false; + std::chrono::steady_clock::time_point now{}; + + TickCache makeCache(const std::chrono::seconds ttl = std::chrono::seconds{600}, + const std::size_t maxSupersets = 1) { + return TickCache( + [this](const std::string&, const int months) { + if (throwOnLoad) { + throw std::runtime_error("loader failed"); + } + ++supersetLoads; + loaderSeq = ++sequence; + lastMonthsRequested = months; + Superset superset; + superset.boundaries = makeBoundaries(months); + superset.ticks.push_back(tickAt(kSnapshot - std::chrono::days{1})); + return superset; + }, + [this](const std::string&, const int, const int) { + ++windowLoads; + loaderSeq = ++sequence; + return std::vector{tickAt(kSnapshot - std::chrono::days{400})}; + }, + ttl, maxSupersets, kMonths, + [this] { return now; }); + } + + std::function beforeLoad() { + return [this] { + ++beforeLoads; + beforeLoadSeq = ++sequence; + }; + } +}; + +} // namespace + +TEST_CASE("TickCache serves repeat windows from one superset load", "[tickCache]") { + CacheFixture fx; + auto cache = fx.makeCache(); + + const auto first = cache.get("EURUSD", 2, 0, fx.beforeLoad()); + CHECK(fx.supersetLoads == 1); + CHECK(fx.beforeLoads == 1); + // beforeLoad must fire BEFORE the load it announces, so the drain loop's + // quiesce provably precedes the eviction inside. + CHECK(fx.beforeLoadSeq < fx.loaderSeq); + // The first load already requests the default depth, not the window's. + CHECK(fx.lastMonthsRequested == kMonths); + + // Every window that fits the superset is a pure hit: no loader, no hook. + const std::vector> windows{{2, 0}, {2, 2}, {1, 3}, {kMonths, 0}}; + for (const auto& [last, offset] : windows) { + const auto slice = cache.get("EURUSD", last, offset, fx.beforeLoad()); + CHECK(slice.superset == first.superset); + } + CHECK(fx.supersetLoads == 1); + CHECK(fx.windowLoads == 0); + CHECK(fx.beforeLoads == 1); +} + +TEST_CASE("TickCache reloads after the TTL expires", "[tickCache]") { + CacheFixture fx; + auto cache = fx.makeCache(std::chrono::seconds{600}); + + cache.get("EURUSD", 2, 0, fx.beforeLoad()); + fx.now += std::chrono::seconds{599}; + cache.get("EURUSD", 2, 0, fx.beforeLoad()); + CHECK(fx.supersetLoads == 1); + + fx.now += std::chrono::seconds{1}; // exactly TTL old now — stale + cache.get("EURUSD", 2, 0, fx.beforeLoad()); + CHECK(fx.supersetLoads == 2); + CHECK(fx.beforeLoads == 2); +} + +TEST_CASE("TickCache evicts the resident superset when a new key arrives", "[tickCache]") { + CacheFixture fx; + auto cache = fx.makeCache(std::chrono::seconds{600}, /*maxSupersets=*/1); + + cache.get("EURUSD", 2, 0, fx.beforeLoad()); + cache.get("USDJPY", 2, 0, fx.beforeLoad()); + CHECK(fx.supersetLoads == 2); + + // EURUSD was evicted to make room, so it must load again. + cache.get("EURUSD", 2, 0, fx.beforeLoad()); + CHECK(fx.supersetLoads == 3); + CHECK(fx.beforeLoads == 3); +} + +TEST_CASE("TickCache sends too-deep windows to the ad-hoc loader without evicting", "[tickCache]") { + CacheFixture fx; + auto cache = fx.makeCache(); + + cache.get("EURUSD", 2, 0, fx.beforeLoad()); + CHECK(fx.supersetLoads == 1); + + // Deeper than the resident 4-month superset: ad-hoc window load, resident + // superset untouched. + const auto deep = cache.get("EURUSD", kMonths, 3, fx.beforeLoad()); + CHECK(fx.windowLoads == 1); + CHECK(fx.supersetLoads == 1); + CHECK(fx.beforeLoads == 2); + CHECK(deep.begin == 0); + CHECK(deep.count == deep.superset->ticks.size()); + + // ...and the resident superset still serves fitting windows as hits. + cache.get("EURUSD", 2, 2, fx.beforeLoad()); + CHECK(fx.supersetLoads == 1); + CHECK(fx.beforeLoads == 2); +} + +TEST_CASE("TickCache sizes the first load to the deepest of default and requested", "[tickCache]") { + CacheFixture fx; + auto cache = fx.makeCache(); + + // Deeper than the default: the superset grows to fit rather than falling + // to the ad-hoc path, so later shallow windows still hit. + cache.get("EURUSD", kMonths, 2, fx.beforeLoad()); + CHECK(fx.supersetLoads == 1); + CHECK(fx.lastMonthsRequested == kMonths + 2); + + cache.get("EURUSD", 2, 0, fx.beforeLoad()); + CHECK(fx.supersetLoads == 1); +} + +TEST_CASE("TickCache propagates loader failures and retries on the next get", "[tickCache]") { + CacheFixture fx; + auto cache = fx.makeCache(); + + fx.throwOnLoad = true; + CHECK_THROWS_AS(cache.get("EURUSD", 2, 0, fx.beforeLoad()), std::runtime_error); + + fx.throwOnLoad = false; + const auto slice = cache.get("EURUSD", 2, 0, fx.beforeLoad()); + CHECK(fx.supersetLoads == 1); + CHECK(slice.superset != nullptr); +} diff --git a/tests/tickPacket.cpp b/tests/tickPacket.cpp index 8273ddf..b913dbf 100644 --- a/tests/tickPacket.cpp +++ b/tests/tickPacket.cpp @@ -21,21 +21,21 @@ namespace { // Build the 40-byte little-endian packet exactly as the C# sender would // (see the layout in tickPacket.cppm), so the test exercises the real contract. -std::array makePacket(double bid, double ask, +std::array makePacket(double bid, double ask, std::int64_t tsMicros, const std::string& symbol) { - std::array packet{}; + std::array packet{}; const auto put = [&](std::size_t offset, auto value) { const auto raw = std::bit_cast>(value); std::ranges::copy(raw, packet.begin() + offset); }; - put(ingest::kBidOffset, bid); - put(ingest::kAskOffset, ask); - put(ingest::kTsOffset, tsMicros); + put(tick_packet::kBidOffset, bid); + put(tick_packet::kAskOffset, ask); + put(tick_packet::kTsOffset, tsMicros); - for (std::size_t i = 0; i < symbol.size() && i < ingest::kSymbolSize; ++i) { - packet[ingest::kSymbolOffset + i] = static_cast(symbol[i]); + for (std::size_t i = 0; i < symbol.size() && i < tick_packet::kSymbolSize; ++i) { + packet[tick_packet::kSymbolOffset + i] = static_cast(symbol[i]); } return packet; } @@ -52,7 +52,7 @@ TEST_CASE("decodeTick parses a packet into scaled PriceData", "[tickPacket]") { const std::int64_t tsMicros = 1'719'360'000'000'000LL; const auto packet = makePacket(/*bid=*/1.10000, /*ask=*/1.10001, tsMicros, "EURUSD"); - const auto tick = ingest::decodeTick(packet); + const auto tick = tick_packet::decodeTick(packet); REQUIRE(tick.has_value()); CHECK(tick->symbol == "EURUSD"); // EURUSD price multiplier is 100000: 1.10000 -> 110000, 1.10001 -> 110001. @@ -66,14 +66,14 @@ TEST_CASE("decodeTick parses a packet into scaled PriceData", "[tickPacket]") { TEST_CASE("decodeTick scales by the per-symbol multiplier", "[tickPacket]") { SECTION("JPY pair uses x1000") { const auto packet = makePacket(156.123, 156.125, kValidTs, "USDJPY"); - const auto tick = ingest::decodeTick(packet); + const auto tick = tick_packet::decodeTick(packet); REQUIRE(tick.has_value()); CHECK(tick->bid == 156123); CHECK(tick->ask == 156125); } SECTION("index uses x100") { const auto packet = makePacket(5432.10, 5432.20, kValidTs, "USA500IDXUSD"); - const auto tick = ingest::decodeTick(packet); + const auto tick = tick_packet::decodeTick(packet); REQUIRE(tick.has_value()); CHECK(tick->bid == 543210); CHECK(tick->ask == 543220); @@ -83,11 +83,11 @@ TEST_CASE("decodeTick scales by the per-symbol multiplier", "[tickPacket]") { TEST_CASE("decodeTick rejects malformed input", "[tickPacket]") { SECTION("wrong packet size") { std::array tooSmall{}; - CHECK_FALSE(ingest::decodeTick(tooSmall).has_value()); + CHECK_FALSE(tick_packet::decodeTick(tooSmall).has_value()); } SECTION("unknown symbol is dropped") { const auto packet = makePacket(1.0, 1.0, kValidTs, "NOPE"); - CHECK_FALSE(ingest::decodeTick(packet).has_value()); + CHECK_FALSE(tick_packet::decodeTick(packet).has_value()); } } @@ -100,7 +100,7 @@ TEST_CASE("decodeTick golden vector pins the on-the-wire byte layout", "[tickPac // offset constants breaks this test even though the self-consistent tests // stay green. (To turn this into a true cross-language check, regenerate // these bytes from the real C# Serialize output for the same tick.) - constexpr std::array golden{ + constexpr std::array golden{ std::byte{0x9A}, std::byte{0x99}, std::byte{0x99}, std::byte{0x99}, std::byte{0x99}, std::byte{0x99}, std::byte{0xF1}, std::byte{0x3F}, // bid 1.10000 std::byte{0x0B}, std::byte{0x5E}, std::byte{0xF4}, std::byte{0x15}, @@ -113,7 +113,7 @@ TEST_CASE("decodeTick golden vector pins the on-the-wire byte layout", "[tickPac std::byte{0x00}, std::byte{0x00}, std::byte{0x00}, std::byte{0x00}, }; - const auto tick = ingest::decodeTick(golden); + const auto tick = tick_packet::decodeTick(golden); REQUIRE(tick.has_value()); CHECK(tick->symbol == "EURUSD"); CHECK(tick->bid == 110000); @@ -126,19 +126,19 @@ TEST_CASE("decodeTick golden vector pins the on-the-wire byte layout", "[tickPac TEST_CASE("decodeTick drops corrupt ticks (hardening guards)", "[tickPacket]") { SECTION("zero bid is dropped") { const auto packet = makePacket(/*bid=*/0.0, /*ask=*/1.10001, kValidTs, "EURUSD"); - CHECK_FALSE(ingest::decodeTick(packet).has_value()); + CHECK_FALSE(tick_packet::decodeTick(packet).has_value()); } SECTION("zero ask is dropped") { const auto packet = makePacket(/*bid=*/1.10000, /*ask=*/0.0, kValidTs, "EURUSD"); - CHECK_FALSE(ingest::decodeTick(packet).has_value()); + CHECK_FALSE(tick_packet::decodeTick(packet).has_value()); } SECTION("zero (epoch) timestamp is dropped") { const auto packet = makePacket(1.10000, 1.10001, /*tsMicros=*/0, "EURUSD"); - CHECK_FALSE(ingest::decodeTick(packet).has_value()); + CHECK_FALSE(tick_packet::decodeTick(packet).has_value()); } SECTION("negative timestamp is dropped") { const auto packet = makePacket(1.10000, 1.10001, /*tsMicros=*/-1, "EURUSD"); - CHECK_FALSE(ingest::decodeTick(packet).has_value()); + CHECK_FALSE(tick_packet::decodeTick(packet).has_value()); } SECTION("price that overflows INT32 when scaled is dropped") { // EURUSD scales x100000; INT32 max is 2,147,483,647, so any price above @@ -146,6 +146,6 @@ TEST_CASE("decodeTick drops corrupt ticks (hardening guards)", "[tickPacket]") { // near this (hence "latent"), but the guard must still reject it rather // than store a truncated value. 30000 * 100000 = 3,000,000,000 > INT32. const auto packet = makePacket(/*bid=*/1.10000, /*ask=*/30000.0, kValidTs, "EURUSD"); - CHECK_FALSE(ingest::decodeTick(packet).has_value()); + CHECK_FALSE(tick_packet::decodeTick(packet).has_value()); } } diff --git a/tests/trackingReport.cpp b/tests/trackingReport.cpp new file mode 100644 index 0000000..653c5f9 --- /dev/null +++ b/tests/trackingReport.cpp @@ -0,0 +1,287 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// trackingReport — the pure half of the tracking consumer: the PH#-then-PO# +// lookup order, the close-pip arithmetic (scales INJECTED — the machinery is +// under test, not the current symbolScale table), and the live-trades +// document shape. No Redis, no Elastic. + +#include +#include + +#include + +#include +#include +#include + +#include "shared/redis/positionManager.hpp" + +import dealPacket; +import trackingReport; + +using Catch::Approx; + +namespace { + +// A decodable PO#/PH# payload via the real codec, so these tests track the +// stored wire shape without hand-rolling JSON. +std::string samplePayload() { + redis_positions::PositionRecord record; + record.dealId = "DIAAAA"; + record.dealReference = "ref-1"; + record.symbol = "EURUSD"; + record.epic = "CS.D.EURUSD.MINI.IP"; + record.direction = "BUY"; + record.size = 1.5; + record.level = 117000; + record.strategyId = "strat-1"; + record.strategyName = "TestStrategy"; + record.openedAtMicros = 1234567890; + return redis_positions::encodePositionRecord(record); +} + +deal_packet::Deal sampleDeal() { + deal_packet::Deal deal; + deal.level = 1.17123; + deal.size = 1.5; + deal.stopLevel = std::nullopt; + deal.limitLevel = std::nullopt; + deal.dealReference = "ref-1"; + deal.dealId = "DIAAAA"; + deal.dealIdOrigin = "DIAAAA"; + deal.epic = "CS.D.EURUSD.MINI.IP"; + deal.direction = "SELL"; // the closing side + deal.status = "DELETED"; + deal.dealStatus = "ACCEPTED"; + deal.currency = "GBP"; + deal.channel = "WTP"; + deal.expiry = "-"; + deal.timestamp = "2026-07-14T09:30:00.250"; + deal.guaranteedStop = "false"; + return deal; +} + +// Getter stubs that record the call order. +tracking_report::PayloadGetter getterOf( + std::vector& calls, const std::string& name, + std::optional> result) { + return [&calls, name, result = std::move(result)](const std::string&) { + calls.push_back(name); + return result; + }; +} + +} // namespace + +// ---- computeClosePips ------------------------------------------------------- + +TEST_CASE("computeClosePips converts an FX-style close to pips", + "[trackingReport]") { + // 1.17123 x 100000 = 117123 points; (117123 - 117000) / 10 = 12.3 pips. + const auto pips = + tracking_report::computeClosePips(1.17123, 117000, "BUY", 1.0, + 100000, 10); + REQUIRE(pips.has_value()); + CHECK(*pips == Approx(12.3)); +} + +TEST_CASE("computeClosePips weights by position size", "[trackingReport]") { + const auto pips = + tracking_report::computeClosePips(1.17123, 117000, "BUY", 2.5, + 100000, 10); + REQUIRE(pips.has_value()); + CHECK(*pips == Approx(30.75)); +} + +TEST_CASE("computeClosePips signs by the position direction", + "[trackingReport]") { + // Same price move: a gain for the BUY is a loss for the SELL, and a BUY + // closing below its open goes negative. + const auto sell = + tracking_report::computeClosePips(1.17123, 117000, "SELL", 1.0, + 100000, 10); + REQUIRE(sell.has_value()); + CHECK(*sell == Approx(-12.3)); + + const auto losingBuy = + tracking_report::computeClosePips(1.16900, 117000, "BUY", 1.0, + 100000, 10); + REQUIRE(losingBuy.has_value()); + CHECK(*losingBuy == Approx(-10.0)); +} + +TEST_CASE("computeClosePips handles a JPY-style scale pair", + "[trackingReport]") { + // priceScale 1000, 10 points per pip: 154.750 closes a 154500-point open + // 250 points = 25 pips higher. + const auto pips = + tracking_report::computeClosePips(154.750, 154500, "BUY", 1.0, + 1000, 10); + REQUIRE(pips.has_value()); + CHECK(*pips == Approx(25.0)); +} + +TEST_CASE("computeClosePips handles a metals-style scale pair", + "[trackingReport]") { + // priceScale 1000, 1000 points per pip: exactly one pip. + const auto pips = + tracking_report::computeClosePips(2346.678, 2345678, "BUY", 1.0, + 1000, 1000); + REQUIRE(pips.has_value()); + CHECK(*pips == Approx(1.0)); +} + +TEST_CASE("computeClosePips refuses unknown scales", "[trackingReport]") { + // symbol_scale::kUnknown is 0 — either scale missing must skip the calc, + // not multiply/divide the P&L by zero. + CHECK_FALSE(tracking_report::computeClosePips(1.17123, 117000, "BUY", 1.0, + 0, 10) + .has_value()); + CHECK_FALSE(tracking_report::computeClosePips(1.17123, 117000, "BUY", 1.0, + 100000, 0) + .has_value()); +} + +// ---- lookupPosition --------------------------------------------------------- + +TEST_CASE("lookupPosition reads history first and skips the live getter on a " + "hit", + "[trackingReport]") { + std::vector calls; + const auto record = tracking_report::lookupPosition( + "ref-1", getterOf(calls, "PH", std::optional(samplePayload())), + getterOf(calls, "PO", std::optional(samplePayload()))); + REQUIRE(record.has_value()); + CHECK(record->strategyId == "strat-1"); + CHECK(calls == std::vector{"PH"}); +} + +TEST_CASE("lookupPosition falls back to the live key when history misses", + "[trackingReport]") { + std::vector calls; + // Outer engaged, inner empty: the read worked, the key is missing. + const auto missing = + std::optional>{std::optional{}}; + const auto record = tracking_report::lookupPosition( + "ref-1", getterOf(calls, "PH", missing), + getterOf(calls, "PO", std::optional(samplePayload()))); + REQUIRE(record.has_value()); + CHECK(record->symbol == "EURUSD"); + CHECK(calls == std::vector{"PH", "PO"}); +} + +TEST_CASE("lookupPosition falls back when the history read fails", + "[trackingReport]") { + std::vector calls; + const auto record = tracking_report::lookupPosition( + "ref-1", getterOf(calls, "PH", std::nullopt), // outer: Redis failure + getterOf(calls, "PO", std::optional(samplePayload()))); + REQUIRE(record.has_value()); + CHECK(calls == std::vector{"PH", "PO"}); +} + +TEST_CASE("lookupPosition falls back past an undecodable history payload", + "[trackingReport]") { + std::vector calls; + const auto record = tracking_report::lookupPosition( + "ref-1", getterOf(calls, "PH", std::optional(std::string{"not json"})), + getterOf(calls, "PO", std::optional(samplePayload()))); + REQUIRE(record.has_value()); + CHECK(calls == std::vector{"PH", "PO"}); +} + +TEST_CASE("lookupPosition yields nothing when both families miss", + "[trackingReport]") { + std::vector calls; + const auto missing = + std::optional>{std::optional{}}; + const auto record = tracking_report::lookupPosition( + "ref-1", getterOf(calls, "PH", missing), getterOf(calls, "PO", missing)); + CHECK_FALSE(record.has_value()); + CHECK(calls == std::vector{"PH", "PO"}); +} + +// ---- serializeDeal ---------------------------------------------------------- + +TEST_CASE("serializeDeal carries every field, absent optionals as null", + "[trackingReport]") { + const auto deal = sampleDeal(); + const auto parsed = nlohmann::json::parse(tracking_report::serializeDeal(deal)); + CHECK(parsed.size() == 16); + CHECK(parsed["level"].get() == Approx(1.17123)); + CHECK(parsed["size"].get() == Approx(1.5)); + CHECK(parsed["stopLevel"].is_null()); + CHECK(parsed["limitLevel"].is_null()); + CHECK(parsed["dealReference"] == "ref-1"); + CHECK(parsed["dealId"] == "DIAAAA"); + CHECK(parsed["dealIdOrigin"] == "DIAAAA"); + CHECK(parsed["epic"] == "CS.D.EURUSD.MINI.IP"); + CHECK(parsed["direction"] == "SELL"); + CHECK(parsed["status"] == "DELETED"); + CHECK(parsed["dealStatus"] == "ACCEPTED"); + CHECK(parsed["currency"] == "GBP"); + CHECK(parsed["channel"] == "WTP"); + CHECK(parsed["expiry"] == "-"); + CHECK(parsed["timestamp"] == "2026-07-14T09:30:00.250"); + CHECK(parsed["guaranteedStop"] == "false"); +} + +// ---- buildLiveTradeDocument ------------------------------------------------- + +TEST_CASE("buildLiveTradeDocument enriches from the position record", + "[trackingReport]") { + const auto deal = sampleDeal(); + const auto record = + redis_positions::decodePositionRecord(samplePayload()); + REQUIRE(record.has_value()); + + const auto parsed = nlohmann::json::parse( + tracking_report::buildLiveTradeDocument(deal, record, "FALLBACK", + "demo", + "2026-07-14T09:30:01Z", 12.3)); + CHECK(parsed["date"] == "2026-07-14T09:30:01Z"); + CHECK(parsed["env"] == "demo"); + CHECK(parsed["symbol"] == "EURUSD"); // the position's, not the fallback + CHECK(parsed["action"] == "DELETED"); + CHECK(parsed["strategy"] == "strat-1"); + CHECK(parsed["dealId"] == "DIAAAA"); + CHECK(parsed["dealReference"] == "ref-1"); + CHECK(parsed["level"].get() == Approx(1.17123)); + CHECK(parsed["pips"].get() == Approx(12.3)); + // The raw-deal audit string is itself valid JSON of the same deal. + const auto inner = nlohmann::json::parse(parsed["json"].get()); + CHECK(inner["dealReference"] == "ref-1"); + CHECK(inner["stopLevel"].is_null()); +} + +TEST_CASE("buildLiveTradeDocument degrades to fallbacks without a record", + "[trackingReport]") { + auto deal = sampleDeal(); + deal.dealId.clear(); + deal.level = std::nullopt; + + const auto parsed = nlohmann::json::parse( + tracking_report::buildLiveTradeDocument(deal, std::nullopt, "EURUSD", + "demo", "2026-07-14T09:30:01Z", + std::nullopt)); + CHECK(parsed["symbol"] == "EURUSD"); // the epic-mapped fallback + CHECK(parsed["strategy"] == "Unknown"); + CHECK_FALSE(parsed.contains("dealId")); // omitted when empty + CHECK_FALSE(parsed.contains("level")); // omitted when absent + CHECK_FALSE(parsed.contains("pips")); +} + +TEST_CASE("buildLiveTradeDocument treats a zero level as absent", + "[trackingReport]") { + auto deal = sampleDeal(); + deal.level = 0.0; // IG sends 0 on some UPDATED events + const auto parsed = nlohmann::json::parse( + tracking_report::buildLiveTradeDocument(deal, std::nullopt, "EURUSD", + "demo", "2026-07-14T09:30:01Z", + std::nullopt)); + CHECK_FALSE(parsed.contains("level")); +} diff --git a/tests/tradeDocument.cpp b/tests/tradeDocument.cpp new file mode 100644 index 0000000..9944a28 --- /dev/null +++ b/tests/tradeDocument.cpp @@ -0,0 +1,113 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include + +#include +#include + +#include + +#include "run/reporting/tradeDocument.hpp" + +namespace { + +// A fully-populated document (armed stop and limit) for serialisation checks. +TradeDocument sampleDoc() { + return TradeDocument{ + .RUN_ID = "run-42", + .timestamp = "2024-03-01T10:15:30.250Z", + .ingestedAt = "2026-07-04T12:00:00Z", + .hostname = "test-host", + .strategyUuid = "uuid-1", + .strategyName = "RandomStrategy", + .tradeId = "T7", + .symbol = "EURUSD", + .direction = "LONG", + .size = 2, + .entryPrice = 1.10001, + .entryBid = 1.09999, + .entryAsk = 1.10001, + .closePrice = 1.10051, + .stopPrice = 1.09899, + .limitPrice = 1.10099, + .stopDistancePips = 10, + .limitDistancePips = 10, + .openTime = "2024-03-01T10:00:00.000Z", + .closeTime = "2024-03-01T10:15:30.250Z", + .holdSeconds = 930.25, + .pnlPoints = 100, + .pnlPips = 5.0, + .liquidated = false, + .scalingFactor = 10, + .priceScale = 100000, + }; +} + +} // namespace + +TEST_CASE("TradeDocument serialises the Kibana wire shape", "[tradeDocument]") { + const nlohmann::json j = sampleDoc(); + + // The timestamp field must land as `@timestamp` (the Kibana convention + // shared with the per-run docs), not as a literal "timestamp" key. + CHECK(j.at("@timestamp") == "2024-03-01T10:15:30.250Z"); + CHECK_FALSE(j.contains("timestamp")); + + CHECK(j.at("RUN_ID") == "run-42"); + CHECK(j.at("ingestedAt") == "2026-07-04T12:00:00Z"); + CHECK(j.at("hostname") == "test-host"); + CHECK(j.at("strategyUuid") == "uuid-1"); + CHECK(j.at("strategyName") == "RandomStrategy"); + CHECK(j.at("tradeId") == "T7"); + CHECK(j.at("symbol") == "EURUSD"); + CHECK(j.at("direction") == "LONG"); + CHECK(j.at("size") == 2); + CHECK(j.at("entryPrice") == 1.10001); + CHECK(j.at("closePrice") == 1.10051); + CHECK(j.at("openTime") == "2024-03-01T10:00:00.000Z"); + CHECK(j.at("closeTime") == "2024-03-01T10:15:30.250Z"); + CHECK(j.at("holdSeconds") == 930.25); + CHECK(j.at("pnlPoints") == 100); + CHECK(j.at("pnlPips") == 5.0); + CHECK(j.at("liquidated") == false); + CHECK(j.at("scalingFactor") == 10); + CHECK(j.at("priceScale") == 100000); + + // Both exit legs are armed here, so real levels must be emitted. + CHECK(j.at("stopPrice") == 1.09899); + CHECK(j.at("limitPrice") == 1.10099); +} + +TEST_CASE("TradeDocument nulls disarmed stop/limit levels", "[tradeDocument]") { + // A zero distance means the leg was never armed: the precomputed trigger + // price is meaningless and must serialise as null, not as a fake level. + TradeDocument doc = sampleDoc(); + doc.stopDistancePips = 0; + doc.limitDistancePips = 0; + + const nlohmann::json j = doc; + CHECK(j.at("stopPrice").is_null()); + CHECK(j.at("limitPrice").is_null()); +} + +TEST_CASE("isoUtcMillis formats UTC with millisecond precision", + "[tradeDocument]") { + using namespace std::chrono; + + // 2024-03-01T10:15:30.250Z, built from a known epoch offset. + const auto tp = std::chrono::sys_days{2024y / 3 / 1} + 10h + 15min + 30s + 250ms; + CHECK(TradeDocument::isoUtcMillis(tp) == "2024-03-01T10:15:30.250Z"); + + // Whole seconds keep the .000 so the field parses as a consistent date + // format in Elasticsearch. + const auto whole = std::chrono::sys_days{2024y / 3 / 1}; + CHECK(TradeDocument::isoUtcMillis(whole) == "2024-03-01T00:00:00.000Z"); + + // Ticks are sub-second; two closes 1ms apart must not collapse onto the + // same stamp. + CHECK(TradeDocument::isoUtcMillis(tp + 1ms) == "2024-03-01T10:15:30.251Z"); +} diff --git a/tests/tradeLocks.cpp b/tests/tradeLocks.cpp new file mode 100644 index 0000000..5ab121a --- /dev/null +++ b/tests/tradeLocks.cpp @@ -0,0 +1,25 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// The lock KEY FORMAT is a wire contract shared with the C# engine's +// TradeLocks (LOCK##) — a drifted prefix or separator +// would silently split the lock space between the two engines, so it is +// pinned here without needing a Redis server. The SET NX PX behaviour itself +// lives behind RedisOperations and is exercised in the live smoke test. + +#include + +#include "shared/redis/tradeLocks.hpp" + +TEST_CASE("lockKey matches the C# TradeLocks key format", "[tradeLocks]") { + CHECK(redis_locks::lockKey("abc-123", "LONG") == "LOCK#abc-123#LONG"); + CHECK(redis_locks::lockKey("abc-123", "SHORT") == "LOCK#abc-123#SHORT"); +} + +TEST_CASE("lockKey keeps strategy UUIDs distinct", "[tradeLocks]") { + CHECK(redis_locks::lockKey("a", "LONG") != redis_locks::lockKey("b", "LONG")); + CHECK(redis_locks::lockKey("a", "LONG") != redis_locks::lockKey("a", "SHORT")); +} diff --git a/tests/tradeManager.cpp b/tests/tradeManager.cpp index 741abe5..a7df8c7 100644 --- a/tests/tradeManager.cpp +++ b/tests/tradeManager.cpp @@ -27,6 +27,8 @@ import reviewStopAndLimit; // trading::reviewStopAndLimit import runLoop; // trading::runTicks, RiskLimits, RunStatus import operations; // Operations::run import strategy; // IStrategy +import barStore; // bars::BarStore — decide() interface + gate tests +import entryConditions; // conditions gate for the runTicks ATR tests import priceData; // PriceData import trade; // Direction, Trade @@ -57,21 +59,24 @@ tradingDefinitions::Configuration makeRandomStrategyConfig() { config.STRATEGY.UUID = "test-strategy-uuid"; auto& vars = config.STRATEGY.TRADING_VARIABLES; vars.STRATEGY = "RandomStrategy"; - vars.STOP_DISTANCE_IN_PIPS = 10; - vars.LIMIT_DISTANCE_IN_PIPS = 10; + vars.STOP_DISTANCE_IN_ATR = 10; + vars.LIMIT_DISTANCE_IN_ATR = 10; vars.TRADING_SIZE = 1; return config; } // Per-test TradingVariables so each runTicks case can dial SL/TP independently // (e.g. stop=0 to isolate the take-profit path). STRATEGY is unused by runTicks. +// With the ATR gate off (these tests don't pass one), runTicks forwards the +// values to openTrade as literal pip distances, so the scripted SL/TP geometry +// below stays exact. tradingDefinitions::TradingVariables makeVars(int32_t stopPips, int32_t limitPips, int32_t size) { tradingDefinitions::TradingVariables vars; vars.STRATEGY = "Scripted"; - vars.STOP_DISTANCE_IN_PIPS = stopPips; - vars.LIMIT_DISTANCE_IN_PIPS = limitPips; + vars.STOP_DISTANCE_IN_ATR = stopPips; + vars.LIMIT_DISTANCE_IN_ATR = limitPips; vars.TRADING_SIZE = size; return vars; } @@ -87,19 +92,39 @@ struct ScriptedStrategy : IStrategy { explicit ScriptedStrategy(std::vector> signals) : script(std::move(signals)) {} - std::optional decide(const PriceData& /*tick*/) override { + std::optional decide(const PriceData& /*tick*/, + const bars::BarStore& /*barStore*/) override { return index < script.size() ? script[index++] : std::nullopt; } - void during(const PriceData& /*tick*/, TradeManager& /*tradeManager*/) override {} + void during(const PriceData& /*tick*/, const bars::BarStore& /*barStore*/, + TradeManager& /*tradeManager*/) override {} }; // Always signals LONG — used to probe re-entry behaviour (gating while a // position is open; same-tick re-entry after a stop-out). struct AlwaysLongStrategy : IStrategy { - std::optional decide(const PriceData& /*tick*/) override { + std::optional decide(const PriceData& /*tick*/, + const bars::BarStore& /*barStore*/) override { return Direction::LONG; } - void during(const PriceData& /*tick*/, TradeManager& /*tradeManager*/) override {} + void during(const PriceData& /*tick*/, const bars::BarStore& /*barStore*/, + TradeManager& /*tradeManager*/) override {} +}; + +// AlwaysLongStrategy that counts its calls — lets the peak-hours tests prove +// decide() was skipped on an out-of-session tick while during() still ran. +struct CountingLongStrategy : IStrategy { + int decideCalls = 0; + int duringCalls = 0; + std::optional decide(const PriceData& /*tick*/, + const bars::BarStore& /*barStore*/) override { + ++decideCalls; + return Direction::LONG; + } + void during(const PriceData& /*tick*/, const bars::BarStore& /*barStore*/, + TradeManager& /*tradeManager*/) override { + ++duringCalls; + } }; // Seeds a closed EURUSD trade with an exact realized PnL. A zero-spread tick at @@ -207,6 +232,83 @@ TEST_CASE("LONG stops out when bid drops one pip below entry bid", "[tradeManage CHECK(*exit == 109990); } +// ENTRY_SLIPPAGE_TENTH_PIPS stress toggle: the haircut worsens what the entry +// PAID and nothing else. EURUSD is 10 points/pip, so 3 tenths = 3 points. The +// SL/TP anchors stay on the raw tick (exitReferencePrice), so a stressed and +// an unstressed run see identical market levels — only the PnL differs. +TEST_CASE("entry slippage worsens the LONG fill and leaves the anchors alone", + "[tradeManager]") { + TradeManager manager; + manager.entrySlippageTenthPips = 3; + PriceData entryTick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); + manager.openTrade(entryTick, 1, Direction::LONG, 1, 1); + auto trades = manager.getActiveTrades(); + auto trade = trades.find(entryTick.symbol); + REQUIRE(trade != trades.end()); + + CHECK(trade->second.entryPrice == 110013); // ask + 0.3 pip against us + CHECK(trade->second.entryAsk == 110010); // raw tick preserved (audit) + CHECK(trade->second.entryBid == 110000); + CHECK(trade->second.exitReferencePrice == 110000); // anchor untouched + CHECK(trade->second.stopPrice == 110000 - 10); // same as unslipped + CHECK(trade->second.limitPrice == 110000 + 10); + // Open-tick equity dips by spread (10) + slippage (3). + CHECK(trade->second.floatingPnl == -13); + CHECK(manager.unrealizedPnl() == -13); +} + +TEST_CASE("entry slippage worsens the SHORT fill symmetrically", + "[tradeManager]") { + TradeManager manager; + manager.entrySlippageTenthPips = 3; + PriceData entryTick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); + manager.openTrade(entryTick, 1, Direction::SHORT, 1, 1); + auto trades = manager.getActiveTrades(); + auto trade = trades.find(entryTick.symbol); + REQUIRE(trade != trades.end()); + + CHECK(trade->second.entryPrice == 109997); // bid - 0.3 pip against us + CHECK(trade->second.exitReferencePrice == 110010); // anchor = raw ask + CHECK(trade->second.stopPrice == 110010 + 10); + CHECK(trade->second.limitPrice == 110010 - 10); + CHECK(trade->second.floatingPnl == -13); // spread + slippage +} + +// The realized cost of the stress is exactly slipPoints x size: same entry +// tick, same close price, PnL differs by the haircut alone. +TEST_CASE("entry slippage costs slipPoints times size on a closed round-trip", + "[tradeManager]") { + PriceData entryTick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); + const std::int32_t closePrice = 110100; + + TradeManager plain; + plain.openTrade(entryTick, 2, Direction::LONG); + plain.closeTrade(entryTick.symbol, closePrice, entryTick); + + TradeManager stressed; + stressed.entrySlippageTenthPips = 3; + stressed.openTrade(entryTick, 2, Direction::LONG); + stressed.closeTrade(entryTick.symbol, closePrice, entryTick); + + CHECK(plain.calculatePnl() == (110100 - 110010) * 2); + CHECK(plain.calculatePnl() - stressed.calculatePnl() == 3 * 2); +} + +// Tenth-pips convert through the SYMBOL's points-per-pip: XAUUSD is 1000 +// points/pip, so the same 3-tenths setting is 300 points there, not 3. +TEST_CASE("entry slippage scales per symbol (metals: 3 tenths = 300 points)", + "[tradeManager]") { + TradeManager manager; + manager.entrySlippageTenthPips = 3; + PriceData entryTick(2400500, 2400000, std::chrono::system_clock::now(), "XAUUSD"); + manager.openTrade(entryTick, 1, Direction::LONG); + auto trades = manager.getActiveTrades(); + auto trade = trades.find(entryTick.symbol); + REQUIRE(trade != trades.end()); + CHECK(trade->second.entryPrice == 2400500 + 300); + CHECK(trade->second.exitReferencePrice == 2400000); +} + // LONG × SL/TP × flat tick: passing the entry tick itself back through // checkExit must not fire either side. This pins the spread-vs-stop // invariant: SL is anchored on the entry bid (exitReferencePrice), not @@ -503,7 +605,9 @@ TEST_CASE("Operations::run processes a multi-tick stream without throwing", "[tr // so trade outcomes (entry side, exit side, realised PnL) can be asserted // directly. Units: EURUSD has 10 stored price-points per pip. -// LONG entry executes at the ask; the stop/limit reference is the bid. +// LONG entry executes at the ask; the stop/limit reference is the bid. The run +// ends with the position still open, so end-of-data closes it at its last mark +// (the entry bid) — an ordinary close realizing the spread, not a liquidation. TEST_CASE("runTicks: LONG opens at the ask", "[tradeManager]") { TradeManager tm; ScriptedStrategy strategy({Direction::LONG}); @@ -514,17 +618,21 @@ TEST_CASE("runTicks: LONG opens at the ask", "[tradeManager]") { trading::runTicks(tm, strategy, ticks, vars); - REQUIRE(tm.getActiveTrades().size() == 1); - CHECK(tm.getClosedTrades().size() == 0); - const Trade& trade = tm.getActiveTrades().begin()->second; + CHECK(tm.getActiveTrades().size() == 0); + REQUIRE(tm.getClosedTrades().size() == 1); + const Trade& trade = tm.getClosedTrades().front(); CHECK(trade.direction == Direction::LONG); CHECK(trade.entryPrice == 110010); CHECK(trade.exitReferencePrice == 110000); CHECK(trade.entryAsk == 110010); CHECK(trade.entryBid == 110000); + CHECK(trade.closePrice == 110000); // last mark: the entry bid + CHECK(trade.pnl == -10); // the spread + CHECK_FALSE(trade.liquidated); } -// Symmetric SHORT: executes at the bid, exit reference is the ask. +// Symmetric SHORT: executes at the bid, exit reference is the ask; end-of-data +// closes it at the entry ask for the spread. TEST_CASE("runTicks: SHORT opens at the bid", "[tradeManager]") { TradeManager tm; ScriptedStrategy strategy({Direction::SHORT}); @@ -535,13 +643,17 @@ TEST_CASE("runTicks: SHORT opens at the bid", "[tradeManager]") { trading::runTicks(tm, strategy, ticks, vars); - REQUIRE(tm.getActiveTrades().size() == 1); - const Trade& trade = tm.getActiveTrades().begin()->second; + CHECK(tm.getActiveTrades().size() == 0); + REQUIRE(tm.getClosedTrades().size() == 1); + const Trade& trade = tm.getClosedTrades().front(); CHECK(trade.direction == Direction::SHORT); CHECK(trade.entryPrice == 110000); CHECK(trade.exitReferencePrice == 110010); CHECK(trade.entryAsk == 110010); CHECK(trade.entryBid == 110000); + CHECK(trade.closePrice == 110010); // last mark: the entry ask + CHECK(trade.pnl == -10); // the spread + CHECK_FALSE(trade.liquidated); } // No signal -> no position, across many ticks. @@ -564,7 +676,8 @@ TEST_CASE("runTicks: no signal opens nothing", "[tradeManager]") { // Re-entry is gated while a position is open: an always-signalling strategy on // flat ticks (price never reaches the 10-pip SL/TP) must still open only one -// trade for the symbol, not one per tick. +// trade for the symbol, not one per tick. The single position is then closed +// by end-of-data, so exactly one trade exists in total. TEST_CASE("runTicks: re-entry gated while active", "[tradeManager]") { TradeManager tm; AlwaysLongStrategy strategy; @@ -578,8 +691,8 @@ TEST_CASE("runTicks: re-entry gated while active", "[tradeManager]") { trading::runTicks(tm, strategy, ticks, vars); - CHECK(tm.getActiveTrades().size() == 1); - CHECK(tm.getClosedTrades().size() == 0); + CHECK(tm.getActiveTrades().size() == 0); + CHECK(tm.getClosedTrades().size() == 1); } // LONG take-profit: a 10-pip favourable move nets only 90 points (9 pips) @@ -646,6 +759,123 @@ TEST_CASE("runTicks: LONG SL closes at bid, negative PnL", "[tradeManager]") { CHECK(closed.pnl == -110); } +// Peak-hours filter: an out-of-session tick skips decide() entirely (never +// deferred) while during() still runs; the in-session tick enters as normal. +// EURUSD is a Europe symbol, and Wed 2026-07-15 is under BST, so the window +// is 07:00-10:00 UTC. +TEST_CASE("runTicks: peak-hours filter blocks out-of-session entries", + "[tradeManager]") { + TradeManager tm; + CountingLongStrategy strategy; + const auto vars = makeVars(0, 0, 1); // no SL/TP — end-of-data closes + const auto day = std::chrono::sys_days{std::chrono::year{2026} / 7 / 15}; + const std::vector ticks{ + PriceData(110010, 110000, day + std::chrono::hours{6}, "EURUSD"), + PriceData(110020, 110010, day + std::chrono::hours{8}, "EURUSD"), + }; + const trading::RiskLimits limits{.peakHoursOnly = true}; + + trading::runTicks(tm, strategy, ticks, vars, limits); + + CHECK(strategy.decideCalls == 1); // 06:00 tick never reached decide() + CHECK(strategy.duringCalls == 2); // during() is never gated + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().entryPrice == 110020); // the 08:00 ask +} + +// The filter gates ENTRIES only: a stop-loss still fires on an out-of-session +// tick. USDJPY is an Asia symbol (00:00-06:00 UTC), so the 12:00 tick is +// outside its window — yet it closes the trade opened at 01:00. +TEST_CASE("runTicks: peak-hours filter never gates exits", "[tradeManager]") { + TradeManager tm; + ScriptedStrategy strategy({Direction::LONG}); + const auto vars = makeVars(10, 0, 1); // SL only + const auto day = std::chrono::sys_days{std::chrono::year{2026} / 7 / 15}; + const std::vector ticks{ + // open LONG @ ask 110010, SL ref 110000 - 100 points + PriceData(110010, 110000, day + std::chrono::hours{1}, "USDJPY"), + // out-of-session, but bid 109900 hits the stop + PriceData(109910, 109900, day + std::chrono::hours{12}, "USDJPY"), + }; + const trading::RiskLimits limits{.peakHoursOnly = true}; + + trading::runTicks(tm, strategy, ticks, vars, limits); + + CHECK(tm.getActiveTrades().size() == 0); + REQUIRE(tm.getClosedTrades().size() == 1); + const Trade& closed = tm.getClosedTrades().front(); + CHECK(closed.closePrice == 109900); + CHECK_FALSE(closed.liquidated); +} + +// ATR entry conditions, cold store: with the gate wired but the gate series +// short of the 11 bars ATR(10) needs, every entry is skipped BEFORE decide() +// while during() still runs on every tick. Ticks a minute apart never roll a +// second 15m bar, so the store stays cold for the whole run. +TEST_CASE("runTicks: cold ATR gate skips decide() while during() still runs", + "[tradeManager]") { + setenv("OHLC_PREPOPULATE", "0", 1); // hermetic: no QuestDB warm-up query + TradeManager tm; + CountingLongStrategy strategy; + const auto vars = makeVars(1, 3, 1); // ATR multipliers + bars::BarStore store; + const bars::SeriesSpec gate{std::chrono::minutes{15}, 11}; + store.registerSeries(gate.minutes, gate.count); + + const auto day = std::chrono::sys_days{std::chrono::year{2026} / 7 / 15}; + std::vector ticks; + for (int i = 0; i < 5; ++i) { + ticks.emplace_back(110000, 110000, day + std::chrono::minutes{i}, + "EURUSD"); + } + + trading::runTicks(tm, strategy, ticks, vars, {}, &store, gate); + + CHECK(strategy.decideCalls == 0); // gate failed pre-decide on every tick + CHECK(strategy.duringCalls == 5); // during() is never gated + CHECK(tm.getActiveTrades().empty()); + CHECK(tm.getClosedTrades().empty()); +} + +// ATR entry conditions, warm store: zero-spread ticks 16 minutes apart each +// roll a fresh 15m bar and step the price 200 points (20 EURUSD pips), so +// every true range is 200 and ATR(10) reads exactly 200 points. The store +// updates BEFORE the gates, so the gate on tick i sees i+1 bars: tick 10 is +// the first warm one, and its entry carries the dynamic distances +// (stop = 20 pips x 1, limit = 20 pips x 3), not the raw multipliers. +TEST_CASE("runTicks: warm ATR gate opens with dynamic distances", + "[tradeManager]") { + setenv("OHLC_PREPOPULATE", "0", 1); + TradeManager tm; + CountingLongStrategy strategy; + const auto vars = makeVars(1, 3, 1); // ATR multipliers + bars::BarStore store; + const bars::SeriesSpec gate{std::chrono::minutes{15}, 11}; + store.registerSeries(gate.minutes, gate.count); + + const auto day = std::chrono::sys_days{std::chrono::year{2026} / 7 / 15}; + std::vector ticks; + for (int i = 0; i < 12; ++i) { + const std::int32_t price = 110000 + i * 200; + ticks.emplace_back(price, price, day + std::chrono::minutes{16 * i}, + "EURUSD"); + } + + trading::runTicks(tm, strategy, ticks, vars, {}, &store, gate); + + // Ticks 0-9 skipped pre-decide; tick 10 enters; tick 11 is gated by the + // open position, so decide() ran exactly once. + CHECK(strategy.decideCalls == 1); + // The end-of-data close settles the position; the closed trade still + // carries the ATR-derived distances the entry was opened with. + CHECK(tm.getActiveTrades().empty()); + REQUIRE(tm.getClosedTrades().size() == 1); + const Trade& closed = tm.getClosedTrades().front(); + CHECK(closed.entryPrice == 112000); // tick 10's ask + CHECK(closed.stopDistancePips == 20); + CHECK(closed.limitDistancePips == 60); +} + // Exit-before-entry ordering within a single tick: on the tick that stops the // first LONG out, reviewStopAndLimit closes it first, the symbol frees, and the // always-LONG strategy immediately re-enters on that same tick. @@ -661,17 +891,20 @@ TEST_CASE("runTicks: exit then same-tick re-entry", "[tradeManager]") { trading::runTicks(tm, strategy, ticks, vars); - REQUIRE(tm.getClosedTrades().size() == 1); - REQUIRE(tm.getActiveTrades().size() == 1); + // Two trades in close order: the stop-out, then the same-tick re-entry + // (closed by end-of-data at its last mark). + REQUIRE(tm.getClosedTrades().size() == 2); + CHECK(tm.getActiveTrades().size() == 0); const Trade& closed = tm.getClosedTrades().front(); CHECK(closed.entryPrice == 110010); CHECK(closed.closePrice == 109900); - const Trade& reentry = tm.getActiveTrades().begin()->second; + const Trade& reentry = tm.getClosedTrades().back(); CHECK(reentry.direction == Direction::LONG); CHECK(reentry.entryPrice == 109910); CHECK(reentry.exitReferencePrice == 109900); + CHECK_FALSE(reentry.liquidated); } // Two symbols open simultaneously, each entering on the correct side of its own @@ -688,10 +921,13 @@ TEST_CASE("runTicks: multi-symbol", "[tradeManager]") { trading::runTicks(tm, strategy, ticks, vars); - REQUIRE(tm.getActiveTrades().size() == 2); + // Both positions persist to end-of-data, where each closes at its own + // symbol's last mark. + CHECK(tm.getActiveTrades().size() == 0); + REQUIRE(tm.getClosedTrades().size() == 2); const Trade* eur = nullptr; const Trade* aus = nullptr; - for (const auto& [id, trade] : tm.getActiveTrades()) { + for (const auto& trade : tm.getClosedTrades()) { if (trade.symbol == "EURUSD") eur = ™ else if (trade.symbol == "AUSIDXAUD") aus = ™ } @@ -699,14 +935,17 @@ TEST_CASE("runTicks: multi-symbol", "[tradeManager]") { REQUIRE(aus != nullptr); CHECK(eur->entryPrice == 110010); CHECK(aus->entryPrice == 700050); + CHECK_FALSE(eur->liquidated); + CHECK_FALSE(aus->liquidated); } // --- Account loss limit (fail fast) --- // FLOATING drawdown alone must trigger the cutoff: no stop-loss, so the open // LONG's mark-to-market loss is the only thing the limit can see. The crash -// tick marks the trade at -1010 points; the floor is 10000 * 1% = 100 pips = -// 1000 points (pointsPerPip 10), so it breaches and the trade is liquidated. +// tick marks the trade at -1010 points; the pip BUDGET (balance × percent read +// directly as pips — no pip-value model) is 10000 × 1% = 100 pips = 1000 points +// (pointsPerPip 10), so it breaches and the trade is liquidated. TEST_CASE("runTicks: floating drawdown breach liquidates at mark", "[tradeManager]") { TradeManager tm; ScriptedStrategy strategy({Direction::LONG}); @@ -786,14 +1025,72 @@ TEST_CASE("runTicks: max-open-trades cap blocks the second entry", "[tradeManage const auto status = trading::runTicks(tm, strategy, ticks, vars, limits); CHECK(status == trading::RunStatus::Completed); - REQUIRE(tm.getActiveTrades().size() == 1); - CHECK(tm.getActiveTrades().begin()->second.symbol == std::string("EURUSD")); + CHECK(tm.getActiveTrades().size() == 0); + // Only the EURUSD entry got through the cap; end-of-data closed it. + REQUIRE(tm.getClosedTrades().size() == 1); + CHECK(tm.getClosedTrades().front().symbol == std::string("EURUSD")); +} + +// MAX_TRADES_PER_MINUTE is a sliding window over TICK time: with a cap of 1, +// the AUSIDXAUD signal 30s after the EURUSD entry is skipped (the window still +// holds that entry), but exactly 60s after it the entry has aged out (the +// window is half-open) and AUSIDXAUD enters — at THAT tick's price, proving +// the capped attempt was skipped outright, not deferred. A skipped tick never +// reaches decide(), so it doesn't consume a scripted signal either. +TEST_CASE("runTicks: trade rate cap enforces a sliding one-minute window", "[tradeManager]") { + TradeManager tm; + ScriptedStrategy strategy({Direction::LONG, Direction::LONG}); + const auto vars = makeVars(0, 0, 1); // no exits — both persist to end + const auto now = std::chrono::system_clock::now(); + const std::vector ticks{ + PriceData(110010, 110000, now, "EURUSD"), // opens + PriceData(700050, 700000, now + std::chrono::seconds{30}, "AUSIDXAUD"), // capped + PriceData(700150, 700100, now + std::chrono::seconds{60}, "AUSIDXAUD"), // opens + }; + const trading::RiskLimits limits{.maxTradesPerMinute = 1}; + + const auto status = trading::runTicks(tm, strategy, ticks, vars, limits); + + CHECK(status == trading::RunStatus::Completed); + CHECK(tm.getActiveTrades().size() == 0); + REQUIRE(tm.getClosedTrades().size() == 2); + const Trade* aus = nullptr; + for (const auto& trade : tm.getClosedTrades()) { + if (trade.symbol == "AUSIDXAUD") aus = ™ + } + REQUIRE(aus != nullptr); + // Entered on the third tick, not the capped second one. + CHECK(aus->entryPrice == 700150); +} + +// A same-instant burst counts against the cap too: three signals on one +// timestamp with a cap of 2 opens exactly two trades (the window can never +// slide within a single tick's timestamp). +TEST_CASE("runTicks: trade rate cap blocks a same-instant burst", "[tradeManager]") { + TradeManager tm; + ScriptedStrategy strategy({Direction::LONG, Direction::LONG, Direction::LONG}); + const auto vars = makeVars(0, 0, 1); // no exits + const auto now = std::chrono::system_clock::now(); + const std::vector ticks{ + PriceData(110010, 110000, now, "EURUSD"), + PriceData(700050, 700000, now, "AUSIDXAUD"), + PriceData(130010, 130000, now, "GBPUSD"), // capped + }; + const trading::RiskLimits limits{.maxTradesPerMinute = 2}; + + const auto status = trading::runTicks(tm, strategy, ticks, vars, limits); + + CHECK(status == trading::RunStatus::Completed); + REQUIRE(tm.getClosedTrades().size() == 2); + for (const auto& trade : tm.getClosedTrades()) { + CHECK(trade.symbol != std::string("GBPUSD")); + } } // A stop-out that breaches the loss limit must end the run on that tick, BEFORE // the entry phase — so the always-LONG strategy gets no same-tick re-entry, and -// later ticks never run. Floor: 10000 * 0.1% = 10 pips = 100 points; the single -// stop-out loses 110 points (incl. spread), so it breaches. +// later ticks never run. Pip budget: 10000 × 0.1% = 10 pips = 100 points; the +// single stop-out loses 110 points (incl. spread), so it breaches. TEST_CASE("runTicks: loss-limit breach stops run before re-entry", "[tradeManager]") { TradeManager tm; AlwaysLongStrategy strategy; @@ -818,8 +1115,9 @@ TEST_CASE("runTicks: loss-limit breach stops run before re-entry", "[tradeManage } // A realized loss inside the limit must not stop the run: same stop-out, but a -// 5% limit (floor 5000 points) comfortably absorbs the -110-point loss, so the -// run completes and the same-tick re-entry happens. +// 5% budget (500 pips = 5000 points) comfortably absorbs the -110-point loss, +// so the run completes and the same-tick re-entry happens (and is then closed +// by end-of-data as an ordinary, non-liquidated close). TEST_CASE("runTicks: loss within limit runs to completion", "[tradeManager]") { TradeManager tm; AlwaysLongStrategy strategy; @@ -836,8 +1134,9 @@ TEST_CASE("runTicks: loss within limit runs to completion", "[tradeManager]") { const auto status = trading::runTicks(tm, strategy, ticks, vars, limits); CHECK(status == trading::RunStatus::Completed); - CHECK(tm.getClosedTrades().size() == 1); - CHECK(tm.getActiveTrades().size() == 1); + REQUIRE(tm.getClosedTrades().size() == 2); // stop-out + end-of-data close + CHECK(tm.getActiveTrades().size() == 0); + CHECK_FALSE(tm.getClosedTrades().back().liquidated); } // maxLossPercent <= 0 disables the check entirely — losses far past any @@ -861,10 +1160,113 @@ TEST_CASE("runTicks: loss limit disabled for zero and negative", "[tradeManager] const auto status = trading::runTicks(tm, strategy, ticks, vars, limits); CHECK(status == trading::RunStatus::Completed); - CHECK(tm.getActiveTrades().size() == 1); + // Stop-out plus the end-of-data close of the same-tick re-entry. + CHECK(tm.getClosedTrades().size() == 2); + CHECK(tm.getActiveTrades().size() == 0); } } +// --- runTicks performance gate --- + +// The thresholds below are deliberately test-local: they exercise the gate +// machinery (strict more-than comparisons, each check independently +// disableable, the Underperformed status) — not the production values +// Operations configures. + +namespace { + +// Deterministic two-winner fixture for the gate tests: two TP exits (+90 each, +// see the TP cases above for the unit conventions) with zero drawdown, giving +// exactly 2 decisive trades and a modest positive performance score. +trading::RunStatus runTwoWinnerFixture(const trading::RiskLimits& limits) { + TradeManager tm; + ScriptedStrategy strategy({Direction::LONG, Direction::LONG}); + const auto vars = makeVars(0, 10, 1); // TP only + const auto now = std::chrono::system_clock::now(); + const std::vector ticks{ + PriceData(110010, 110000, now, "EURUSD"), // LONG#1 @ ask 110010 + PriceData(110110, 110100, now, "EURUSD"), // TP#1; re-enter @ 110110 + PriceData(110210, 110200, now, "EURUSD"), // TP#2; script exhausted + }; + return trading::runTicks(tm, strategy, ticks, vars, limits); +} + +} // namespace + +TEST_CASE("runTicks: performance gate off by default", "[tradeManager]") { + CHECK(runTwoWinnerFixture({}) == trading::RunStatus::Completed); +} + +// The floor is a strict more-than on decisive (winner/loser) trades: with 2 +// decisive trades a floor of 1 passes, a floor of 2 does not. +TEST_CASE("runTicks: decisive-trade floor gates completion", "[tradeManager]") { + CHECK(runTwoWinnerFixture({.minDecisiveTrades = 1}) + == trading::RunStatus::Completed); + CHECK(runTwoWinnerFixture({.minDecisiveTrades = 2}) + == trading::RunStatus::Underperformed); +} + +// The two-winner fixture scores comfortably above 1 and nowhere near 1000 +// (zero drawdown, positive expectancy); lastMonths supplies the annualisation +// horizon the score needs whenever the score threshold is active. +TEST_CASE("runTicks: performance-score threshold gates completion", "[tradeManager]") { + CHECK(runTwoWinnerFixture({.minPerformanceScore = "1"_dd, .lastMonths = 1}) + == trading::RunStatus::Completed); + CHECK(runTwoWinnerFixture({.minPerformanceScore = "1000"_dd, .lastMonths = 1}) + == trading::RunStatus::Underperformed); +} + +// End-of-data close realizes the CURRENT mark, not the entry: the open LONG has +// floated +90 (bid 110100 vs entry ask 110010) when the data runs out, so the +// forced close books +90 and the account carries no dangling floating PnL into +// reporting (finalPnl and max drawdown now describe the same equity curve). +TEST_CASE("runTicks: end of data closes open trades at their last mark", "[tradeManager]") { + TradeManager tm; + ScriptedStrategy strategy({Direction::LONG}); + const auto vars = makeVars(0, 0, 1); // no SL/TP: only end-of-data can close + const auto now = std::chrono::system_clock::now(); + const std::vector ticks{ + PriceData(110010, 110000, now, "EURUSD"), // open LONG @ ask 110010 + PriceData(110110, 110100, now, "EURUSD"), // mark at bid 110100: +90 + }; + + const auto status = trading::runTicks(tm, strategy, ticks, vars); + + CHECK(status == trading::RunStatus::Completed); + CHECK(tm.getActiveTrades().size() == 0); + REQUIRE(tm.getClosedTrades().size() == 1); + const Trade& closed = tm.getClosedTrades().front(); + CHECK(closed.closePrice == 110100); + CHECK(closed.pnl == 90); + CHECK_FALSE(closed.liquidated); + CHECK(tm.calculatePnl() == 90); + CHECK(tm.unrealizedPnl() == 0); +} + +// The loss floor is a budget on PRICE MOVEMENT, so it scales with trade size: +// PnL is tracked in points × size, and an unscaled floor would silently shrink +// to budget/size pips. Size 2 with a 15-pip budget (150 points, scaled floor +// -300): the stop-out books -110 points × 2 = -220, inside the scaled floor — +// an unscaled floor (-150) would have (wrongly) ended this run. +TEST_CASE("runTicks: loss floor scales with trade size", "[tradeManager]") { + TradeManager tm; + AlwaysLongStrategy strategy; + const auto vars = makeVars(10, 0, 2); // SL only, size 2 + const auto now = std::chrono::system_clock::now(); + const std::vector ticks{ + PriceData(110010, 110000, now, "EURUSD"), + PriceData(109910, 109900, now, "EURUSD"), // stop-out: -110 points × 2 + }; + const trading::RiskLimits limits{.startingBalance = "10000"_dd, + .maxLossPercent = "0.15"_dd, + .pointsPerPip = 10}; + + const auto status = trading::runTicks(tm, strategy, ticks, vars, limits); + + CHECK(status == trading::RunStatus::Completed); + CHECK(tm.calculatePnl() == -240); // -220 stop-out, -20 re-entry spread ×2 +} + // --- Performance score (ResultsSummary::collect) --- // Guard: with no closed trades there is nothing to score, so every score field @@ -913,6 +1315,48 @@ TEST_CASE("score: all winners pin winRate=1 and tradeRatio=100", "[score]") { CHECK(static_cast(stats.performanceScore) > 0.0); } +// winRate is the share of ALL closed trades that won — breakevens count in the +// denominator, so one winner among three breakevens is 25%, not a forced 100% +// (previously a run with zero losers pinned winRate to 1.0 no matter how many +// breakevens it had). +TEST_CASE("score: breakevens dilute winRate", "[score]") { + TradeManager tm; + seedClosedTrade(tm, 100); // +10 pips + seedClosedTrade(tm, 0); // breakeven + seedClosedTrade(tm, 0); // breakeven + seedClosedTrade(tm, 0); // breakeven + const auto config = makeRandomStrategyConfig(); + + const auto stats = ResultsSummary::collect(tm, config); + + CHECK(stats.winners == 1); + CHECK(stats.breakeven == 3); + CHECK(static_cast(stats.winRate) == Catch::Approx(0.25)); + // Still no losers, so tradeRatio stays pinned at the cap. + CHECK(static_cast(stats.tradeRatio) == 100.0); +} + +// A closed trade on a symbol with no known scale (scalingFactor == 0) cannot be +// expressed in pips: it is excluded from winners/losers/breakeven and the pip +// sums alike — counting it as a winner that contributes zero pips would skew +// the averages (and could zero averageLoss into a divide-by-Inf tradeRatio). +TEST_CASE("score: unknown-symbol trades are excluded from pip metrics", "[score]") { + TradeManager tm; + seedClosedTrade(tm, 100); // +10 pips on EURUSD (known scale) + PriceData tick(500, 500, std::chrono::system_clock::now(), "ZZZTEST"); + tm.openTrade(tick, 1, Direction::LONG); + tm.closeTrade("ZZZTEST", 600, tick); // +100 points on an unknown scale + const auto config = makeRandomStrategyConfig(); + + const auto stats = ResultsSummary::collect(tm, config); + + CHECK(stats.tradesClosed == 2); + CHECK(stats.winners == 1); // only the EURUSD winner is measurable + CHECK(stats.losers == 0); + CHECK(stats.breakeven == 0); + CHECK(static_cast(stats.finalPnl) == Catch::Approx(10.0)); +} + // Mixed run pins the derived stats to hand-computable values. PnL sequence in // pips: +10, -4, -3, +2 -> cumulative 10, 6, 3, 5, so the realized peak-to-trough // is 10 - 3 = 7 pips. Against a 10000 balance that is 0.07%. winRate = 2/4 = 0.5; @@ -939,9 +1383,10 @@ TEST_CASE("score: drawdown and ratios match a hand-computed run", "[score]") { // because the only closed trade is a winner — the mark-to-market tracker must // still see the trough the open position sat through. Open @ ask 110010, mark // down to bid 109000 (floating -1010 points = -101 pips), then TP-close at +90. +// Scripted (single-shot) so no same-tick re-entry muddies the trade count. TEST_CASE("score: drawdown captures an intra-trade float, not just closes", "[score]") { TradeManager tm; - AlwaysLongStrategy strategy; + ScriptedStrategy strategy({Direction::LONG}); const auto vars = makeVars(0, 10, 1); // TP only, no SL — let it float const auto now = std::chrono::system_clock::now(); const std::vector ticks{