diff --git a/CMakeLists.txt b/CMakeLists.txt index 34ded42..09093c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,6 +93,20 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) # module flags for loadCommand.cppm — are always present for the editor. set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +# Clang 22 made "reduced BMI" the default for C++20 modules, and reduced BMIs +# drop the predeclared global operator new/delete from the global module +# (llvm/llvm-project#166068). A TU that imports std alongside enough of this +# project's modules then segfaults clang's codegen (EmitBuiltinNewDeleteCall, +# release build's llvm_unreachable) while emitting __builtin_operator_new from +# libc++'s __libcpp_allocate — main.cpp was the victim. Force full BMIs (the +# pre-22 behaviour) until the upstream fix reaches a release. Global +# CMAKE_CXX_FLAGS on purpose: it must also reach CMake's internal `import std` +# target (see the OpenMP note above the flags append near the bottom). +if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" + AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 22) + string(APPEND CMAKE_CXX_FLAGS " -fno-modules-reduced-bmi") +endif() + # Configure libpqxx build set(PQXX_LIBRARIES_INSTALL ON) set(SKIP_BUILD_TEST ON) @@ -198,7 +212,9 @@ string(APPEND CMAKE_CXX_FLAGS " ${OpenMP_CXX_FLAGS}") # compiled into one dedicated TU via # (source/shared/redis/connection/boostRedisImpl.cpp), which isolates its # macro/SSL pollution from the rest of the build. It pulls in Boost.Asio's SSL -# layer, which needs OpenSSL. +# layer, which needs OpenSSL. Boost.Interprocess (the shared-memory control +# channel, source/shared/ipc/engineControl.cpp) is also header-only; on Linux it +# needs librt for shm_open (glibc < 2.34), which is a no-op elsewhere. find_package(Boost REQUIRED CONFIG) find_package(OpenSSL REQUIRED) find_package(Threads REQUIRED) @@ -208,7 +224,8 @@ target_link_libraries(BacktestingEngineLib PUBLIC OpenSSL::SSL OpenSSL::Crypto Threads::Threads - CURL::libcurl) + CURL::libcurl + $<$:rt>) add_subdirectory(external/boost-decimal) target_link_libraries(BacktestingEngineLib PUBLIC Boost::decimal) diff --git a/LICENSE.MD b/LICENSE.MD index 5bf1c6d..55b8002 100644 --- a/LICENSE.MD +++ b/LICENSE.MD @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 Ryan McCaffery +Copyright (c) 2026 Ryan McCaffery https://mccaffers.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/scripts/ingest.sh b/scripts/ingest.sh index bc42579..f96ae0f 100755 --- a/scripts/ingest.sh +++ b/scripts/ingest.sh @@ -1,11 +1,11 @@ #!/bin/bash # Builds the engine and runs the `ingest` subcommand: it binds a UDP socket and -# streams decoded ticks (tickPacket) into the converted QuestDB via ILP-over-HTTP +# streams decoded ticks (tickPacket) into QuestDB via ILP-over-HTTP # (source/ingest/ingestCommand.cppm). Blocks until SIGINT/SIGTERM. # # Config is read from the environment by IngestCommand (defaults in parens): # QUESTDB_HOST QuestDB host (127.0.0.1) -# QUESTDB_ILP_PORT converted-DB ILP/HTTP port (9001) +# QUESTDB_ILP_PORT QuestDB ILP/HTTP port (9000) # INGEST_BIND_ADDR UDP bind address (127.0.0.1) # INGEST_UDP_PORT UDP bind port (11111, == UDPPorts.PortSave) # @@ -30,7 +30,7 @@ fi # QuestDB is down (the writer sheds once its buffer fills), so warn rather than # abort. quest_host="${QUESTDB_HOST:-127.0.0.1}" -quest_port="${QUESTDB_ILP_PORT:-9001}" +quest_port="${QUESTDB_ILP_PORT:-9000}" # Probe a real endpoint (/exec), not "/": QuestDB's web console root can stall # indefinitely, which made a 1s timeout report a healthy DB as unreachable. if ! curl --fail --silent --max-time 2 "http://$quest_host:$quest_port/exec?query=SELECT%201" >/dev/null 2>&1; then diff --git a/scripts/load.sh b/scripts/load.sh index 0bc277d..c16348f 100755 --- a/scripts/load.sh +++ b/scripts/load.sh @@ -15,9 +15,9 @@ if [ ! -f "$BUILD_DIR/$EXECUTABLE_NAME" ]; then exit 1 fi -if ! redis-cli -h localhost ping >/dev/null 2>&1; then - echo "redis-server not reachable on localhost:6379 — aborting" - exit 1 +if ! redis-cli -h "$REDIS_HOST" ping >/dev/null 2>&1; then + echo "redis-server not reachable on $REDIS_HOST:6379 — skipping" + exit 0 fi exec ./"$BUILD_DIR/$EXECUTABLE_NAME" load diff --git a/scripts/run.sh b/scripts/run.sh index c42427b..e1e8e86 100644 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -31,8 +31,8 @@ else exit 1 fi -if ! redis-cli -h localhost ping >/dev/null 2>&1; then - echo "redis-server not reachable on localhost:6379 — skipping" +if ! redis-cli -h "$REDIS_HOST" ping >/dev/null 2>&1; then + echo "redis-server not reachable on $REDIS_HOST:6379 — skipping" exit 0 fi diff --git a/source/ingest/ingestCommand.cppm b/source/ingest/ingestCommand.cppm index b4e018c..6110ff3 100644 --- a/source/ingest/ingestCommand.cppm +++ b/source/ingest/ingestCommand.cppm @@ -14,6 +14,8 @@ module; +#include // POSIX gmtime_r (not exported by `import std`) + #include "ingest/questdbIngestClient.hpp" #include "ingest/udpPorts.hpp" #include "ingest/udpReceiver.hpp" @@ -43,14 +45,40 @@ std::uint16_t parsePort(std::string_view text, std::uint16_t fallback) { return static_cast(value); } +// UTC wall-clock prefix, e.g. "[2026-06-28 15:04:20.785]", to millisecond +// precision. gmtime_r is POSIX (hence the include in the global module +// fragment) — the same approach tradeManager.cppm uses for UTC tick times. +std::string timestamp() { + const auto now = std::chrono::system_clock::now(); + const std::time_t t = std::chrono::system_clock::to_time_t(now); + const auto millis = std::chrono::duration_cast( + now.time_since_epoch()) + .count() % + 1000; + std::tm utc{}; + gmtime_r(&t, &utc); + return std::format("[{:04}-{:02}-{:02} {:02}:{:02}:{:02}.{:03}]", + utc.tm_year + 1900, utc.tm_mon + 1, utc.tm_mday, + utc.tm_hour, utc.tm_min, utc.tm_sec, millis); +} + +// Like std::println to stdout, but timestamped and flushed immediately. stdout +// is fully buffered when redirected (pipe/file/service log), so without the +// flush the once-a-minute reports below would sit in the buffer and only appear +// in a burst when the process exits. +template +void logLine(std::format_string fmt, Args&&... args) { + std::println("{} {}", timestamp(), std::format(fmt, std::forward(args)...)); + std::fflush(stdout); +} + } // namespace int IngestCommand::run(const int argc, const char* argv[]) { const std::string questHost = env::getOr("QUESTDB_HOST", "127.0.0.1"); - // 9001 is the converted (INT32) QuestDB's HTTP/ILP port — the instance the - // backtester reads and that convert.py writes to. (9000 is the legacy DOUBLE - // source DB.) Override with $QUESTDB_ILP_PORT. - const std::uint16_t ilpPort = parsePort(env::getOr("QUESTDB_ILP_PORT", ""), 9001); + // QuestDB serves ILP-over-HTTP (POST /write) on its main HTTP port, 9000 by + // default. Override with $QUESTDB_ILP_PORT. + const std::uint16_t ilpPort = parsePort(env::getOr("QUESTDB_ILP_PORT", ""), 9000); const std::string bindAddr = env::getOr("INGEST_BIND_ADDR", "127.0.0.1"); // Bind port: CLI arg (argv[2]) overrides $INGEST_UDP_PORT overrides kSave. @@ -82,16 +110,52 @@ int IngestCommand::run(const int argc, const char* argv[]) { received.fetch_add(1, std::memory_order_relaxed); }); - std::println("IngestCommand: starting; binding udp://{}:{} -> QuestDB ILP http://{}:{}/write", - bindAddr, bindPort, questHost, ilpPort); - - if (!receiver.run()) { // binds the socket, then blocks until SIGINT/SIGTERM - return 1; // bind failed (bad address / port in use) — logged + logLine("IngestCommand: starting; binding udp://{}:{} -> QuestDB ILP http://{}:{}/write", + bindAddr, bindPort, questHost, ilpPort); + + // receiver.run() blocks the main thread, so a background thread prints + // throughput once a minute. It wakes every second to notice shutdown + // promptly. Plain std::thread + atomic flag on purpose — std::jthread's + // stop_token wait (condition_variable_any) doesn't link under `import std` + // here (see module-migration notes). + std::atomic reporting{true}; + std::thread reporter([&] { + using clock = std::chrono::steady_clock; + constexpr auto interval = std::chrono::minutes(1); + std::uint64_t lastReceived = 0; + auto nextReport = clock::now() + interval; + while (reporting.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + if (clock::now() < nextReport) { + continue; + } + nextReport += interval; + const auto total = received.load(std::memory_order_relaxed); + const auto delta = total - lastReceived; + lastReceived = total; + logLine("IngestCommand: received={} (+{}, ~{}/s); " + "decode-dropped={}, queue-dropped={}, post-failed={}", + total, delta, delta / 60, + dropped.load(std::memory_order_relaxed), writer.droppedLines(), + writer.failedLines()); + } + }); + + const bool ok = receiver.run(); // binds the socket, then blocks until SIGINT/SIGTERM + + reporting.store(false, std::memory_order_relaxed); + reporter.join(); + + if (!ok) { + return 1; // bind failed (bad address / port in use) — logged } // decode-dropped: bad size / unknown symbol. queue-dropped: shed by the // writer when QuestDB couldn't keep up and the buffer hit its cap. - std::println("IngestCommand: shutting down (received={}, decode-dropped={}, queue-dropped={})", - received.load(), dropped.load(), writer.droppedLines()); + // post-failed: lines a POST couldn't persist (transport error / non-2xx). + logLine("IngestCommand: shutting down (received={}, decode-dropped={}, " + "queue-dropped={}, post-failed={})", + received.load(), dropped.load(), writer.droppedLines(), + writer.failedLines()); return 0; } diff --git a/source/ingest/questdbIngestClient.cpp b/source/ingest/questdbIngestClient.cpp index af6a06a..abeef4e 100644 --- a/source/ingest/questdbIngestClient.cpp +++ b/source/ingest/questdbIngestClient.cpp @@ -12,7 +12,9 @@ #include "ingest/questdbIngestClient.hpp" #include +#include #include +#include #include #include #include @@ -33,18 +35,37 @@ constexpr long kTransferTimeoutSeconds = 10; void ensureCurlInit() { static const struct CurlGlobal { - CurlGlobal() { curl_global_init(CURL_GLOBAL_ALL); } + CurlGlobal() { + // Ignore SIGPIPE process-wide. The worker POSTs with CURLOPT_NOSIGNAL + // (so libcurl no longer guards SIGPIPE itself); without this, a write + // to a QuestDB connection that was reset mid-transfer would raise + // SIGPIPE, whose default action kills the process. Asio's signal_set + // only handles SIGINT/SIGTERM, so nothing else catches it. + std::signal(SIGPIPE, SIG_IGN); + curl_global_init(CURL_GLOBAL_ALL); + } ~CurlGlobal() { curl_global_cleanup(); } } guard; (void)guard; } -// Swallow the response body so curl does not dump it to stdout (its default when -// no write callback is set). QuestDB replies 204 on success, 400 + a JSON body -// describing the offending line on a parse/type error. -std::size_t discardResponse(char* /*ptr*/, std::size_t size, std::size_t nmemb, - void* /*userdata*/) { - return size * nmemb; +// Capture the response body (bounded) into the std::string at `userdata`, so a +// non-2xx reply can be logged. QuestDB replies 204 with an empty body on +// success, or 400 + a JSON body naming the offending line on a parse/type error +// — that body is the only place the reason appears. We keep at most +// kMaxResponseLog bytes so a large/hostile reply can't bloat the log line, but +// still tell curl we consumed everything (without a write callback curl would +// dump the body to stdout). +constexpr std::size_t kMaxResponseLog = 1024; + +std::size_t captureResponse(char* ptr, std::size_t size, std::size_t nmemb, + void* userdata) { + const std::size_t n = size * nmemb; + auto* body = static_cast(userdata); + if (body->size() < kMaxResponseLog) { + body->append(ptr, std::min(n, kMaxResponseLog - body->size())); + } + return n; } } // namespace @@ -63,6 +84,8 @@ struct QuestdbIngestClient::Impl { // link bug, and this is a classic TU std::deque queue; std::size_t droppedTotal = 0; // guarded by mtx + std::atomic failedTotal{0}; // POST/HTTP failures; worker writes, + // reporter reads — atomic, no mtx CURL* curl = nullptr; // owned by the worker thread only std::jthread worker; @@ -109,6 +132,7 @@ struct QuestdbIngestClient::Impl { // the queue was empty (nothing sent). bool drainAndPost() { std::string batch; + std::size_t lineCount = 0; { std::unique_lock lock(mtx); const std::size_t n = std::min(queue.size(), batchSize); @@ -116,24 +140,37 @@ struct QuestdbIngestClient::Impl { batch += std::move(queue.front()); queue.pop_front(); } + lineCount = n; } if (batch.empty()) { return false; } - post(batch); + post(batch, lineCount); return true; } - void post(const std::string& body) { + // POST one batch body holding `lineCount` lines. On any failure (init, + // transport error, or non-2xx) those lines did not persist, so they are + // added to failedTotal. + void post(const std::string& body, std::size_t lineCount) { if (!curl) { + failedTotal.fetch_add(lineCount, std::memory_order_relaxed); return; // init failed earlier; error already logged } + std::string responseBody; curl_easy_reset(curl); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_POST, 1L); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.data()); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(body.size())); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, discardResponse); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, captureResponse); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseBody); + // Run signal-free: this POST happens on the worker thread, and libcurl's + // default signal-based (SIGALRM/siglongjmp) timeout path is not + // thread-safe and races the main thread's Asio signal_set. NOSIGNAL also + // suppresses libcurl's SIGPIPE handling — we ignore SIGPIPE globally in + // ensureCurlInit() to compensate. + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); // Bound blocking time so a dead/slow QuestDB can't wedge the worker. curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, kConnectTimeoutSeconds); curl_easy_setopt(curl, CURLOPT_TIMEOUT, kTransferTimeoutSeconds); @@ -142,13 +179,15 @@ struct QuestdbIngestClient::Impl { if (rc != CURLE_OK) { backtest_log::error(std::string("QuestdbIngestClient: POST failed: ") + curl_easy_strerror(rc)); + failedTotal.fetch_add(lineCount, std::memory_order_relaxed); return; } long status = 0; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); if (status < 200 || status >= 300) { backtest_log::error("QuestdbIngestClient: HTTP " + std::to_string(status) - + " from " + url); + + " from " + url + ": " + responseBody); + failedTotal.fetch_add(lineCount, std::memory_order_relaxed); } } }; @@ -194,4 +233,8 @@ std::size_t QuestdbIngestClient::droppedLines() const { return impl_->droppedTotal; } +std::size_t QuestdbIngestClient::failedLines() const { + return impl_->failedTotal.load(std::memory_order_relaxed); +} + } // namespace ingest diff --git a/source/ingest/questdbIngestClient.hpp b/source/ingest/questdbIngestClient.hpp index 7fa8a1b..b52207f 100644 --- a/source/ingest/questdbIngestClient.hpp +++ b/source/ingest/questdbIngestClient.hpp @@ -31,7 +31,7 @@ class QuestdbIngestClient { // when QuestDB is slow/down — past it the oldest lines are dropped so memory // stays bounded rather than growing without limit. explicit QuestdbIngestClient(std::string host, - std::uint16_t port = 9001, + std::uint16_t port = 9000, std::size_t batchSize = 1000, std::chrono::milliseconds flushInterval = std::chrono::milliseconds(100), @@ -50,6 +50,11 @@ class QuestdbIngestClient { // keep up). Thread-safe. [[nodiscard]] std::size_t droppedLines() const; + // Total lines that failed to persist because a POST errored (transport + // failure / timeout) or QuestDB returned a non-2xx status. Distinct from + // droppedLines (those never left the queue). Thread-safe. + [[nodiscard]] std::size_t failedLines() const; + private: struct Impl; std::unique_ptr impl_; diff --git a/source/ingest/tickPacket.cppm b/source/ingest/tickPacket.cppm index 1624e35..0b33941 100644 --- a/source/ingest/tickPacket.cppm +++ b/source/ingest/tickPacket.cppm @@ -61,11 +61,14 @@ template // rather than a hard error, since UDP is best-effort and a stray/garbled or // malformed datagram must not take the ingest down — for any of: // - a wrong-sized packet, -// - a zero bid or ask (the bad-data signature found in the historical feed; -// the C# producer already guards this at SubListener.cs:126 — we re-check -// here as defense in depth against a future non-guarding producer), -// - a non-positive timestamp (unset / pre-1970 — the null/epoch signature of -// the same bad rows), +// - a bid or ask that isn't finite and strictly positive — zero is the +// bad-data signature found in the historical feed (the C# producer already +// guards it at SubListener.cs:126; we re-check here as defense in depth +// against a future non-guarding producer), and NaN/Inf/negatives are the +// same class of corrupt/unset data, +// - an out-of-range timestamp (<= 0 is unset / pre-1970; an implausibly large +// value is corrupt and would overflow the micros->nanos conversion on the +// write path), // - an unknown symbol (multiplier 0), // - a scaled price that overflows INT32 (latent: real prices sit ~300x below // the limit, but we never silently store a truncated value). @@ -78,15 +81,21 @@ template const double ask = detail::readLE(bytes.subspan(kAskOffset)); const std::int64_t tsMicros = detail::readLE(bytes.subspan(kTsOffset)); - // Drop zero-price ticks: a real quote always has a non-zero bid and ask, so - // a 0 here is corrupt/unset data, not a tradable price. - if (bid == 0.0 || ask == 0.0) { + // Drop anything that isn't a real, tradable price. A genuine quote is always + // finite and strictly positive, so this one test rejects every corrupt/unset + // signature at once: NaN/Inf (which would otherwise sail past an == 0 check + // and give std::llround unspecified behaviour), zero, and negatives. + if (!std::isfinite(bid) || !std::isfinite(ask) || bid <= 0.0 || ask <= 0.0) { return std::nullopt; } - // Drop unset / pre-epoch timestamps: a real tick is always strictly after - // the Unix epoch, so <= 0 is the null-date signature. - if (tsMicros <= 0) { + // Drop out-of-range timestamps. A real tick is strictly after the Unix epoch + // (<= 0 is the null-date signature), and the write path converts these micros + // to nanoseconds (x1000) — so a corrupt, implausibly-large value would + // overflow int64 there (signed-overflow UB, garbage timestamp). Bound both + // ends; kMaxTsMicros is 2100-01-01Z, far past any real feed. + constexpr std::int64_t kMaxTsMicros = 4'102'444'800'000'000; + if (tsMicros <= 0 || tsMicros > kMaxTsMicros) { return std::nullopt; } diff --git a/source/ingest/udpReceiver.cpp b/source/ingest/udpReceiver.cpp index f3587a8..1664d7b 100644 --- a/source/ingest/udpReceiver.cpp +++ b/source/ingest/udpReceiver.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -46,19 +47,19 @@ struct UdpReceiver::Impl { // throwing, so a bad address or an already-bound port is a clean error // rather than an uncaught exception / std::terminate. bool open() { - boost::system::error_code ec; - const auto address = asio::ip::make_address(bindAddr, ec); - if (ec) { + boost::system::error_code errorCodes; + const auto address = asio::ip::make_address(bindAddr, errorCodes); + if (errorCodes) { backtest_log::error("UdpReceiver: invalid bind address '" + bindAddr - + "': " + ec.message()); + + "': " + errorCodes.message()); return false; } const udp::endpoint endpoint(address, port); - if (const auto openEc = socket.open(endpoint.protocol(), ec); openEc) { + if (const auto openEc = socket.open(endpoint.protocol(), errorCodes); openEc) { backtest_log::error("UdpReceiver: socket open failed: " + openEc.message()); return false; } - if (const auto bindEc = socket.bind(endpoint, ec); bindEc) { + if (const auto bindEc = socket.bind(endpoint, errorCodes); bindEc) { backtest_log::error("UdpReceiver: cannot bind " + bindAddr + ":" + std::to_string(port) + ": " + bindEc.message()); return false; @@ -69,13 +70,24 @@ struct UdpReceiver::Impl { void receive() { socket.async_receive_from( asio::buffer(buffer), sender, - [this](const boost::system::error_code& ec, std::size_t n) { - if (!ec) { - handler(std::span(buffer.data(), n)); - } else if (ec == asio::error::operation_aborted) { + [this](const boost::system::error_code& errorCodes, const std::size_t n) { + if (!errorCodes) { + // The handler must never take the receive loop down: a single + // bad_alloc (or any handler-thrown exception) would otherwise + // escape io_context::run() and terminate the process, and + // skip the re-arm below. Swallow it, log, keep receiving. + try { + handler(std::span(buffer.data(), n)); + } catch (const std::exception& e) { + backtest_log::error(std::string("UdpReceiver: handler threw: ") + + e.what()); + } catch (...) { + backtest_log::error("UdpReceiver: handler threw non-std exception"); + } + } else if (errorCodes == asio::error::operation_aborted) { return; // socket closed during shutdown — stop re-arming } else { - backtest_log::error("UdpReceiver: " + ec.message()); + backtest_log::error("UdpReceiver: " + errorCodes.message()); } if (socket.is_open()) { receive(); // re-arm for the next datagram @@ -84,8 +96,8 @@ struct UdpReceiver::Impl { } void shutdown() { - boost::system::error_code ec; - if (const auto closeEc = socket.close(ec); closeEc) { + boost::system::error_code errorCodes; + if (const auto closeEc = socket.close(errorCodes); closeEc) { backtest_log::error("UdpReceiver: socket close failed: " + closeEc.message()); } ioc.stop(); @@ -97,7 +109,8 @@ UdpReceiver::UdpReceiver(std::string bindAddr, std::uint16_t port, Handler onDat UdpReceiver::~UdpReceiver() = default; -bool UdpReceiver::run() { +bool UdpReceiver::run() +{ if (!impl_->open()) { return false; // bind failed; reason already logged } @@ -108,7 +121,8 @@ bool UdpReceiver::run() { return true; } -void UdpReceiver::stop() { +void UdpReceiver::stop() +{ // Hand the teardown to the io_context thread so the socket is only touched // from there (Asio objects are not thread-safe for concurrent use). asio::post(impl_->ioc, [this] { impl_->shutdown(); }); diff --git a/source/load/config/ohlcBreakoutStrategy/makeOhlcBreakoutStrategy.cppm b/source/load/config/ohlcBreakoutStrategy/makeOhlcBreakoutStrategy.cppm new file mode 100644 index 0000000..3947f1d --- /dev/null +++ b/source/load/config/ohlcBreakoutStrategy/makeOhlcBreakoutStrategy.cppm @@ -0,0 +1,59 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module makeOhlcBreakoutStrategy; + +import std; +import sweepCombination; // sweep::Combination + +export namespace sweep { + +// Map one swept parameter combination onto an OhlcBreakoutStrategy config. +// Every parameter is read with getInt and no has() fallback: the sweep +// registers all seven names (buildOhlcBreakoutStrategySweep), so a missing one +// is a bug that should throw at load time, matching the strategy ctor's +// fail-fast validation on the run side. +tradingDefinitions::StrategyConfig makeOhlcBreakoutStrategy( + const sweep::Combination& combo) { + using namespace tradingDefinitions; + + return StrategyConfig{ + .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), + .TRADING_VARIABLES = TradingVariables{ + // Must match the dispatch string in run/operations.cppm. + .STRATEGY = "OhlcBreakoutStrategy", + .STOP_DISTANCE_IN_PIPS = combo.getInt("STOP_DISTANCE_IN_PIPS"), + .LIMIT_DISTANCE_IN_PIPS = combo.getInt("LIMIT_DISTANCE_IN_PIPS"), + .TRADING_SIZE = 1, + }, + // Positional contract with OhlcBreakoutStrategy: [0] is the breakout + // timeframe, [1] the trend timeframe. + .OHLC_VARIABLES = { + OHLCVariables{ + .OHLC_COUNT = combo.getInt("BREAKOUT_OHLC_COUNT"), + .OHLC_MINUTES = combo.getInt("BREAKOUT_OHLC_MINUTES"), + }, + OHLCVariables{ + .OHLC_COUNT = combo.getInt("TREND_OHLC_COUNT"), + .OHLC_MINUTES = combo.getInt("TREND_OHLC_MINUTES"), + }, + }, + .STRATEGY_VARIABLES = StrategyVariables{ + .OHLC_BREAKOUT_VARIABLES = OHLCBreakoutVariables{ + .BUFFER_PIPS = combo.getInt("BUFFER_PIPS"), + }, + }, + }; +} + +} // namespace sweep diff --git a/source/load/config/ohlcBreakoutStrategy/ohlcBreakoutStrategySweep.cppm b/source/load/config/ohlcBreakoutStrategy/ohlcBreakoutStrategySweep.cppm new file mode 100644 index 0000000..55751e3 --- /dev/null +++ b/source/load/config/ohlcBreakoutStrategy/ohlcBreakoutStrategySweep.cppm @@ -0,0 +1,47 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module ohlcBreakoutStrategySweep; + +export import parameterGenerator; // buildOhlcBreakoutStrategySweep() returns sweep::ParameterGenerator + +import std; // replaces , + +export namespace sweep { + +// The symbol groups THIS sweep runs against (see kSymbolGroupsOverride in +// randomStrategySweep for the full semantics: empty = default set, entries +// are validated against symbol_scale::kTable at compile time). Named +// per-strategy because the sweep modules are imported side by side +// (loadCommand, tests) and two exported sweep::kSymbolGroupsOverride would +// collide. +inline constexpr std::array kOhlcBreakoutSymbolGroupsOverride = + std::to_array({"EURUSD"}); + +// Declares which parameters to sweep for the OhlcBreakoutStrategy — all seven. +// Seven dimensions multiply fast: this set expands to 40^4 x 6 x 12 x 12 +// ≈ 2.2 BILLION combinations per run (loadCommand shows the exact count and +// asks for confirmation before queueing). Tune the values here; +// makeOhlcBreakoutStrategy reads back every name registered below, and the +// strategy ctor requires OHLC_COUNT >= 2 and OHLC_MINUTES >= 1. +ParameterGenerator buildOhlcBreakoutStrategySweep() { + ParameterGenerator generator; + generator.setSymbolGroups(); + // Breakout timeframe: the closed-candle range price must clear. + generator.addRange("BREAKOUT_OHLC_MINUTES", 20, 20, 120); + generator.addRange("BREAKOUT_OHLC_COUNT", 20, 20, 120); + // Trend timeframe: EMA over its closes (period = count / 2) is the filter. + generator.addRange("TREND_OHLC_MINUTES", 20, 20, 120); + generator.addRange("TREND_OHLC_COUNT", 20, 20, 120); + // Padding on the breakout levels, in pips (0 = raw range). + generator.addRange("BUFFER_PIPS", 0, 2, 10); + // Exits are central (Operations enforces SL/TP), so the distances sweep too. + generator.addRange("STOP_DISTANCE_IN_PIPS", 10, 20, 110); + generator.addRange("LIMIT_DISTANCE_IN_PIPS", 10, 20, 110); + return generator; +} + +} // namespace sweep diff --git a/source/load/makeStrategy.cppm b/source/load/config/randomStrategy/makeStrategy.cppm similarity index 83% rename from source/load/makeStrategy.cppm rename to source/load/config/randomStrategy/makeStrategy.cppm index 3be1330..73dc64e 100644 --- a/source/load/makeStrategy.cppm +++ b/source/load/config/randomStrategy/makeStrategy.cppm @@ -9,22 +9,22 @@ module; #include #include -#include "shared/utilities/parameterSweep.hpp" // sweep::Combination -#include "shared/tradingDefinitions/strategy.hpp" +#include "shared/tradingDefinitions/strategyConfig.hpp" export module makeStrategy; import std; +import sweepCombination; // sweep::Combination export namespace sweep { -// Map one swept parameter combination onto a concrete Strategy. Each +// Map one swept parameter combination onto a concrete StrategyConfig. Each // combination gets its own UUID so a single backtest result is uniquely // identifiable and traceable back to its inputs. -tradingDefinitions::Strategy makeStrategy(const sweep::Combination& combo) { +tradingDefinitions::StrategyConfig makeStrategy(const sweep::Combination& combo) { using namespace tradingDefinitions; - return Strategy{ + return StrategyConfig{ .UUID = boost::uuids::to_string(boost::uuids::random_generator()()), .TRADING_VARIABLES = TradingVariables{ .STRATEGY = "RandomStrategy", diff --git a/source/load/sweep/randomStrategySweep.cppm b/source/load/config/randomStrategy/randomStrategySweep.cppm similarity index 51% rename from source/load/sweep/randomStrategySweep.cppm rename to source/load/config/randomStrategy/randomStrategySweep.cppm index adc305f..4c982c9 100644 --- a/source/load/sweep/randomStrategySweep.cppm +++ b/source/load/config/randomStrategy/randomStrategySweep.cppm @@ -4,23 +4,31 @@ // This code is licensed under MIT license (see LICENSE.txt for details) // --------------------------------------- -module; - -#include -#include -#include +export module randomStrategySweep; -#include "shared/utilities/parameterSweep.hpp" +export import parameterGenerator; // buildRandomStrategySweep() returns sweep::ParameterGenerator -export module randomStrategySweep; +import std; // replaces , export namespace sweep { +// The symbol groups THIS sweep runs against. Leave empty to run the full +// default set (sweep::kSymbolGroups in runConfigurationBuilder); list groups +// here to narrow the sweep, e.g. +// std::to_array({"EURUSD", "GBPUSD"}) // two runs +// std::to_array({"EURUSD,GBPUSD"}) // one run, both instruments +// setSymbolGroups validates every symbol against symbol_scale::kTable at +// compile time, so a typo here fails the build instead of queueing a run the +// worker rejects at runtime. +inline constexpr std::array kSymbolGroupsOverride = + std::to_array({"EURUSD"}); + // Declares which parameters to sweep for the RandomStrategy. Keeping the ranges // in one place means a new strategy (or extra swept parameter) is a localised // edit: register it here, then read it back in makeStrategy(). ParameterGenerator buildRandomStrategySweep() { ParameterGenerator generator; + generator.setSymbolGroups(); // generator.addRange("OHLC_COUNT", 80, 20, 140); // 80, 100, 120, 140 // generator.addList("OHLC_MINUTES", {1, 3, 5, 8}); generator.addRange("LIMIT_DISTANCE_IN_PIPS", 1, 1, 100); @@ -28,16 +36,4 @@ ParameterGenerator buildRandomStrategySweep() { return generator; } -// Resolves a generator by name (as passed on the `load` command line). Throws -// std::invalid_argument for an unknown name so the caller can report it. A new -// generator is one extra branch here plus an entry in `valid`. -ParameterGenerator buildSweep(std::string_view name) { - constexpr std::string_view valid = "random"; - if (name == "random") { - return buildRandomStrategySweep(); - } - throw std::invalid_argument("unknown sweep generator '" + std::string(name) - + "' (valid: " + std::string(valid) + ")"); -} - } // namespace sweep diff --git a/source/load/config/runConfigurationBuilder.cppm b/source/load/config/runConfigurationBuilder.cppm new file mode 100644 index 0000000..e5ab825 --- /dev/null +++ b/source/load/config/runConfigurationBuilder.cppm @@ -0,0 +1,88 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +module; + +#include +#include + +#include "shared/tradingDefinitions/config/runConfiguration.hpp" + +export module runConfigurationBuilder; + +import std; // replaces , , +import symbolGroups; // sweep::allSymbolsKnown — validates kSymbolGroups (see static_assert) + +export namespace sweep { + +// The DEFAULT symbols a sweep targets, used whenever the sweep itself doesn't +// override them (see resolveSymbolGroups below and e.g. kSymbolGroupsOverride +// in randomStrategySweep). Each entry becomes its OWN run (its own RUN_ID, +// strategy list and run descriptor). An entry may itself be a comma-separated +// list to evaluate multiple instruments inside a single run, e.g. +// "EURUSD,AUDUSD". So {"EURUSD,AUDUSD"} is one run over two instruments, +// while {"EURUSD", "AUDUSD"} is two independent runs. Edit and rebuild to change. +// +// constexpr std::array of string_views (not a runtime std::vector) +// so the static_assert below can validate it against symbol_scale::kTable at +// compile time. std::to_array deduces the size from the list, so adding/removing +// an entry needs no hand-maintained count. +inline constexpr std::array kSymbolGroups = std::to_array({ + "AUDUSD", "EURUSD", "GBRIDXGBP", "GBPUSD", "NZDUSD", + "USDJPY", "GBPJPY", "EURJPY", "USDCAD", "FRAIDXEUR", + "EURGBP", "USA500IDXUSD", "AUSIDXAUD", "USDCHF", "XAUUSD", + "XAGUSD", "USATECHIDXUSD", "EURCHF", "DEUIDXEUR", "USA30IDXUSD", + "LIGHTCMDUSD", "JPNIDXJPY", "BRENTCMDUSD", "AUDNZD", "EURAUD", + "HKGIDXHKD", "COPPERCMDUSD", "USDSEK", "EURNOK"}); + +// Compile-time guard: every symbol named in kSymbolGroups must exist in +// symbol_scale::kTable. The run side validates symbols against that same table +// (SqlManager::loadPriceData throws on an unknown one), so a typo here would +// still queue the run, only for the worker to reject it at runtime. Catch it +// here, at build time, before a wasted run reaches the queue. +static_assert(allSymbolsKnown(kSymbolGroups), + "sweep::kSymbolGroups names a symbol missing from symbol_scale::kTable — " + "add it to the table (symbolScale.cppm) or fix the typo"); + +// Resolve which symbol groups a sweep runs against: the sweep's own override +// when it set one (ParameterGenerator::setSymbolGroups), otherwise the full +// default kSymbolGroups. Returns owned strings so both sources come back as +// one type. +std::vector resolveSymbolGroups(const std::vector& sweepGroups) { + if (!sweepGroups.empty()) { + return sweepGroups; + } + return kSymbolGroups + | std::views::transform([](std::string_view group) { return std::string(group); }) + | std::ranges::to(); +} + +// The run-level descriptor: what tick data to pull from QuestDB, plus the risk +// limits every strategy in the run runs under. Shared by every strategy in this +// run and linked to them by RUN_ID. SYMBOLS is one cleaned, comma-separated +// group from kSymbolGroups (see cleanSymbols in load/utility/symbolGroups.cppm). +tradingDefinitions::RunConfiguration makeRunConfiguration(const std::string& runId, + const std::string& symbols) { + using namespace boost::decimal::literals; + return tradingDefinitions::RunConfiguration{ + .RUN_ID = runId, + .SYMBOLS = symbols, + .LAST_MONTHS = 6, + .STARTING_BALANCE = tradingDefinitions::DEFAULT_STARTING_BALANCE, + // Cut a run off once it has lost 5% of the account (fail fast); + // set <= 0 to run without any loss cutoff. + .MAX_LOSS_PERCENT = 5_DD, + // Cap on simultaneously open positions per run (<= 0 = unlimited). + // The per-symbol gate already limits to one trade per symbol, so this + // only bites on multi-symbol runs. + .MAX_OPEN_TRADES = 1, + // Flip to false to silence liquidated runs from Elasticsearch once + // sweeps scale up and loss-limit cutoffs are expected noise. + .REPORT_FAILURES = true, + }; +} + +} // namespace sweep diff --git a/source/load/loadCommand.cppm b/source/load/loadCommand.cppm index 3574c14..e71bd6b 100644 --- a/source/load/loadCommand.cppm +++ b/source/load/loadCommand.cppm @@ -4,7 +4,7 @@ // This code is licensed under MIT license (see LICENSE.txt for details) // --------------------------------------- -module; +module; #include @@ -12,73 +12,213 @@ module; #include #include "shared/utilities/env.hpp" -#include "shared/utilities/parameterSweep.hpp" #include "shared/utilities/queueKeys.hpp" -#include "shared/redis/producer/redisLoader.hpp" +#include "load/redisLoader.hpp" #include "shared/tradingDefinitions/config/runConfiguration.hpp" // RunConfiguration -> json -#include "shared/tradingDefinitions/strategy.hpp" // Strategy -> json (makeStrategy result) +#include "shared/tradingDefinitions/strategyConfig.hpp" // StrategyConfig -> json (makeStrategy result) export module loadCommand; import std; // replaces , , , +import parameterGenerator; // sweep::ParameterGenerator, sweep::Combination import randomStrategySweep; // buildRandomStrategySweep -import runConfigurationBuilder; // makeRunConfiguration +import ohlcBreakoutStrategySweep; // buildOhlcBreakoutStrategySweep +import runConfigurationBuilder; // makeRunConfiguration, resolveSymbolGroups import makeStrategy; // sweep::makeStrategy +import makeOhlcBreakoutStrategy; // sweep::makeOhlcBreakoutStrategy +import symbolGroups; // sweep::cleanSymbols + +export namespace sweep { + +// Maps a swept combination onto a concrete strategy's config (minting a fresh +// UUID) — implemented per sweep by makeStrategy / makeOhlcBreakoutStrategy. +using StrategyFactory = + tradingDefinitions::StrategyConfig (*)(const Combination&); + +// Builds the keyed payloads for grid indices [begin, end): each combination is +// decoded lazily (combinationAt), mapped through the factory, and keyed under +// queue_keys::strategyPayloadKey(runId, ). Exported so +// tests can pin the chunk/key/payload contract without Redis. +std::vector buildStrategyChunk( + const ParameterGenerator& generator, + StrategyFactory strategyFactory, + const std::string& runId, + std::size_t begin, + std::size_t end); + +} // namespace sweep export class LoadCommand { public: static int run(int argc, const char* argv[]); }; -int LoadCommand::run(const int argc, const char* argv[]) { +namespace { + +// Groups digits with commas (1,000,000 — UK/US convention) so large grid +// sizes are readable at a glance. Hand-rolled rather than {:L} because the +// global locale defaults to "C" (no grouping) and named locales like +// en_GB.UTF-8 are not guaranteed to exist on every host. +std::string withThousands(const std::size_t n) { + std::string s = std::to_string(n); + for (std::size_t pos = s.size(); pos > 3;) { + pos -= 3; + s.insert(pos, ","); + } + return s; +} - // One RUN_ID identifies the whole sweep; each combination becomes its own - // queue entry, distinguished by its parameter values. - const auto runId = boost::uuids::to_string(boost::uuids::random_generator()()); +// Payloads per pipelined Redis request: bounds producer memory (the grid is +// never materialised) and keeps each request comfortably sized. +constexpr std::size_t kChunkSize = 1000; - // Select the sweep generator from the command line, e.g. `load random`. - // Defaults to "random" — the only generator today — when omitted. An - // unknown name is a usage error, not a crash, so report it and bail. - const std::string_view sweepName = argc > 2 ? argv[2] : "random"; - sweep::ParameterGenerator generator; - try { - generator = sweep::buildSweep(sweepName); - } catch (const std::exception& ex) { - std::println(stderr, "LoadCommand: {}", ex.what()); - return 1; +// Progress heartbeat for large grids, in strategies. A multiple of kChunkSize +// so the boundary always lands on a chunk edge. +constexpr std::size_t kProgressEvery = 100'000; + +// Safety-net expiry on payload keys so a crashed or abandoned run cannot leak +// them forever. The normal cleanup is the worker's GETDEL. +constexpr long kPayloadTtlSeconds = 7L * 24 * 60 * 60; + +// Feeds RedisLoader::loadKeyedPayloadStream one lazily-built chunk at a time, +// walking the grid indices [0, total) exactly once. +class SweepChunkSource final : public RedisLoader::ChunkSource { +public: + SweepChunkSource(const sweep::ParameterGenerator& generator, + const sweep::StrategyFactory strategyFactory, + std::string runId, + const std::size_t total) + : generator_(generator), + strategyFactory_(strategyFactory), + runId_(std::move(runId)), + total_(total) {} + + std::vector next() override { + const std::size_t begin = next_; + const std::size_t end = std::min(begin + kChunkSize, total_); + next_ = end; + if (begin != 0 && begin % kProgressEvery == 0) { + std::println("LoadCommand: queued {}/{} strategies...", + withThousands(begin), withThousands(total_)); + } + return sweep::buildStrategyChunk(generator_, strategyFactory_, runId_, + begin, end); } - const auto combinations = generator.generateAllCombinations(); +private: + const sweep::ParameterGenerator& generator_; + sweep::StrategyFactory strategyFactory_; + std::string runId_; + std::size_t total_; + std::size_t next_ = 0; +}; - std::println("LoadCommand: sweeping {} parameter combination(s) for RUN_ID={}", combinations.size(), runId); +} // namespace + +std::vector sweep::buildStrategyChunk( + const ParameterGenerator& generator, + const StrategyFactory strategyFactory, + const std::string& runId, + const std::size_t begin, + const std::size_t end) { + std::vector chunk; + chunk.reserve(end - begin); + for (std::size_t i = begin; i < end; ++i) { + const tradingDefinitions::StrategyConfig config = + strategyFactory(generator.combinationAt(i)); + // Pin to JSON explicitly to trigger the implicit StrategyConfig + // conversion for .dump(). + const nlohmann::json j = config; + chunk.push_back({queue_keys::strategyPayloadKey(runId, config.UUID), + j.dump()}); + } + return chunk; +} - // Serialise every swept strategy first. combinations is a sized range, so - // std::ranges::to reserves up front (no manual reserve needed). The json type - // is pinned explicitly because makeStrategy returns a Strategy and relies on - // the implicit conversion for .dump(). - const auto strategyPayloads = combinations - | std::views::transform([](const sweep::Combination& combo) { - // Pin to JSON explicitly to trigger implicit Strategy conversion - const nlohmann::json j = sweep::makeStrategy(combo); - return j.dump(); - }) - | std::ranges::to(); +int LoadCommand::run(const int argc, const char* argv[]) { - // Push all strategies BEFORE the run descriptor. A worker that sees the run - // immediately drains the strategy list and retires the run when empty, so the - // full set must already be present the moment the run becomes visible. - const auto strategyKey = queue_keys::strategyKey(runId); + // Select the sweep from the command line, e.g. `load random`. Defaults to + // "random" when omitted. An unknown name is a usage error, not a crash, so + // report it and bail. Each sweep pairs its parameter grid with the factory + // that maps a combination onto that strategy's config, so both are chosen + // together — a new sweep is one extra branch here plus a mention in the + // error. + const std::string_view sweepName = argc > 2 ? argv[2] : "ohlcBreakout"; + sweep::ParameterGenerator generator; + sweep::StrategyFactory strategyFactory = nullptr; + if (sweepName == "random") { + generator = sweep::buildRandomStrategySweep(); + strategyFactory = sweep::makeStrategy; + } else if (sweepName == "ohlcBreakout") { + generator = sweep::buildOhlcBreakoutStrategySweep(); + strategyFactory = sweep::makeOhlcBreakoutStrategy; + } else { + std::println(stderr, + "LoadCommand: unknown sweep generator '{}' (valid: random, ohlcBreakout)", + sweepName); + return 1; + } + + // Surface the full sweep size (the grid fanned out across every run) and + // wait for confirmation BEFORE anything touches Redis. The count is just + // the product of the range sizes, so an over-eager grid is caught while + // backing out is still free — Redis storage is O(count), and once a run is + // advertised workers start draining it immediately. EOF (closed stdin) + // counts as a decline so a non-interactive invocation can't sail past the + // prompt. + const auto symbolGroups = sweep::resolveSymbolGroups(generator.symbolGroups()); + const auto combinationCount = generator.combinationCount(); + std::println("LoadCommand: '{}' sweep: {} combination(s) x {} symbol group(s) = {} backtest(s)", + sweepName, withThousands(combinationCount), symbolGroups.size(), + withThousands(combinationCount * symbolGroups.size())); + std::print("Press Enter to queue them (Ctrl+C to abort)... "); + std::fflush(stdout); + if (std::string ack; !std::getline(std::cin, ack)) { + std::println(stderr, "LoadCommand: aborted, no confirmation on stdin"); + return 1; + } const auto redisHost = env::getOr("REDIS_HOST", "127.0.0.1"); - if (const auto strategyStatus = RedisLoader::loadPayloadBatch(redisHost, 6379, strategyKey, strategyPayloads); - strategyStatus != 0) - { - return strategyStatus; + + // Fan out: every resolved symbol group becomes its own run — the sweep's + // own list when it set one, the full kSymbolGroups default otherwise. A + // comma-separated entry like "EURUSD,AUDUSD" is one run over multiple + // instruments; separate entries are independent runs (each with its own + // RUN_ID, strategy list and run descriptor on the shared RUN queue). + for (const auto& group : symbolGroups) { + const std::string symbols = sweep::cleanSymbols(group); + + // One RUN_ID per run; each combination becomes its own payload key, + // distinguished by its parameter values and freshly-minted UUID. + const auto runId = boost::uuids::to_string(boost::uuids::random_generator()()); + + std::println("LoadCommand: sweeping {} parameter combination(s) for RUN_ID={} symbols={}", + withThousands(combinationCount), runId, symbols); + + // Stream every strategy into Redis BEFORE the run descriptor: payload + // keys first, then their names onto the run's list (RedisLoader keeps + // that order per chunk). A worker that sees the run immediately drains + // the name list and retires the run when empty, so the full set must + // already be present the moment the run becomes visible. + SweepChunkSource source(generator, strategyFactory, runId, combinationCount); + if (const auto strategyStatus = RedisLoader::loadKeyedPayloadStream( + redisHost, 6379, queue_keys::strategyKey(runId), source, + kPayloadTtlSeconds); + strategyStatus != 0) + { + return strategyStatus; + } + + // Now advertise the run so workers can pick it up. runJson is pinned to + // nlohmann::JSON (not auto) because makeRunConfiguration returns a + // RunConfiguration and relies on the implicit conversion for .dump(). + const nlohmann::json runJson = sweep::makeRunConfiguration(runId, symbols); + if (const auto runStatus = RedisLoader::loadPayload(redisHost, 6379, queue_keys::RUN, runJson.dump()); + runStatus != 0) + { + return runStatus; + } } - // Now advertise the run so workers can pick it up. runJson is pinned to - // nlohmann::JSON (not auto) because makeRunConfiguration returns a - // RunConfiguration and relies on the implicit conversion for .dump(). - const nlohmann::json runJson = sweep::makeRunConfiguration(runId); - return RedisLoader::loadPayload(redisHost, 6379, queue_keys::RUN, runJson.dump()); + return 0; } diff --git a/source/load/redisLoader.cpp b/source/load/redisLoader.cpp new file mode 100644 index 0000000..f228875 --- /dev/null +++ b/source/load/redisLoader.cpp @@ -0,0 +1,185 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "load/redisLoader.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "shared/utilities/base64.hpp" +#include "shared/redis/connection/redisConnection.hpp" +#include "shared/redis/operations/redisOperations.hpp" + +namespace asio = boost::asio; +namespace redis = boost::redis; + +namespace { + +// A whitespace-only (or empty) payload is rejected before it reaches Redis. +bool isBlank(const std::string& rawJson) { + return std::ranges::all_of(rawJson, [](unsigned char c) { + return std::isspace(c); + }); +} + +asio::awaitable pushOnce( + std::shared_ptr conn, + std::string queueKey, + std::string encoded) { + RedisOperations ops(conn); + co_await ops.listPushFront(std::move(queueKey), std::move(encoded)); + + // Ephemeral connection: cancel so io_context::run() returns after the push. + conn->cancel(); + co_return; +} + +// Drains `source` chunk by chunk over the one borrowed connection. Per chunk: +// pipelined SET (PX ttl) of every payload under its own key, THEN one +// pipelined LPUSH of the key names — awaited in that order so a consumer can +// never pop a name whose payload is not yet stored. Chunk production happens +// between awaits, so memory stays bounded at one chunk. Returns the total +// number of payloads stored. +asio::awaitable pushKeyedStream( + std::shared_ptr conn, + std::string listKey, + RedisLoader::ChunkSource& source, + std::chrono::milliseconds ttl) { + RedisOperations ops(conn); + std::size_t total = 0; + try { + for (;;) { + std::vector chunk = source.next(); + if (chunk.empty()) { + break; + } + + std::vector> keyedValues; + std::vector keyNames; + keyedValues.reserve(chunk.size()); + keyNames.reserve(chunk.size()); + for (RedisLoader::KeyedPayload& payload : chunk) { + if (payload.key.empty() || isBlank(payload.rawJson)) { + throw std::invalid_argument( + "RedisLoader: empty payload or key rejected"); + } + keyNames.push_back(payload.key); + keyedValues.emplace_back(std::move(payload.key), + Base64::b64encode(payload.rawJson)); + } + + co_await ops.setMultipleWithTTL(std::move(keyedValues), ttl); + co_await ops.listPushFront(listKey, std::move(keyNames)); + total += chunk.size(); + } + } catch (...) { + // Cancel on the failure path too: the connection's detached async_run + // otherwise keeps io_context::run() from ever returning. + conn->cancel(); + throw; + } + + // Ephemeral connection: cancel so io_context::run() returns after the push. + conn->cancel(); + co_return total; +} + +} // namespace + +int RedisLoader::loadPayload(const std::string& redisHost, + int redisPort, + const std::string& queueKey, + const std::string& rawJson) { + if (isBlank(rawJson)) { + std::cerr << "RedisLoader: empty payload rejected" << std::endl; + return 1; + } + + const std::string encoded = Base64::b64encode(rawJson); + + asio::io_context ioc; + auto conn = redis_util::makeRedisConnection(ioc, redisHost, redisPort); + + std::exception_ptr pushError; + + asio::co_spawn( + ioc, + pushOnce(conn, queueKey, encoded), + [&pushError](std::exception_ptr e) { + if (e) { + pushError = e; + } + }); + + ioc.run(); + + if (pushError) { + try { + std::rethrow_exception(pushError); + } catch (const boost::system::system_error& ex) { + std::cerr << "Redis LPUSH failed: " << ex.what() << std::endl; + } + return 3; + } + + return 0; +} + +int RedisLoader::loadKeyedPayloadStream(const std::string& redisHost, + const int redisPort, + const std::string& listKey, + ChunkSource& source, + const long ttlSeconds) { + asio::io_context ioc; + auto conn = redis_util::makeRedisConnection(ioc, redisHost, redisPort); + + std::exception_ptr pushError; + std::size_t stored = 0; + + asio::co_spawn( + ioc, + pushKeyedStream(conn, listKey, source, std::chrono::seconds(ttlSeconds)), + [&pushError, &stored](const std::exception_ptr& e, std::size_t total) { + if (e) { + pushError = e; + } else { + stored = total; + } + }); + + ioc.run(); + + if (pushError) { + try { + std::rethrow_exception(pushError); + } catch (const boost::system::system_error& ex) { + std::cerr << "Redis payload stream failed: " << ex.what() << std::endl; + return 3; + } catch (const std::exception& ex) { + std::cerr << ex.what() << std::endl; + return 1; + } + } + + std::cout << "RedisLoader: stored " << stored + << " keyed payload(s), key names on " << listKey << std::endl; + + return 0; +} diff --git a/source/load/redisLoader.hpp b/source/load/redisLoader.hpp new file mode 100644 index 0000000..fc643c7 --- /dev/null +++ b/source/load/redisLoader.hpp @@ -0,0 +1,51 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include + +// LPUSH pairs with RedisRunner's RPOP so consumers observe FIFO ordering. +class RedisLoader { +public: + // One strategy destined for its own Redis string key: `key` is the full + // payload key name (see queue_keys::strategyPayloadKey) and `rawJson` the + // strategy JSON, Base64-encoded before storage. + struct KeyedPayload { + std::string key; + std::string rawJson; + }; + + // Pull interface for loadKeyedPayloadStream: next() returns the next chunk + // of payloads to store, or an empty vector when the sweep is exhausted. + // Called on the loader's coroutine between Redis writes, so producing a + // chunk lazily bounds memory at one chunk regardless of grid size. + class ChunkSource { + public: + virtual ~ChunkSource() = default; + virtual std::vector next() = 0; + }; + + // LPUSHes a single Base64-encoded payload onto queueKey without assuming a + // payload type (run descriptor or strategy). + static int loadPayload(const std::string& redisHost, + int redisPort, + const std::string& queueKey, + const std::string& rawJson); + + // Streams every chunk from `source` over ONE connection. Per chunk, one + // pipelined SET (with ttlSeconds expiry) per payload key, then the key + // NAMES are LPUSHed onto listKey — in that order, so a consumer can never + // pop a name whose payload is not yet stored. The TTL is a safety net for + // crashed/abandoned runs; the consumer's GETDEL is the normal cleanup. + static int loadKeyedPayloadStream(const std::string& redisHost, + int redisPort, + const std::string& listKey, + ChunkSource& source, + long ttlSeconds); +}; diff --git a/source/load/sweep/parameterGenerator.cppm b/source/load/sweep/parameterGenerator.cppm new file mode 100644 index 0000000..7ffe315 --- /dev/null +++ b/source/load/sweep/parameterGenerator.cppm @@ -0,0 +1,153 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +// Parameter sweep generator: declare named numeric ranges and expand them into +// every combination (the cartesian product), so a strategy can be backtested +// across a grid of parameters in a single `load`. +// +// This is the C++ analogue of the C# StrategyParameterGenerator. C++ has no +// reflection, so a combination is returned keyed by parameter name +// (sweep::Combination) and the caller maps those values onto the strongly-typed +// StrategyConfig fields (see sweep::makeStrategy in +// source/load/config/randomStrategy/makeStrategy.cppm). +export module parameterGenerator; + +export import parameterRange; // addRange() takes unique_ptr +export import sweepCombination; // generateAllCombinations() returns Combination + +import std; +import symbolGroups; // allSymbolsKnown — setSymbolGroups' compile-time guard + +export namespace sweep { + +// Holds the registered ranges and expands them into every combination. +class ParameterGenerator { +public: + // Register an arbitrary range under `name`. The convenience overloads below + // mirror the C# Add* helpers and cover the common cases. + ParameterGenerator& addRange(std::string name, + std::unique_ptr range) { + ranges_.emplace_back(std::move(name), std::move(range)); + return *this; + } + + ParameterGenerator& addRange(std::string name, double start, double step, + double end) { + return addRange(std::move(name), + std::make_unique(start, step, end)); + } + + ParameterGenerator& addList(std::string name, std::vector values) { + return addRange(std::move(name), + std::make_unique(std::move(values))); + } + + ParameterGenerator& addExponential(std::string name, double base, + double multiplier, int iterations) { + return addRange( + std::move(name), + std::make_unique(base, multiplier, iterations)); + } + + // The symbol groups this sweep targets, set by the strategy sweep builder + // (e.g. randomStrategySweep's kSymbolGroupsOverride). Empty means "no + // override": the load command then falls back to the full default set — + // see sweep::resolveSymbolGroups in runConfigurationBuilder. The groups + // come in as a non-type template parameter (a reference to the caller's + // constexpr array) so the guard below runs at compile time; they are + // stored as owned strings so the generator doesn't hold views into caller + // storage. + template + ParameterGenerator& setSymbolGroups() { + // Same compile-time guard as kSymbolGroups (runConfigurationBuilder): + // a symbol missing from symbol_scale::kTable would queue a run the + // worker then rejects at runtime (SqlManager::loadPriceData throws). + // Checking here means every sweep that sets an override gets the + // guard without repeating it. + static_assert(allSymbolsKnown(Groups), + "setSymbolGroups: a group names a symbol missing from " + "symbol_scale::kTable — add it to the table " + "(symbolScale.cppm) or fix the typo"); + symbolGroups_.assign(Groups.begin(), Groups.end()); + return *this; + } + + const std::vector& symbolGroups() const { return symbolGroups_; } + + // Expand every registered range into the full cartesian product. Ranges are + // expanded in registration order, so the last-registered parameter varies + // fastest, keeping the output order stable and predictable. + std::vector generateAllCombinations() const { + std::vector result(1); // seed with one empty combination + for (const auto& [name, range] : ranges_) { + const std::vector values = range->generateValues(); + std::vector expanded; + expanded.reserve(result.size() * values.size()); + for (const Combination& base : result) { + for (const double value : values) { + Combination next = base; + next.set(name, value); + expanded.push_back(std::move(next)); + } + } + result = std::move(expanded); + } + return result; + } + + std::size_t parameterCount() const { return ranges_.size(); } + + // How many combinations the grid expands to — the product of every range's + // value count — without materialising them. generateAllCombinations() is + // O(product) in time and memory, so callers use this to surface the sweep + // size (and get confirmation) while backing out is still free. + std::size_t combinationCount() const { + std::size_t count = 1; + for (const auto& [name, range] : ranges_) { + count *= range->generateValues().size(); + } + return count; + } + + // Lazily decode combination `index` of the cartesian product — the + // counterpart of generateAllCombinations()[index] for grids too large to + // expand. The product is read as a mixed-radix number over the ranges, + // registration order most- to least-significant digit, so the + // last-registered parameter varies fastest — the same order as the eager + // expansion (pinned by the lazy-mirror test in tests/sweep.cpp). + // Precondition: index < combinationCount(). + Combination combinationAt(std::size_t index) const { + Combination combo; + for (const auto& [name, range] : ranges_ | std::views::reverse) { + const std::vector values = range->generateValues(); + combo.set(name, values[index % values.size()]); + index /= values.size(); + } + return combo; + } + + // Each range's name and value count, in registration order. Lazy consumers + // (the sweep tests) derive axis strides for combinationAt from these, so a + // parameter can be walked through all its values without materialising the + // grid. + std::vector> rangeValueCounts() const { + std::vector> counts; + counts.reserve(ranges_.size()); + for (const auto& [name, range] : ranges_) { + counts.emplace_back(name, range->generateValues().size()); + } + return counts; + } + +private: + // A vector (not a map) preserves registration order for reproducible output. + std::vector>> ranges_; + + // Symbol-group override (see setSymbolGroups); empty = use the default set. + std::vector symbolGroups_; +}; + +} // namespace sweep diff --git a/source/load/sweep/parameterRange.cppm b/source/load/sweep/parameterRange.cppm new file mode 100644 index 0000000..ec559a0 --- /dev/null +++ b/source/load/sweep/parameterRange.cppm @@ -0,0 +1,84 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module parameterRange; + +import std; + +export namespace sweep { + +// Abstract numeric range, the analogue of C#'s IParameterRange. A range knows +// how to enumerate the discrete values a single parameter should take. +class ParameterRange { +public: + virtual ~ParameterRange() = default; + [[nodiscard]] virtual std::vector generateValues() const = 0; +}; + +// Inclusive [start, end] stepped by `step`, e.g. {80, 20, 140} -> 80,100,120,140. +// The epsilon absorbs floating point drift so the upper bound stays inclusive. +class LinearRange final : public ParameterRange { +public: + LinearRange(const double start, const double step, const double end) + : start_(start), step_(step), end_(end) {} + + [[nodiscard]] std::vector generateValues() const override { + std::vector values; + if (step_ <= 0.0) { // defensive: a non-positive step cannot advance + values.push_back(start_); + return values; + } + for (double value = start_; value <= end_ + 1e-9; value += step_) { + values.push_back(value); + } + return values; + } + +private: + double start_; + double step_; + double end_; +}; + +// An explicit set of values, e.g. {1, 3, 5, 8}, analogue of C#'s +// ExplicitParameterRange / AddParameterList. +class ExplicitRange final : public ParameterRange { +public: + explicit ExplicitRange(std::vector values) + : values_(std::move(values)) {} + + [[nodiscard]] std::vector generateValues() const override { return values_; } + +private: + std::vector values_; +}; + +// base * multiplier^i for `iterations` values, e.g. {10, 2, 5} -> 10,20,40,80,160. +class ExponentialRange final : public ParameterRange { +public: + ExponentialRange(double base, double multiplier, int iterations) + : base_(base), multiplier_(multiplier), iterations_(iterations) {} + + [[nodiscard]] std::vector generateValues() const override { + std::vector values; + if (iterations_ > 0) { + values.reserve(static_cast(iterations_)); + } + double current = base_; + for (int i = 0; i < iterations_; ++i) { + values.push_back(current); + current *= multiplier_; + } + return values; + } + +private: + double base_; + double multiplier_; + int iterations_; +}; + +} // namespace sweep diff --git a/source/load/sweep/runConfigurationBuilder.cppm b/source/load/sweep/runConfigurationBuilder.cppm deleted file mode 100644 index 026928b..0000000 --- a/source/load/sweep/runConfigurationBuilder.cppm +++ /dev/null @@ -1,43 +0,0 @@ -// Backtesting Engine in C++ -// -// (c) 2026 Ryan McCaffery | https://mccaffers.com -// This code is licensed under MIT license (see LICENSE.txt for details) -// --------------------------------------- - -module; - -#include -#include - -#include "shared/tradingDefinitions/config/runConfiguration.hpp" - -export module runConfigurationBuilder; - -import std; // replaces - -export namespace sweep { - -// The run-level descriptor: what tick data to pull from QuestDB, plus the risk -// limits every strategy in the sweep runs under. Shared by every strategy in -// this sweep and linked to them by RUN_ID. -tradingDefinitions::RunConfiguration makeRunConfiguration(const std::string& runId) { - using namespace boost::decimal::literals; - return tradingDefinitions::RunConfiguration{ - .RUN_ID = runId, - .SYMBOLS = "EURUSD", - .LAST_MONTHS = 6, - .STARTING_BALANCE = tradingDefinitions::DEFAULT_STARTING_BALANCE, - // Cut a run off once it has lost 5% of the account (fail fast); - // set <= 0 to run without any loss cutoff. - .MAX_LOSS_PERCENT = 5_DD, - // Cap on simultaneously open positions per run (<= 0 = unlimited). - // The per-symbol gate already limits to one trade per symbol, so this - // only bites on multi-symbol runs. - .MAX_OPEN_TRADES = 1, - // Flip to false to silence liquidated runs from Elasticsearch once - // sweeps scale up and loss-limit cutoffs are expected noise. - .REPORT_FAILURES = true, - }; -} - -} // namespace sweep diff --git a/source/load/sweep/sweepCombination.cppm b/source/load/sweep/sweepCombination.cppm new file mode 100644 index 0000000..4ea31df --- /dev/null +++ b/source/load/sweep/sweepCombination.cppm @@ -0,0 +1,44 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module sweepCombination; + +import std; + +export namespace sweep { + +// One point in the parameter grid: parameter name -> chosen value. Values are +// stored as doubles; integer parameters round-trip exactly, so getInt() is safe. +class Combination { +public: + bool has(const std::string& name) const { return values_.count(name) > 0; } + + double get(const std::string& name) const { + const auto it = values_.find(name); + if (it == values_.end()) { + throw std::out_of_range( + "sweep::Combination: unknown parameter '" + name + "'"); + } + return it->second; + } + + int getInt(const std::string& name) const { + return static_cast(std::lround(get(name))); + } + + // Assign a parameter value; used by ParameterGenerator while expanding the + // grid (module attachment rules rule out a cross-module friend here). + void set(std::string name, double value) { + values_[std::move(name)] = value; + } + + const std::map& values() const { return values_; } + +private: + std::map values_; +}; + +} // namespace sweep diff --git a/source/load/utility/symbolGroups.cppm b/source/load/utility/symbolGroups.cppm new file mode 100644 index 0000000..ac833c2 --- /dev/null +++ b/source/load/utility/symbolGroups.cppm @@ -0,0 +1,91 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// symbolGroups — tokenising and validating symbol groups. +// +// A "symbol group" is one comma-separated list of instruments evaluated inside +// a single run, e.g. "EURUSD,AUDUSD" (see sweep::kSymbolGroups in +// runConfigurationBuilder). Humans write groups with stray whitespace and empty +// fields; the run side (loadTicks in backtestRunner) splits SYMBOLS on ',' +// WITHOUT trimming, so every group must be normalised before it reaches the RUN +// queue. One constexpr tokenizer (splitSymbols) backs both jobs: +// +// - cleanSymbols() : runtime normalisation for the RUN descriptor. +// - allSymbolsKnown() : compile-time static_assert guard that every symbol a +// sweep names exists in symbol_scale::kTable — the run +// side rejects unknown symbols at runtime +// (SqlManager::loadPriceData throws), so catch the typo +// at build time, before a doomed run is queued. + +export module symbolGroups; + +import std; // replaces , , , +import symbolScale; // symbol_scale::get / kUnknown — allSymbolsKnown lookup + +export namespace sweep { + +// Split one symbol group on ',' into its symbols: trim spaces/tabs around each +// field and drop empty ones ("EURUSD, AUDUSD," -> {"EURUSD", "AUDUSD"}). The +// returned views point into `group`, so they only live as long as its storage. +// constexpr so the static_assert guards below can run it at compile time +// (transient constexpr allocation — the vector never escapes the evaluation). +constexpr std::vector splitSymbols(std::string_view group) { + std::vector symbols; + for (std::size_t pos = 0; pos <= group.size();) { + const std::size_t comma = group.find(',', pos); + const std::size_t end = + comma == std::string_view::npos ? group.size() : comma; + std::string_view token = group.substr(pos, end - pos); + while (!token.empty() && (token.front() == ' ' || token.front() == '\t')) { + token.remove_prefix(1); + } + while (!token.empty() && (token.back() == ' ' || token.back() == '\t')) { + token.remove_suffix(1); + } + if (!token.empty()) { + symbols.push_back(token); + } + if (comma == std::string_view::npos) { + break; + } + pos = comma + 1; + } + return symbols; +} + +// Normalise one symbol group into a clean comma-separated string, ready for +// the RUN descriptor: "EURUSD, AUDUSD" must become "EURUSD,AUDUSD" or the run +// side's untrimmed lookup of " AUDUSD" fails. +std::string cleanSymbols(std::string_view group) { + std::string result; + for (const std::string_view symbol : splitSymbols(group)) { + if (!result.empty()) { + result += ','; + } + result += symbol; + } + return result; +} + +// Compile-time guard for a symbol-group list: every symbol in every group must +// exist in symbol_scale::kTable. Used by the static_asserts on +// sweep::kSymbolGroups (runConfigurationBuilder) and inside +// ParameterGenerator::setSymbolGroups (covering every per-sweep override) so a +// typo'd symbol fails the build instead of queueing a run the worker rejects +// at runtime. +[[nodiscard]] constexpr bool allSymbolsKnown( + std::span groups) { + for (const std::string_view group : groups) { + for (const std::string_view symbol : splitSymbols(group)) { + if (symbol_scale::get(symbol) == symbol_scale::kUnknown) { + return false; + } + } + } + return true; +} + +} // namespace sweep diff --git a/source/main.cpp b/source/main.cpp index 5e17d3c..664a797 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -6,7 +6,6 @@ import std; -// backtesting engine headers import loadCommand; import runCommand; import ingestCommand; @@ -19,7 +18,6 @@ int main(const int argc, const char* argv[]) { } const std::string_view subcommand = argv[1]; - if (subcommand == "load") return LoadCommand::run(argc, argv); if (subcommand == "run") return RunCommand::run(argc, argv); if (subcommand == "ingest") return IngestCommand::run(argc, argv); diff --git a/source/run/execution/backtestRunner.cppm b/source/run/execution/backtestRunner.cppm index 768d8f7..1e68fd8 100644 --- a/source/run/execution/backtestRunner.cppm +++ b/source/run/execution/backtestRunner.cppm @@ -37,8 +37,7 @@ export std::vector loadTicks(const std::string& questdbHost, } // Get all the tick data out of QuestDB for these symbols - std::vector ticks = - SqlManager::loadPriceData(db, symbols, lastMonths); + std::vector ticks = SqlManager::loadPriceData(db, symbols, lastMonths); std::printf("Total ticks streamed: %zu\n", ticks.size()); diff --git a/source/run/operations.cppm b/source/run/operations.cppm index 297464a..c3c5ce1 100644 --- a/source/run/operations.cppm +++ b/source/run/operations.cppm @@ -7,7 +7,7 @@ module; #include "shared/utilities/backtestLog.hpp" -#include "shared/reporting/elasticPublisher.hpp" +#include "run/reporting/elasticPublisher.hpp" #include "shared/tradingDefinitions/config/configuration.hpp" #include "run/reporting/tradingResults.hpp" @@ -22,6 +22,7 @@ import resultsSummary; // ResultsSummary import symbolScale; // symbol_scale::get import strategy; // IStrategy import randomStrategy; // RandomStrategy +import ohlcBreakoutStrategy; // OhlcBreakoutStrategy import strategyErrors; // UnknownStrategyError import elasticClient; // ElasticClient @@ -40,6 +41,9 @@ std::unique_ptr selectStrategy(const tradingDefinitions::Configuratio if (name == "RandomStrategy") { return std::make_unique(config.STRATEGY); } + if (name == "OhlcBreakoutStrategy") { + return std::make_unique(config.STRATEGY); + } throw UnknownStrategyError(name); } @@ -52,10 +56,6 @@ void Operations::run(const std::vector& ticks, // run. steady_clock is monotonic, the correct clock for elapsed durations. const auto runStart = std::chrono::steady_clock::now(); - std::println("Operations: new run starting RUN_ID={} strategy={}", - config.RUN_ID, - config.STRATEGY.TRADING_VARIABLES.STRATEGY); - TradeManager tradeManager; auto strategy = selectStrategy(config); @@ -106,10 +106,30 @@ void Operations::run(const std::vector& ticks, // but a throw from JSON serialisation or the network layer bypasses their // logging, so the catch handlers emit the single warning for that path. try { + // Every outcome doc carries the host that produced it, so a bad node in + // a distributed sweep can be traced from any of the three indices. + const std::string hostname = TradeFinal::localHostname(); + + // Compact terminal record for this run, emitted for every run regardless + // of outcome or the REPORT_FAILURES silencer below: the outcome flag (a + // completed run is success=1; a loss-limit cutoff is success=0), how long + // it took, and the host that produced it, alongside the run config. Sent + // first so the silenced-failure early return below cannot skip it. + const TradeFinal tradeFinal{ + config.RUN_ID, + TradingResults::nowIsoUtc(), + durationSeconds, + status == trading::RunStatus::LossLimitBreached ? 0 : 1, + hostname, + config, + }; + ElasticClient::putTradeFinal(tradeFinal); + if (status == trading::RunStatus::LossLimitBreached) { - // Silencer for large sweeps: liquidated runs are expected noise - // once the system is trusted, so the run config can opt out of - // reporting them. Completed runs always report. + // Silencer for large sweeps: liquidated runs are expected noise once + // the system is trusted, so the run config can opt out of the + // detailed failure doc here. The terminal record above and completed + // runs always report. if (!config.REPORT_FAILURES) { return; } @@ -128,6 +148,7 @@ void Operations::run(const std::vector& ticks, TradingResults::nowIsoUtc(), durationSeconds, reason.str(), + hostname, config, ResultsSummary::collect(tradeManager, config), }; @@ -137,6 +158,7 @@ void Operations::run(const std::vector& ticks, config.RUN_ID, TradingResults::nowIsoUtc(), durationSeconds, + hostname, config, ResultsSummary::collect(tradeManager, config), }; diff --git a/source/shared/redis/consumer/drainRuns.cpp b/source/run/queue/drainRuns.cpp similarity index 57% rename from source/shared/redis/consumer/drainRuns.cpp rename to source/run/queue/drainRuns.cpp index 48deeaf..0547bb8 100644 --- a/source/shared/redis/consumer/drainRuns.cpp +++ b/source/run/queue/drainRuns.cpp @@ -4,10 +4,12 @@ // This code is licensed under MIT license (see LICENSE.txt for details) // --------------------------------------- -#include "shared/redis/consumer/drainRuns.hpp" +#include "run/queue/drainRuns.hpp" #include +#include #include +#include #include #include #include @@ -21,11 +23,12 @@ #include #include +#include "shared/ipc/engineControl.hpp" #include "shared/utilities/jsonParser.hpp" #include "shared/utilities/queueKeys.hpp" #include "shared/utilities/threadPool.hpp" -#include "shared/reporting/elasticPublisher.hpp" -#include "shared/redis/consumer/runQueue.hpp" +#include "run/reporting/elasticPublisher.hpp" +#include "run/queue/runQueue.hpp" #include "run/execution/runnerBridge.hpp" namespace asio = boost::asio; @@ -39,14 +42,43 @@ asio::awaitable drainRuns(std::shared_ptr conn, try { - // Get CPU threads available, max of 6 + // Map the shared-memory control channel so a local monitor can watch the + // in-flight count and request a graceful stop. Best-effort: if the + // segment can't be created the engine still runs, just unmonitored. + std::optional control; + std::atomic* gauge = nullptr; + try { + control.emplace(); + gauge = &control->state().active_jobs; + } catch (const std::exception& ex) { + std::println(stderr, "RedisRunner: shm control unavailable: {}", ex.what()); + } + const auto stopRequested = [&] { + return control && + control->state().stop_signal.load(std::memory_order_acquire) != 0; + }; + bool stopping = false; + + // Size the pool to 80% of the available CPU threads so the box keeps + // headroom for Redis/IO and the host stays responsive. At least one + // worker always runs; there is no upper cap. hardware_concurrency() + // returns 0 when it can't tell, in which case we fall back to 1. const unsigned hw = std::thread::hardware_concurrency(); - ThreadPool pool(std::clamp(hw != 0 ? hw : 6u, 1u, 6u)); + const unsigned threads = hw != 0 ? std::max(1u, hw * 4 / 5) : 1u; + ThreadPool pool(threads, gauge); // Loop forever, claiming one run per iteration. An empty queue makes us - // wait and re-peek (below); only an exception blows the loop + // wait and re-peek (below); only an exception or a stop request blows the + // loop. bool waitingLogged = false; for (;;) { + // Stop requested between runs (or while idle on the empty-queue timer, + // which `continue`s back here): submit nothing further and fall + // through to the drain-and-pause below. + if (stopRequested()) { + stopping = true; + break; + } const std::optional descriptorB64 = co_await run_queue::peekRunTail(conn); if (!descriptorB64.has_value()) { // Queue empty: stay alive and poll until work reappears. The timer @@ -83,15 +115,37 @@ asio::awaitable drainRuns(std::shared_ptr conn, } quiesce{pool}; // Drain the run's strategy list, competing with any other workers. - // Redis stays on this coroutine thread; only the CPU-bound backtest - // is handed to the pool. submit() applies backpressure, so we keep - // popping at the rate the workers can absorb. + // The list carries payload KEY NAMES: RPOP hands each name to + // exactly one worker, then GETDEL consumes the payload stored + // under it (atomically, leaving nothing behind). Redis stays on + // this coroutine thread; only the CPU-bound backtest is handed to + // the pool. submit() applies backpressure, so we keep popping at + // the rate the workers can absorb. int strategiesRun = 0; for (;;) { + // Stop requested mid-run: stop submitting new strategies. The + // Quiesce guard / pool.wait() below still drains everything + // already in flight, and the gauge falls to 0 as it does. + if (stopRequested()) { + stopping = true; + break; + } + const std::optional payloadKey = + co_await run_queue::popStrategyKey(conn, strategyKey); + if (!payloadKey.has_value()) { + break; // strategy list drained + } const std::optional strategyB64 = - co_await run_queue::popStrategy(conn, strategyKey); + co_await run_queue::takeStrategyPayload(conn, *payloadKey); if (!strategyB64.has_value()) { - break; // strategy list drained + // Popped a name whose payload is gone — a peer that crashed + // between its RPOP and GETDEL can't cause this (the name + // went with it); this is the safety-net TTL having reaped a + // long-stale payload. Skip it; the run still drains. + std::println(stderr, + "RedisRunner: payload {} missing (expired?), skipping", + *payloadKey); + continue; } // Reassemble the Configuration the rest of the pipeline expects. @@ -120,6 +174,13 @@ asio::awaitable drainRuns(std::shared_ptr conn, std::rethrow_exception(err); } + // Stop requested: the in-flight work has now drained. Leave this run + // in Redis (we stopped mid-list) and break out to pause — don't load + // the next run. + if (stopping) { + break; + } + // Retire the run. A failure here propagates and aborts the loop, we // never re-peek the same run and reload its ticks in a tight loop. co_await run_queue::removeRun(conn, *descriptorB64); @@ -128,6 +189,19 @@ asio::awaitable drainRuns(std::shared_ptr conn, runCfg.RUN_ID, strategiesRun, strategiesRun == 1 ? "y" : "ies"); } + + // A stop signal drains the pool (above) and then parks the engine here: + // alive but idle, never accepting more work. This is a one-way pause, not + // a shutdown — the process stays up and the control segment stays mapped + // (active_jobs = 0, stop_signal = 1) so the monitor can confirm the drain. + if (stopping) { + std::println("RedisRunner: stop signal received — drained and paused."); + for (;;) { + asio::steady_timer timer(co_await asio::this_coro::executor); + timer.expires_after(std::chrono::seconds(1)); + co_await timer.async_wait(asio::use_awaitable); + } + } } catch (const std::exception& ex) { std::println(stderr, "RedisRunner aborted: {}", ex.what()); elastic::putEngineException( diff --git a/source/shared/redis/consumer/drainRuns.hpp b/source/run/queue/drainRuns.hpp similarity index 100% rename from source/shared/redis/consumer/drainRuns.hpp rename to source/run/queue/drainRuns.hpp diff --git a/source/shared/redis/consumer/redisRunner.cpp b/source/run/queue/redisRunner.cpp similarity index 93% rename from source/shared/redis/consumer/redisRunner.cpp rename to source/run/queue/redisRunner.cpp index 7256f04..01be893 100644 --- a/source/shared/redis/consumer/redisRunner.cpp +++ b/source/run/queue/redisRunner.cpp @@ -4,7 +4,7 @@ // This code is licensed under MIT license (see LICENSE.txt for details) // --------------------------------------- -#include "shared/redis/consumer/redisRunner.hpp" +#include "run/queue/redisRunner.hpp" #include #include @@ -16,9 +16,9 @@ #include #include "shared/utilities/backtestLog.hpp" -#include "shared/reporting/elasticPublisher.hpp" +#include "run/reporting/elasticPublisher.hpp" #include "shared/redis/connection/redisConnection.hpp" -#include "shared/redis/consumer/drainRuns.hpp" +#include "run/queue/drainRuns.hpp" namespace asio = boost::asio; diff --git a/source/shared/redis/consumer/redisRunner.hpp b/source/run/queue/redisRunner.hpp similarity index 100% rename from source/shared/redis/consumer/redisRunner.hpp rename to source/run/queue/redisRunner.hpp diff --git a/source/run/queue/runQueue.cpp b/source/run/queue/runQueue.cpp new file mode 100644 index 0000000..1966ed2 --- /dev/null +++ b/source/run/queue/runQueue.cpp @@ -0,0 +1,57 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "run/queue/runQueue.hpp" + +#include +#include +#include + +#include + +#include "shared/redis/operations/redisOperations.hpp" +#include "shared/utilities/queueKeys.hpp" + +namespace asio = boost::asio; +namespace redis = boost::redis; + +namespace run_queue { + +// All three helpers wrap a borrowed connection in a RedisOperations and delegate +// the Redis mechanics, keeping only the queue semantics (which key, FIFO peek, +// idempotent remove) here. The connection is NOT cancelled, it stays open for the +// rest of the worker loop. RedisOperations is a named local so it outlives the +// inner async awaits. + +asio::awaitable> peekRunTail( + std::shared_ptr conn) { + RedisOperations ops(std::move(conn)); + // LINDEX -1 reads the tail (oldest) run without removing it. + co_return co_await ops.listIndex(queue_keys::RUN, -1); +} + +asio::awaitable> popStrategyKey( + std::shared_ptr conn, + std::string strategyKey) { + RedisOperations ops(std::move(conn)); + co_return co_await ops.listPopBack(std::move(strategyKey)); +} + +asio::awaitable> takeStrategyPayload( + std::shared_ptr conn, + std::string payloadKey) { + RedisOperations ops(std::move(conn)); + co_return co_await ops.getDelete(std::move(payloadKey)); +} + +asio::awaitable removeRun(std::shared_ptr conn, + std::string descriptorB64) { + RedisOperations ops(std::move(conn)); + // count 0 removes every occurrence, so retiring the run is idempotent. + co_await ops.listRemove(queue_keys::RUN, 0, std::move(descriptorB64)); +} + +} // namespace run_queue diff --git a/source/shared/redis/consumer/runQueue.hpp b/source/run/queue/runQueue.hpp similarity index 67% rename from source/shared/redis/consumer/runQueue.hpp rename to source/run/queue/runQueue.hpp index e594c35..7fad14a 100644 --- a/source/shared/redis/consumer/runQueue.hpp +++ b/source/run/queue/runQueue.hpp @@ -25,11 +25,20 @@ namespace run_queue { boost::asio::awaitable> peekRunTail( std::shared_ptr conn); -// Claims one strategy off the run's per-RUN_ID list. nullopt once drained. -boost::asio::awaitable> popStrategy( +// Claims one strategy payload KEY NAME off the run's per-RUN_ID list (the list +// carries names, not payloads — see queueKeys.hpp). RPOP hands each name to +// exactly one consumer. nullopt once drained. +boost::asio::awaitable> popStrategyKey( std::shared_ptr conn, std::string strategyKey); +// Atomically takes (GETDEL) the payload stored under a popped key name, so the +// payload is consumed exactly once and nothing is left behind. nullopt when the +// key is gone — already consumed by a peer or reaped by its safety-net TTL. +boost::asio::awaitable> takeStrategyPayload( + std::shared_ptr conn, + std::string payloadKey); + // Retires a run by removing its descriptor. Idempotent: LREM removes 0 if a peer // worker already retired it. boost::asio::awaitable removeRun( diff --git a/source/run/reporting/elasticClient.cppm b/source/run/reporting/elasticClient.cppm index b8e8dc9..e29c4ab 100644 --- a/source/run/reporting/elasticClient.cppm +++ b/source/run/reporting/elasticClient.cppm @@ -8,7 +8,7 @@ module; #include -#include "shared/reporting/elasticPublisher.hpp" +#include "run/reporting/elasticPublisher.hpp" #include "run/reporting/tradingResults.hpp" export module elasticClient; @@ -22,6 +22,8 @@ export class ElasticClient { public: static int putTradingResults(const TradingResults& results); static int putTradingFailure(const TradingFailure& failure); + // Compact per-run terminal record; lands in index "trading_final". + static int putTradeFinal(const TradeFinal& result); }; int ElasticClient::putTradingResults(const TradingResults& results) { @@ -31,3 +33,7 @@ int ElasticClient::putTradingResults(const TradingResults& results) { int ElasticClient::putTradingFailure(const TradingFailure& failure) { return elastic::putDocument("trading_failures", nlohmann::json(failure).dump()); } + +int ElasticClient::putTradeFinal(const TradeFinal& result) { + return elastic::putDocument("trading_final", nlohmann::json(result).dump()); +} diff --git a/source/shared/reporting/elasticPublisher.cpp b/source/run/reporting/elasticPublisher.cpp similarity index 99% rename from source/shared/reporting/elasticPublisher.cpp rename to source/run/reporting/elasticPublisher.cpp index 4b8326e..8d969c7 100644 --- a/source/shared/reporting/elasticPublisher.cpp +++ b/source/run/reporting/elasticPublisher.cpp @@ -4,7 +4,7 @@ // This code is licensed under MIT license (see LICENSE.txt for details) // --------------------------------------- -#include "shared/reporting/elasticPublisher.hpp" +#include "run/reporting/elasticPublisher.hpp" #include #include diff --git a/source/shared/reporting/elasticPublisher.hpp b/source/run/reporting/elasticPublisher.hpp similarity index 96% rename from source/shared/reporting/elasticPublisher.hpp rename to source/run/reporting/elasticPublisher.hpp index 5f82dab..32b417e 100644 --- a/source/shared/reporting/elasticPublisher.hpp +++ b/source/run/reporting/elasticPublisher.hpp @@ -8,7 +8,7 @@ #include -#include "shared/reporting/engineException.hpp" +#include "run/reporting/engineException.hpp" // Minimal Elasticsearch HTTP client — PUT-only, for indexing run outcomes and // engine exceptions. Host is read from $ELASTIC_HOST (default diff --git a/source/shared/reporting/engineException.hpp b/source/run/reporting/engineException.hpp similarity index 100% rename from source/shared/reporting/engineException.hpp rename to source/run/reporting/engineException.hpp diff --git a/source/run/reporting/tradingResults.cpp b/source/run/reporting/tradingResults.cpp index ebe32b3..78682a4 100644 --- a/source/run/reporting/tradingResults.cpp +++ b/source/run/reporting/tradingResults.cpp @@ -6,12 +6,24 @@ #include "run/reporting/tradingResults.hpp" -#include "shared/reporting/elasticPublisher.hpp" +#include // gethostname + +#include "run/reporting/elasticPublisher.hpp" std::string TradingResults::nowIsoUtc() { return elastic::nowIsoUtc(); } +std::string TradeFinal::localHostname() { + char buf[256]; + if (::gethostname(buf, sizeof(buf)) != 0) { + return "unknown"; + } + // POSIX leaves truncation behaviour unspecified, so guarantee termination. + buf[sizeof(buf) - 1] = '\0'; + return buf; +} + // decimalToJsonNumber emits decimal64_t fields as JSON numbers (not the // string-encoded form the shared adl_serializer uses) so Elasticsearch/Kibana // types them numerically. These reporting structs are output-only — never parsed @@ -68,6 +80,7 @@ void to_json(nlohmann::json& j, const TradingResults& r) { {"RUN_ID", r.RUN_ID}, {"@timestamp", r.timestamp}, {"durationSeconds", r.durationSeconds}, + {"hostname", r.hostname}, {"config", reportConfigJson(r.config)}, {"results", r.results}, }; @@ -79,7 +92,19 @@ void to_json(nlohmann::json& j, const TradingFailure& f) { {"@timestamp", f.timestamp}, {"durationSeconds", f.durationSeconds}, {"reason", f.reason}, + {"hostname", f.hostname}, {"config", reportConfigJson(f.config)}, {"results", f.results}, }; } + +void to_json(nlohmann::json& j, const TradeFinal& f) { + j = nlohmann::json{ + {"RUN_ID", f.RUN_ID}, + {"@timestamp", f.timestamp}, + {"durationSeconds", f.durationSeconds}, + {"success", f.success}, + {"hostname", f.hostname}, + {"config", reportConfigJson(f.config)}, + }; +} diff --git a/source/run/reporting/tradingResults.hpp b/source/run/reporting/tradingResults.hpp index 721c87e..97122f8 100644 --- a/source/run/reporting/tradingResults.hpp +++ b/source/run/reporting/tradingResults.hpp @@ -55,6 +55,7 @@ struct TradingResults { std::string RUN_ID; std::string timestamp; double durationSeconds; // wall-clock seconds for this run's backtest + std::string hostname; // machine that ran the backtest tradingDefinitions::Configuration config; TradingResultsStats results; @@ -69,10 +70,31 @@ struct TradingFailure { std::string timestamp; double durationSeconds; // wall-clock seconds until the cutoff std::string reason; + std::string hostname; // machine that ran the backtest tradingDefinitions::Configuration config; TradingResultsStats results; }; +// Compact terminal record emitted once per run, regardless of how it ended: +// the input Configuration plus the bare outcome flag, how long it took, and the +// host that produced it. Unlike TradingResults/TradingFailure it carries no +// per-trade stats — it is the at-a-glance "this run finished" signal. Lands in +// index "trading_final". +struct TradeFinal { + std::string RUN_ID; + std::string timestamp; + double durationSeconds; // wall-clock seconds for this run + int success; // 1 = completed, 0 = loss-limit cutoff + std::string hostname; // machine that ran the backtest + tradingDefinitions::Configuration config; + + // Host this process is running on ("unknown" if it can't be resolved). + // Lives here (a normal TU) rather than in the import-std module that calls + // it, since it needs the POSIX gethostname() header. + static std::string localHostname(); +}; + void to_json(nlohmann::json& j, const TradingResultsStats& s); void to_json(nlohmann::json& j, const TradingResults& r); void to_json(nlohmann::json& j, const TradingFailure& f); +void to_json(nlohmann::json& j, const TradeFinal& f); diff --git a/source/run/runCommand.cppm b/source/run/runCommand.cppm index 40a420d..92ffed7 100644 --- a/source/run/runCommand.cppm +++ b/source/run/runCommand.cppm @@ -8,7 +8,7 @@ module; #include "shared/utilities/env.hpp" #include "shared/utilities/jsonParser.hpp" -#include "shared/redis/consumer/redisRunner.hpp" +#include "run/queue/redisRunner.hpp" export module runCommand; @@ -22,6 +22,8 @@ public: int RunCommand::run(const int argc, const char* argv[]) { + env::printDiagnostics(argc, argv); + if (argc < 3) { std::println(stderr, "Usage: BacktestingEngine run \n" " BacktestingEngine run "); diff --git a/source/run/trading/reviewStopAndLimit.cppm b/source/run/trading/reviewStopAndLimit.cppm index 7f2ac27..bf4a660 100644 --- a/source/run/trading/reviewStopAndLimit.cppm +++ b/source/run/trading/reviewStopAndLimit.cppm @@ -6,35 +6,29 @@ export module reviewStopAndLimit; -import std; // replaces , , , +import std; // replaces import tradeManager; // TradeManager +import trade; // Trade import exitRules; // trading::exit_rules::checkExit import priceData; // PriceData export namespace trading { -// Walk every active trade, close any whose SL/TP has been hit on this -// tick. Two-phase to avoid invalidating the map iterator while erasing. +// Close the tick symbol's open trade if its SL/TP has been hit on this tick. +// TradeManager keys open trades by symbol (at most one per symbol), so this +// is a single O(1) lookup plus checkExit's integer comparisons — nothing +// allocates on the per-tick path. // -// Symbol-aware: a tick for one instrument must never trigger an exit on -// a trade opened for a different instrument. The per-trade filter below -// is the production fix — `exit_rules::checkExit` is symbol-agnostic -// and would otherwise compare e.g. an AUSIDXAUD price (thousands) -// against a EURUSD stop level (~1.10) and spuriously close the trade. +// Symbol-keying is also what keeps the check symbol-safe: a tick for one +// instrument can never reach a trade on another. `exit_rules::checkExit` is +// symbol-agnostic and would otherwise compare e.g. an AUSIDXAUD price +// (thousands) against a EURUSD stop level (~1.10) and spuriously close the +// trade. inline void reviewStopAndLimit(TradeManager& tradeManager, const PriceData& tick) { - const auto& openTrades = tradeManager.getActiveTrades(); - if (openTrades.empty()) return; - - std::vector> toClose; - toClose.reserve(openTrades.size()); - for (const auto& [id, trade] : openTrades) { - if (trade.symbol != tick.symbol) continue; - if (auto exitPrice = trading::exit_rules::checkExit(trade, tick)) { - toClose.emplace_back(id, *exitPrice); - } - } - for (const auto& [id, exitPrice] : toClose) { - tradeManager.closeTrade(id, exitPrice, tick); + const Trade* trade = tradeManager.findActiveTrade(tick.symbol); + if (trade == nullptr) return; + if (auto exitPrice = trading::exit_rules::checkExit(*trade, tick)) { + tradeManager.closeTrade(tick.symbol, *exitPrice, tick); } } diff --git a/source/run/trading/runLoop.cppm b/source/run/trading/runLoop.cppm index 7a01ee7..4b5b455 100644 --- a/source/run/trading/runLoop.cppm +++ b/source/run/trading/runLoop.cppm @@ -65,7 +65,7 @@ inline RunStatus runTicks(TradeManager& tradeManager, const std::vector& ticks, const tradingDefinitions::TradingVariables& vars, const RiskLimits& limits = {}) { - const boost::decimal::decimal64_t zero{0}; + constexpr boost::decimal::decimal64_t zero{0}; const bool lossLimitActive = limits.maxLossPercent > zero; // Lowest equity the run may reach, in int64 points. balance * loss% gives // the floor in pips; scaling by points-per-pip puts it in the same integer diff --git a/source/run/trading/tradeManager.cppm b/source/run/trading/tradeManager.cppm index 9e1949f..51c28e0 100644 --- a/source/run/trading/tradeManager.cppm +++ b/source/run/trading/tradeManager.cppm @@ -20,8 +20,24 @@ import priceData; // PriceData export class TradeManager { private: - std::unordered_map activeTrades; + // Transparent hasher so the per-tick lookups can probe with the tick's + // symbol as a string_view — no temporary std::string per tick. + struct SymbolHash { + using is_transparent = void; + std::size_t operator()(std::string_view symbol) const noexcept { + return std::hash{}(symbol); + } + }; + // Open trades keyed by SYMBOL. The engine allows at most one open trade + // per symbol (runTicks gates entries on hasActiveTradeForSymbol), so every + // per-tick operation — mark-to-market, SL/TP review, the re-entry gate — + // is a single O(1) find instead of a walk of the whole map. + std::unordered_map> activeTrades; std::vector closedTrades; + // Trade ids only need to be unique within a run (reporting scopes them by + // RUN_ID), so a plain per-manager counter replaces the old process-global + // atomic that every worker thread contended on. + std::uint64_t tradeCounter{0}; // Running sums (int64 points-per-lot) maintained by openTrade/markToMarket/ // closeTrade so the per-tick loss-limit check in runTicks stays O(1): // realized PnL across closed trades, and floating (mark-to-market) PnL @@ -48,13 +64,18 @@ public: std::int32_t limitDistancePips = 0); std::size_t reviewAccount() const; bool hasActiveTradeForSymbol(std::string_view symbol) const; - // `liquidated` marks the close as forced by the account loss limit - // rather than earned via SL/TP or strategy logic. - bool closeTrade(const std::string& tradeId, + // The symbol's open trade, or nullptr when it has none. O(1); the pointer + // is invalidated by the next open/close. + const Trade* findActiveTrade(std::string_view symbol) const; + // Close the symbol's open trade. `liquidated` marks the close as forced by + // the account loss limit rather than earned via SL/TP or strategy logic. + bool closeTrade(std::string_view symbol, std::int32_t closePrice, const PriceData& tick, bool liquidated = false); - const std::unordered_map& getActiveTrades() const; + // Keyed by symbol (see activeTrades above). + const std::unordered_map>& + getActiveTrades() const; const std::vector& getClosedTrades() const; // Realized PnL (int64 points-per-lot) across all closed trades. O(1). std::int64_t calculatePnl() const; @@ -74,11 +95,6 @@ public: }; namespace { -std::string nextTradeId() { - static std::atomic counter{0}; - return std::format("T{}", counter.fetch_add(1)); -} - // Floating PnL (int64 points-per-lot) of an open trade valued at `mark` — the // same formula closeTrade uses for realized PnL, so liquidating at the last // mark realizes exactly the floating amount. Pure integer: no decimal on the @@ -104,9 +120,10 @@ std::string TradeManager::openTrade(const PriceData& tick, std::int32_t limitDistancePips) { auto price = (direction == Direction::LONG) ? tick.ask : tick.bid; Trade trade(price, size, direction, tick.symbol); + trade.openTime = tick.timestamp; // simulation time, not wall clock trade.entryBid = tick.bid; trade.entryAsk = tick.ask; - trade.id = nextTradeId(); + trade.id = std::format("T{}", tradeCounter++); trade.stopDistancePips = stopDistancePips; trade.limitDistancePips = limitDistancePips; trade.exitReferencePrice = (direction == Direction::LONG) ? tick.bid : tick.ask; @@ -128,33 +145,43 @@ std::string TradeManager::openTrade(const PriceData& tick, // equity dips by the spread the moment a trade opens. trade.lastMarkPrice = trade.exitReferencePrice; trade.floatingPnl = floatingPnlAt(trade, trade.lastMarkPrice); - openPnl += trade.floatingPnl; - activeTrades[trade.id] = trade; + // try_emplace copy-constructs the key (pair::first) before it moves the + // Trade into pair::second, so keying on trade.symbol here is safe. If the + // symbol already has an open trade the insert is refused — accounting is + // untouched and the existing trade's id comes back. Production never hits + // that: runTicks gates entries on hasActiveTradeForSymbol. + auto [it, inserted] = activeTrades.try_emplace(trade.symbol, std::move(trade)); + if (!inserted) { + return it->second.id; + } + openPnl += it->second.floatingPnl; updateDrawdown(); // equity dips by the spread the moment a trade opens - return trade.id; + return it->second.id; } void TradeManager::markToMarket(const PriceData& tick) { - for (auto& [id, trade] : activeTrades) { - if (trade.symbol != tick.symbol) continue; - const auto mark = (trade.direction == Direction::LONG) ? tick.bid : tick.ask; - const auto updated = floatingPnlAt(trade, mark); - openPnl += updated - trade.floatingPnl; - trade.floatingPnl = updated; - trade.lastMarkPrice = mark; - } - updateDrawdown(); // capture floating drawdown at this tick's marks + const auto it = activeTrades.find(std::string_view{tick.symbol}); + if (it == activeTrades.end()) return; // no position: equity unchanged + Trade& trade = it->second; + const auto mark = (trade.direction == Direction::LONG) ? tick.bid : tick.ask; + const auto updated = floatingPnlAt(trade, mark); + openPnl += updated - trade.floatingPnl; + trade.floatingPnl = updated; + trade.lastMarkPrice = mark; + updateDrawdown(); // capture floating drawdown at this tick's mark } void TradeManager::closeAllTrades(const PriceData& tick) { - // Snapshot ids/prices first: closeTrade mutates activeTrades. + // Snapshot symbols/prices first: closeTrade mutates activeTrades. Runs at + // most once per run (loss-limit breach), so the allocation is off the + // per-tick path. std::vector> toClose; toClose.reserve(activeTrades.size()); - for (const auto& [id, trade] : activeTrades) { - toClose.emplace_back(id, trade.lastMarkPrice); + for (const auto& [symbol, trade] : activeTrades) { + toClose.emplace_back(symbol, trade.lastMarkPrice); } - for (const auto& [id, price] : toClose) { - closeTrade(id, price, tick, /*liquidated=*/true); + for (const auto& [symbol, price] : toClose) { + closeTrade(symbol, price, tick, /*liquidated=*/true); } } @@ -163,19 +190,24 @@ std::size_t TradeManager::reviewAccount() const { } bool TradeManager::hasActiveTradeForSymbol(std::string_view symbol) const { - return std::any_of(activeTrades.begin(), activeTrades.end(), - [symbol](const auto& pair) { - return pair.second.symbol == symbol; - }); + return activeTrades.contains(symbol); } -bool TradeManager::closeTrade(const std::string& tradeId, +const Trade* TradeManager::findActiveTrade(std::string_view symbol) const { + const auto it = activeTrades.find(symbol); + return it != activeTrades.end() ? &it->second : nullptr; +} + +bool TradeManager::closeTrade(std::string_view symbol, std::int32_t closePrice, const PriceData& tick, bool liquidated) { - auto it = activeTrades.find(tradeId); + auto it = activeTrades.find(symbol); if (it != activeTrades.end()) { - Trade closed = it->second; + // Move the trade out (ints survive the move; only the strings transfer) + // so closing never copies the 5-string Trade struct. + Trade closed = std::move(it->second); + activeTrades.erase(it); closed.closePrice = closePrice; closed.closeTime = tick.timestamp; closed.liquidated = liquidated; @@ -184,10 +216,8 @@ bool TradeManager::closeTrade(const std::string& tradeId, // Realized PnL in int64 points-per-lot (converted to pips at reporting). closed.pnl = static_cast(diff) * closed.size; closedPnl += closed.pnl; - openPnl -= it->second.floatingPnl; // realized now, no longer floating + openPnl -= closed.floatingPnl; // realized now, no longer floating closed.floatingPnl = 0; - closedTrades.push_back(closed); - activeTrades.erase(it); updateDrawdown(); // realized exit may differ from the last mark // Per-trade chatter is skipped under concurrent backtests (quiet), @@ -211,12 +241,14 @@ bool TradeManager::closeTrade(const std::string& tradeId, << std::noshowpos << std::endl; } + closedTrades.push_back(std::move(closed)); return true; } return false; } -const std::unordered_map& TradeManager::getActiveTrades() const { +const std::unordered_map>& +TradeManager::getActiveTrades() const { return activeTrades; } diff --git a/source/shared/ipc/engineControl.cpp b/source/shared/ipc/engineControl.cpp new file mode 100644 index 0000000..3a17871 --- /dev/null +++ b/source/shared/ipc/engineControl.cpp @@ -0,0 +1,59 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "shared/ipc/engineControl.hpp" + +#include +#include +#include +#include + +#include +#include + +// This is the one TU that names Boost.Interprocess. It mirrors the isolation +// pattern of boostRedisImpl.cpp: the heavy, macro-laden header is included here +// and nowhere else, behind the pimpl declared in engineControl.hpp. +namespace bip = boost::interprocess; +namespace fs = std::filesystem; + +namespace ipc { + +struct EngineControlChannel::Impl { + bip::file_mapping file; + bip::mapped_region region; +}; + +EngineControlChannel::EngineControlChannel(std::string path) + : path_(std::move(path)), impl_(std::make_unique()) { + // Recreate the backing file from scratch so a segment left behind by a + // previous (crashed) run can't carry stale state forward. (Same intent as + // the struct_shm_remove guard in the reference snippet.) + const fs::path file_path(path_); + if (file_path.has_parent_path()) { + fs::create_directories(file_path.parent_path()); + } + { std::ofstream create(path_, std::ios::binary | std::ios::trunc); } + fs::resize_file(file_path, sizeof(EngineState)); + + impl_->file = bip::file_mapping(path_.c_str(), bip::read_write); + impl_->region = bip::mapped_region(impl_->file, bip::read_write); + + // Placement-new the atomics into the mapped bytes, then publish a known-good + // initial state (idle, running). + state_ = ::new (impl_->region.get_address()) EngineState(); + state_->active_jobs.store(0, std::memory_order_relaxed); + state_->stop_signal.store(0, std::memory_order_relaxed); +} + +EngineControlChannel::~EngineControlChannel() { + state_ = nullptr; + impl_.reset(); // unmap + close before removing the file + std::error_code ec; + fs::remove(fs::path(path_), ec); // best-effort; ignore if already gone +} + +} // namespace ipc diff --git a/source/shared/ipc/engineControl.hpp b/source/shared/ipc/engineControl.hpp new file mode 100644 index 0000000..600ce8b --- /dev/null +++ b/source/shared/ipc/engineControl.hpp @@ -0,0 +1,84 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include + +// A tiny shared-memory control channel between the C++ engine and a local Python +// monitor (source/backtesting-controller). The engine broadcasts how many +// backtests are in flight; the monitor can flip a stop flag to ask for a graceful +// drain-and-pause. Zero network, zero serialisation — both sides map the same +// 8-byte block. +// +// The block is a memory-mapped FILE at a fixed path (DEFAULT_PATH below), mapped +// via Boost.Interprocess `file_mapping`. We use a file rather than +// `shared_memory_object` because the latter is not portable to a `mmap`-based +// Python reader on macOS: there, Boost falls back to a file under its own private +// directory while Python's `multiprocessing.shared_memory` only speaks `shm_open`, +// so the two never meet. A shared file path is deterministic and identical on +// macOS and Linux; the Python side just `mmap`s the same path. +// +// Boost.Interprocess is confined to engineControl.cpp (see the boostRedisImpl.cpp +// pattern) so includers of this header stay free of Boost headers and macros. +namespace ipc { + +// A sentinel written at offset 0 so a reader can prove it mapped *this* engine's +// control block before trusting the rest of the bytes — not a stale file from an +// older struct layout, a half-initialised file, or some unrelated file that +// happens to sit at the same path. +inline constexpr std::uint32_t MAGIC = 0xDEADBEEFu; + +// The exact bytes mapped into both processes: a magic sentinel followed by two +// naturally-aligned, always lock-free 32-bit atomics, 12 bytes total with no +// padding: +// [0, 4) magic — fixed 0xDEADBEEF identity/layout marker +// [4, 8) active_jobs — C++ writes, Python reads +// [8, 12) stop_signal — Python writes (0 = run, 1 = stop), C++ reads +// A single aligned read/write of each field is atomic on x86-64 and arm64, so the +// Python side can use raw `struct` access on the corresponding byte ranges. +struct EngineState { + const std::uint32_t magic = MAGIC; + std::atomic active_jobs; + std::atomic stop_signal; +}; + +static_assert(std::atomic::is_always_lock_free, + "EngineState relies on lock-free atomics for cross-process access"); +static_assert(sizeof(EngineState) == 12, "EngineState must be a packed 12-byte block"); +static_assert(alignof(EngineState) == 4, "EngineState fields must be 4-byte aligned"); + +// The default backing file. The Python controller defaults to the same path. +inline constexpr const char* DEFAULT_PATH = "/tmp/EngineControlShm"; + +// RAII owner of the mapped file. Construction creates and zeroes the file +// (replacing any stale one left by a crashed run); destruction unmaps and removes +// it. Throws on failure — callers that treat the channel as best-effort should +// construct it inside a try/catch. +class EngineControlChannel { +public: + explicit EngineControlChannel(std::string path = DEFAULT_PATH); + ~EngineControlChannel(); + + EngineControlChannel(const EngineControlChannel&) = delete; + EngineControlChannel& operator=(const EngineControlChannel&) = delete; + + // The live, mapped state. Valid for the lifetime of this object. + EngineState& state() noexcept { return *state_; } + + const std::string& path() const noexcept { return path_; } + +private: + std::string path_; + struct Impl; // hides the Boost.Interprocess members + std::unique_ptr impl_; + EngineState* state_ = nullptr; +}; + +} // namespace ipc diff --git a/source/shared/models/ohlcObject.cppm b/source/shared/models/ohlcObject.cppm new file mode 100644 index 0000000..e7de3ca --- /dev/null +++ b/source/shared/models/ohlcObject.cppm @@ -0,0 +1,32 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +export module ohlcObject; + +import std; // replaces , , + +// One OHLC (open/high/low/close) bar, built from a stream of ticks by the +// ohlcBuilder module. Prices are the same scaled fixed-point INT32 points used +// everywhere in the engine (see priceData / symbolScale) — NOT decimals. +// +// C# analogy: OhlcObject { DateTime date; decimal open/high/low/close; }. +// The defaults mirror the C# ones: high starts at the smallest representable +// value and low at the largest, so the first real price always wins the +// max/min comparisons in the builder. +export struct OhlcObject { + // Simulation time of the first tick in this bar (C# DateTime.MinValue + // analogue: a default time_point is the clock's epoch). + std::chrono::system_clock::time_point date{}; + + std::int32_t open = 0; + std::int32_t close = 0; + std::int32_t high = std::numeric_limits::min(); + std::int32_t low = std::numeric_limits::max(); + + // Set once the next bar opens — a complete bar's OHLC values are final, + // the in-progress (last) bar's are still moving with each tick. + bool complete = false; +}; diff --git a/source/shared/models/trade.cppm b/source/shared/models/trade.cppm index 0e0bf53..9e5bd56 100644 --- a/source/shared/models/trade.cppm +++ b/source/shared/models/trade.cppm @@ -24,6 +24,8 @@ export struct Trade { std::int32_t entryBid; std::int32_t entryAsk; std::int32_t size; + // Simulation time of the entry tick, set by TradeManager::openTrade — + // never wall-clock, so open/close durations are meaningful in backtests. std::chrono::system_clock::time_point openTime; Direction direction; @@ -63,13 +65,14 @@ export struct Trade { // logic — lets reporting separate forced closes from organic ones. bool liquidated = false; - // Default constructor + // Default constructor. openTime stays at the epoch — TradeManager::openTrade + // stamps it from the entry tick. Trade() : entryPrice(0), entryBid(0), entryAsk(0), size(0), direction(Direction::LONG), scalingFactor(0), stopDistancePips(0), limitDistancePips(0), exitReferencePrice(0), stopPrice(0), limitPrice(0), closePrice(0), pnl(0), lastMarkPrice(0), floatingPnl(0), - openTime(std::chrono::system_clock::now()) {} + openTime{} {} // Copy constructor Trade(const Trade& other) = default; @@ -82,7 +85,7 @@ export struct Trade { entryBid(0), entryAsk(0), size(quantity), - openTime(std::chrono::system_clock::now()), + openTime{}, direction(dir), symbol(tradeSymbol), scalingFactor(symbol_scale::get(tradeSymbol)), diff --git a/source/shared/redis/consumer/runQueue.cpp b/source/shared/redis/consumer/runQueue.cpp deleted file mode 100644 index 850300a..0000000 --- a/source/shared/redis/consumer/runQueue.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// Backtesting Engine in C++ -// -// (c) 2026 Ryan McCaffery | https://mccaffers.com -// This code is licensed under MIT license (see LICENSE.txt for details) -// --------------------------------------- - -#include "shared/redis/consumer/runQueue.hpp" - -#include -#include -#include - -#include -#include - -#include "shared/utilities/queueKeys.hpp" - -namespace asio = boost::asio; -namespace redis = boost::redis; - -namespace { - -// One Redis command on the shared connection, returning a bulk string (or nil). -// A nil reply (RPOP/LINDEX out of range) maps to nullopt. The connection is NOT -// cancelled here, it stays open for the rest of the worker loop. -asio::awaitable> execOptionalString( - const std::shared_ptr conn, - const redis::request req) { - redis::response> resp; - co_await conn->async_exec(req, resp, asio::use_awaitable); - co_return std::move(std::get<0>(resp).value()); -} - -// One Redis command on the shared connection whose reply we ignore (e.g. LREM). -asio::awaitable execIgnore(std::shared_ptr conn, - redis::request req) { - redis::generic_response resp; - co_await conn->async_exec(req, resp, asio::use_awaitable); - co_return; -} - -} // namespace - -namespace run_queue { - -asio::awaitable> peekRunTail( - std::shared_ptr conn) { - redis::request req; - req.push("LINDEX", queue_keys::RUN, "-1"); - co_return co_await execOptionalString(std::move(conn), std::move(req)); -} - -asio::awaitable> popStrategy( - std::shared_ptr conn, - std::string strategyKey) { - redis::request req; - req.push("RPOP", strategyKey); - co_return co_await execOptionalString(std::move(conn), std::move(req)); -} - -asio::awaitable removeRun(std::shared_ptr conn, - std::string descriptorB64) { - redis::request req; - req.push("LREM", queue_keys::RUN, "0", descriptorB64); - co_await execIgnore(std::move(conn), std::move(req)); -} - -} // namespace run_queue diff --git a/source/shared/redis/operations/redisOperations.cpp b/source/shared/redis/operations/redisOperations.cpp new file mode 100644 index 0000000..d56f2c0 --- /dev/null +++ b/source/shared/redis/operations/redisOperations.cpp @@ -0,0 +1,333 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#include "shared/redis/operations/redisOperations.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace asio = boost::asio; +namespace redis = boost::redis; + +RedisOperations::RedisOperations(std::shared_ptr conn) + : conn_(std::move(conn)) {} + +// ---- Private exec primitives ---- + +asio::awaitable> RedisOperations::execOptionalString( + redis::request req) { + redis::response> resp; + co_await conn_->async_exec(req, resp, asio::use_awaitable); + co_return std::move(std::get<0>(resp).value()); +} + +asio::awaitable RedisOperations::execIgnore(redis::request req) { + redis::generic_response resp; + co_await conn_->async_exec(req, resp, asio::use_awaitable); + co_return; +} + +template +asio::awaitable RedisOperations::execValue(redis::request req) { + redis::response resp; + co_await conn_->async_exec(req, resp, asio::use_awaitable); + co_return std::move(std::get<0>(resp).value()); +} + +// ---- Strings ---- + +asio::awaitable> RedisOperations::getString( + std::string key) { + redis::request req; + req.push("GET", key); + co_return co_await execOptionalString(std::move(req)); +} + +asio::awaitable RedisOperations::setString(std::string key, + std::string value) { + redis::request req; + req.push("SET", key, value); + co_await execIgnore(std::move(req)); +} + +asio::awaitable RedisOperations::setString( + std::string key, + std::string value, + SetWhen when, + std::optional expiry) { + // SET key value [PX ms] [NX|XX]. Built as a dynamic argument range because + // the optional flags vary at runtime. + std::vector args{std::move(key), std::move(value)}; + if (expiry) { + args.emplace_back("PX"); + args.emplace_back(std::to_string(expiry->count())); + } + if (when == SetWhen::NotExists) { + args.emplace_back("NX"); + } else if (when == SetWhen::Exists) { + args.emplace_back("XX"); + } + + redis::request req; + req.push_range("SET", args); + + // A conditional SET replies with the stored value ("OK") on success or nil + // when the NX/XX condition blocked the write. + const std::optional reply = + co_await execOptionalString(std::move(req)); + co_return reply.has_value(); +} + +asio::awaitable RedisOperations::setIfNotExists( + std::string key, + std::string value, + std::optional expiry) { + co_return co_await setString(std::move(key), std::move(value), + SetWhen::NotExists, expiry); +} + +asio::awaitable RedisOperations::setMultiple( + std::vector> keyValuePairs) { + if (keyValuePairs.empty()) { + co_return; + } + redis::request req; + // Each pair serializes to two bulk strings (key, value): MSET k1 v1 k2 v2 ... + req.push_range("MSET", keyValuePairs); + co_await execIgnore(std::move(req)); +} + +asio::awaitable RedisOperations::setMultipleWithTTL( + std::vector> keyValuePairs, + std::chrono::milliseconds ttl) { + if (keyValuePairs.empty()) { + co_return; + } + const std::string ttlMs = std::to_string(ttl.count()); + redis::request req; + for (const auto& [key, value] : keyValuePairs) { + req.push("SET", key, value, "PX", ttlMs); + } + co_await execIgnore(std::move(req)); +} + +asio::awaitable> RedisOperations::getDelete( + std::string key) { + redis::request req; + req.push("GETDEL", key); + co_return co_await execOptionalString(std::move(req)); +} + +asio::awaitable> +RedisOperations::getMultiple(std::vector keys) { + std::map result; + if (keys.empty()) { + co_return result; + } + + redis::request req; + req.push_range("MGET", keys); + + auto values = + co_await execValue>>( + std::move(req)); + + // MGET preserves request order; zip values back onto their keys, skipping + // nils so missing keys are simply absent (mirrors the C# behaviour). + for (std::size_t i = 0; i < keys.size() && i < values.size(); ++i) { + if (values[i].has_value()) { + result.emplace(std::move(keys[i]), std::move(*values[i])); + } + } + co_return result; +} + +// ---- Keys / TTL ---- + +asio::awaitable RedisOperations::deleteKey(std::string key) { + redis::request req; + req.push("DEL", key); + co_return (co_await execValue(std::move(req))) > 0; +} + +asio::awaitable RedisOperations::deleteKeys(std::vector keys) { + if (keys.empty()) { + co_return 0; + } + redis::request req; + req.push_range("DEL", keys); + co_return static_cast(co_await execValue(std::move(req))); +} + +asio::awaitable RedisOperations::setTTL(std::string key, + std::chrono::milliseconds ttl) { + redis::request req; + req.push("PEXPIRE", key, ttl.count()); + co_return (co_await execValue(std::move(req))) != 0; +} + +asio::awaitable RedisOperations::keyExpire( + std::string key, + std::optional expiry) { + redis::request req; + if (expiry) { + req.push("PEXPIRE", key, expiry->count()); + } else { + req.push("PERSIST", key); + } + co_await execIgnore(std::move(req)); +} + +asio::awaitable RedisOperations::keyRename(std::string sourceKey, + std::string destinationKey) { + redis::request req; + req.push("RENAME", sourceKey, destinationKey); + co_await execIgnore(std::move(req)); + co_return true; +} + +asio::awaitable> RedisOperations::getKeysByPattern( + std::string pattern) { + std::vector keys; + std::string cursor = "0"; + + // SCAN replies with a nested array [next-cursor, [keys...]] which the typed + // adapters can't model in one reply, so decode the flat RESP3 node tree: + // depth 0 -> the outer 2-element array + // depth 1 -> [0] the cursor, [1] the keys array + // depth 2 -> each key + // Loop until the cursor wraps back to "0". + do { + redis::request req; + req.push("SCAN", cursor, "MATCH", pattern, "COUNT", "250"); + + redis::generic_response resp; + co_await conn_->async_exec(req, resp, asio::use_awaitable); + + const auto& nodes = resp.value(); + if (nodes.size() < 2) { + break; // malformed reply; nothing more to read + } + cursor = nodes[1].value; // first depth-1 element is the cursor + + for (std::size_t i = 2; i < nodes.size(); ++i) { + if (nodes[i].depth == 2 && + nodes[i].data_type == redis::resp3::type::blob_string) { + keys.push_back(nodes[i].value); + } + } + } while (cursor != "0"); + + co_return keys; +} + +// ---- Sets ---- + +asio::awaitable RedisOperations::setAdd(std::string key, + std::string value) { + redis::request req; + req.push("SADD", key, value); + co_return (co_await execValue(std::move(req))) != 0; +} + +asio::awaitable RedisOperations::setRemove(std::string key, + std::string value) { + redis::request req; + req.push("SREM", key, value); + co_return (co_await execValue(std::move(req))) != 0; +} + +asio::awaitable> RedisOperations::setMembers( + std::string key) { + redis::request req; + req.push("SMEMBERS", key); + co_return co_await execValue>(std::move(req)); +} + +asio::awaitable RedisOperations::setContains(std::string key, + std::string value) { + redis::request req; + req.push("SISMEMBER", key, value); + co_return (co_await execValue(std::move(req))) != 0; +} + +asio::awaitable RedisOperations::setLength(std::string key) { + redis::request req; + req.push("SCARD", key); + co_return static_cast(co_await execValue(std::move(req))); +} + +// ---- Lists ---- + +asio::awaitable RedisOperations::listPushFront(std::string key, + std::string value) { + redis::request req; + req.push("LPUSH", key, value); + co_await execIgnore(std::move(req)); +} + +asio::awaitable RedisOperations::listPushFront( + std::string key, + std::vector values) { + redis::request req; + for (const std::string& value : values) { + req.push("LPUSH", key, value); + } + co_await execIgnore(std::move(req)); +} + +asio::awaitable> RedisOperations::listPopBack( + std::string key) { + redis::request req; + req.push("RPOP", key); + co_return co_await execOptionalString(std::move(req)); +} + +asio::awaitable> RedisOperations::listIndex( + std::string key, + long index) { + redis::request req; + req.push("LINDEX", key, index); + co_return co_await execOptionalString(std::move(req)); +} + +asio::awaitable RedisOperations::listRemove(std::string key, + long count, + std::string value) { + redis::request req; + req.push("LREM", key, count, value); + co_await execIgnore(std::move(req)); +} + +// ---- Generic ---- + +asio::awaitable RedisOperations::execute( + std::vector commandAndArgs) { + redis::generic_response resp; + if (commandAndArgs.empty()) { + co_return resp; + } + + redis::request req; + if (commandAndArgs.size() == 1) { + req.push(commandAndArgs[0]); + } else { + req.push_range(commandAndArgs[0], commandAndArgs.begin() + 1, + commandAndArgs.end()); + } + + co_await conn_->async_exec(req, resp, asio::use_awaitable); + co_return resp; +} diff --git a/source/shared/redis/operations/redisOperations.hpp b/source/shared/redis/operations/redisOperations.hpp new file mode 100644 index 0000000..4d5cce9 --- /dev/null +++ b/source/shared/redis/operations/redisOperations.hpp @@ -0,0 +1,165 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +// Conditional-set semantics for SET, the analogue of StackExchange.Redis' When. +// Always issues a plain SET; NotExists/Exists add the NX/XX flag. +enum class SetWhen { Always, NotExists, Exists }; + +// Thin command layer over a borrowed Boost.Redis connection. Centralizes +// request building, async_exec and reply decoding so no caller hardcodes a Redis +// command string. Modelled on the C# IRedisOperations surface, adapted to the +// Boost.Redis coroutine model: there is no connection pool, CommandFlags or +// IBatch here (Boost.Redis multiplexes one connection per io_context and batches +// by pushing multiple commands into a single request). This class does NOT own +// the connection lifecycle, it never calls cancel(); creation lives in +// redis_util::makeRedisConnection and teardown stays with the caller. +class RedisOperations final { +public: + explicit RedisOperations(std::shared_ptr conn); + + // ---- Strings ---- + + // GET key. nullopt when the key is missing. + boost::asio::awaitable> getString(std::string key); + + // SET key value (unconditional). + boost::asio::awaitable setString(std::string key, std::string value); + + // SET key value [PX ms] [NX|XX]. Returns true when the value was stored + // (false when a NotExists/Exists condition prevented it). + boost::asio::awaitable setString( + std::string key, + std::string value, + SetWhen when, + std::optional expiry); + + // SET key value NX [PX ms]. Returns true when the key did not already exist. + boost::asio::awaitable setIfNotExists( + std::string key, + std::string value, + std::optional expiry = std::nullopt); + + // MSET k1 v1 k2 v2 ... (one round trip, no expiry). + boost::asio::awaitable setMultiple( + std::vector> keyValuePairs); + + // SET k v PX ms for every pair, pipelined into one request (one round + // trip). MSET cannot attach a TTL, hence per-key SETs. + boost::asio::awaitable setMultipleWithTTL( + std::vector> keyValuePairs, + std::chrono::milliseconds ttl); + + // GETDEL key: atomically read and remove, so a value handed to one caller + // can never be observed by another. nullopt when the key is missing. + // Requires Redis >= 6.2. + boost::asio::awaitable> getDelete( + std::string key); + + // MGET k1 k2 ... Missing keys are omitted from the result map. + boost::asio::awaitable> getMultiple( + std::vector keys); + + // ---- Keys / TTL ---- + + // DEL key. true when the key existed and was removed. + boost::asio::awaitable deleteKey(std::string key); + + // DEL k1 k2 ... Returns the number of keys actually removed. + boost::asio::awaitable deleteKeys(std::vector keys); + + // PEXPIRE key ms. true when the expiry was applied. + boost::asio::awaitable setTTL(std::string key, + std::chrono::milliseconds ttl); + + // PEXPIRE key ms when expiry is set, otherwise PERSIST key. + boost::asio::awaitable keyExpire( + std::string key, + std::optional expiry); + + // RENAME source destination. Returns true on success (throws if source is + // missing, matching the underlying RENAME error). + boost::asio::awaitable keyRename(std::string sourceKey, + std::string destinationKey); + + // SCAN-based key enumeration (never KEYS). Returns all keys matching pattern. + boost::asio::awaitable> getKeysByPattern( + std::string pattern); + + // ---- Sets ---- + + // SADD key value. true when the member was newly added. + boost::asio::awaitable setAdd(std::string key, std::string value); + + // SREM key value. true when the member was present and removed. + boost::asio::awaitable setRemove(std::string key, std::string value); + + // SMEMBERS key. + boost::asio::awaitable> setMembers(std::string key); + + // SISMEMBER key value. + boost::asio::awaitable setContains(std::string key, std::string value); + + // SCARD key. + boost::asio::awaitable setLength(std::string key); + + // ---- Lists (engine work-queue ops; no C# equivalent) ---- + + // LPUSH key value (prepend). + boost::asio::awaitable listPushFront(std::string key, + std::string value); + + // LPUSH key v1 v2 ... as N separate entries pipelined in one request, so a + // later RPOP yields them in insertion order. + boost::asio::awaitable listPushFront(std::string key, + std::vector values); + + // RPOP key. nullopt when the list is empty. + boost::asio::awaitable> listPopBack( + std::string key); + + // LINDEX key index. nullopt when the index is out of range. + boost::asio::awaitable> listIndex(std::string key, + long index); + + // LREM key count value. + boost::asio::awaitable listRemove(std::string key, + long count, + std::string value); + + // ---- Generic escape hatch (C# ExecuteAsync) ---- + + // Runs an arbitrary command (first element) with its arguments and returns + // the raw reply tree for the caller to interpret. + boost::asio::awaitable execute( + std::vector commandAndArgs); + +private: + // One command returning a bulk string (or nil -> nullopt). + boost::asio::awaitable> execOptionalString( + boost::redis::request req); + + // One command whose reply is ignored. + boost::asio::awaitable execIgnore(boost::redis::request req); + + // One command decoded into a single typed reply (e.g. long long, vectors). + template + boost::asio::awaitable execValue(boost::redis::request req); + + std::shared_ptr conn_; +}; diff --git a/source/shared/redis/producer/redisLoader.cpp b/source/shared/redis/producer/redisLoader.cpp deleted file mode 100644 index c8a1cd7..0000000 --- a/source/shared/redis/producer/redisLoader.cpp +++ /dev/null @@ -1,158 +0,0 @@ -// Backtesting Engine in C++ -// -// (c) 2026 Ryan McCaffery | https://mccaffers.com -// This code is licensed under MIT license (see LICENSE.txt for details) -// --------------------------------------- - -#include "shared/redis/producer/redisLoader.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "shared/utilities/base64.hpp" -#include "shared/redis/connection/redisConnection.hpp" - -namespace asio = boost::asio; -namespace redis = boost::redis; - -namespace { - -// A whitespace-only (or empty) payload is rejected before it reaches Redis. -bool isBlank(const std::string& rawJson) { - return std::ranges::all_of(rawJson, [](unsigned char c) { - return std::isspace(c); - }); -} - -asio::awaitable pushOnce( - std::shared_ptr conn, - std::string queueKey, - std::string encoded) { - redis::request req; - req.push("LPUSH", queueKey, encoded); - - redis::generic_response resp; - co_await conn->async_exec(req, resp, asio::use_awaitable); - - conn->cancel(); - co_return; -} - -// Pipelines one LPUSH per payload over a single connection (one round trip). -// LPUSH prepends, so RPOP later yields the payloads in insertion order. -asio::awaitable pushManyOnce( - std::shared_ptr conn, - std::string queueKey, - std::vector encodedPayloads) { - redis::request req; - for (const std::string& encoded : encodedPayloads) { - req.push("LPUSH", queueKey, encoded); - } - - redis::generic_response resp; - co_await conn->async_exec(req, resp, asio::use_awaitable); - - conn->cancel(); - co_return; -} - -} // namespace - -int RedisLoader::loadPayload(const std::string& redisHost, - int redisPort, - const std::string& queueKey, - const std::string& rawJson) { - if (isBlank(rawJson)) { - std::cerr << "RedisLoader: empty payload rejected" << std::endl; - return 1; - } - - const std::string encoded = Base64::b64encode(rawJson); - - asio::io_context ioc; - auto conn = redis_util::makeRedisConnection(ioc, redisHost, redisPort); - - std::exception_ptr pushError; - - asio::co_spawn( - ioc, - pushOnce(conn, queueKey, encoded), - [&pushError](std::exception_ptr e) { - if (e) { - pushError = e; - } - }); - - ioc.run(); - - if (pushError) { - try { - std::rethrow_exception(pushError); - } catch (const boost::system::system_error& ex) { - std::cerr << "Redis LPUSH failed: " << ex.what() << std::endl; - } - return 3; - } - - return 0; -} - -int RedisLoader::loadPayloadBatch(const std::string& redisHost, - const int redisPort, - const std::string& queueKey, - const std::vector& rawJsonPayloads) { - if (rawJsonPayloads.empty()) { - return 0; // nothing to push - } - - std::vector encoded; - encoded.reserve(rawJsonPayloads.size()); - for (const std::string& rawJson : rawJsonPayloads) { - if (isBlank(rawJson)) { - std::cerr << "RedisLoader: empty payload rejected" << std::endl; - return 1; - } - encoded.push_back(Base64::b64encode(rawJson)); - } - - asio::io_context ioc; - auto conn = redis_util::makeRedisConnection(ioc, redisHost, redisPort); - - std::exception_ptr pushError; - - asio::co_spawn( - ioc, - pushManyOnce(conn, queueKey, std::move(encoded)), - [&pushError](const std::exception_ptr& e) { - if (e) { - pushError = e; - } - }); - - ioc.run(); - - if (pushError) { - try { - std::rethrow_exception(pushError); - } catch (const boost::system::system_error& ex) { - std::cerr << "Redis LPUSH failed: " << ex.what() << std::endl; - } - return 3; - } - - std::cout << "RedisLoader: LPUSH " << rawJsonPayloads.size() - << " payload(s) to " << queueKey << std::endl; - - return 0; -} diff --git a/source/shared/redis/producer/redisLoader.hpp b/source/shared/redis/producer/redisLoader.hpp deleted file mode 100644 index a43b16b..0000000 --- a/source/shared/redis/producer/redisLoader.hpp +++ /dev/null @@ -1,29 +0,0 @@ -// Backtesting Engine in C++ -// -// (c) 2026 Ryan McCaffery | https://mccaffers.com -// This code is licensed under MIT license (see LICENSE.txt for details) -// --------------------------------------- - -#pragma once - -#include -#include - -// LPUSH pairs with RedisRunner's RPOP so consumers observe FIFO ordering. -class RedisLoader { -public: - // LPUSHes a single Base64-encoded payload onto queueKey without assuming a - // payload type (run descriptor or strategy). - static int loadPayload(const std::string& redisHost, - int redisPort, - const std::string& queueKey, - const std::string& rawJson); - - // LPUSHes many payloads onto queueKey over one connection. Each payload is - // Base64-encoded; order is preserved (RedisRunner's RPOP then yields them - // in insertion order). - static int loadPayloadBatch(const std::string& redisHost, - int redisPort, - const std::string& queueKey, - const std::vector& rawJsonPayloads); -}; diff --git a/source/shared/tradingDefinitions.hpp b/source/shared/tradingDefinitions.hpp index b32107a..3a80223 100644 --- a/source/shared/tradingDefinitions.hpp +++ b/source/shared/tradingDefinitions.hpp @@ -7,8 +7,9 @@ #include "shared/tradingDefinitions/variables/ohlcVariables.hpp" #include "shared/tradingDefinitions/variables/ohlcRsiVariables.hpp" +#include "shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp" #include "shared/tradingDefinitions/variables/tradingVariables.hpp" #include "shared/tradingDefinitions/variables/strategyVariables.hpp" -#include "shared/tradingDefinitions/strategy.hpp" +#include "shared/tradingDefinitions/strategyConfig.hpp" #include "shared/tradingDefinitions/config/configuration.hpp" #include "shared/tradingDefinitions/config/runConfiguration.hpp" diff --git a/source/shared/tradingDefinitions/config/configuration.hpp b/source/shared/tradingDefinitions/config/configuration.hpp index 68d459d..e20e351 100644 --- a/source/shared/tradingDefinitions/config/configuration.hpp +++ b/source/shared/tradingDefinitions/config/configuration.hpp @@ -9,7 +9,7 @@ #include #include #include "shared/tradingDefinitions/config/runConfiguration.hpp" -#include "shared/tradingDefinitions/strategy.hpp" +#include "shared/tradingDefinitions/strategyConfig.hpp" #include "shared/utilities/decimalJson.hpp" // Intentionally a plain header, NOT a .cppm module. Every JSON-serializable @@ -28,7 +28,7 @@ struct Configuration { boost::decimal::decimal64_t MAX_LOSS_PERCENT{0}; int MAX_OPEN_TRADES{0}; bool REPORT_FAILURES{true}; - Strategy STRATEGY; + StrategyConfig STRATEGY; }; // Hand-written so the original fields stay strictly required while the risk diff --git a/source/shared/tradingDefinitions/strategy.hpp b/source/shared/tradingDefinitions/strategyConfig.hpp similarity index 90% rename from source/shared/tradingDefinitions/strategy.hpp rename to source/shared/tradingDefinitions/strategyConfig.hpp index e89f7b5..5f9263f 100644 --- a/source/shared/tradingDefinitions/strategy.hpp +++ b/source/shared/tradingDefinitions/strategyConfig.hpp @@ -14,13 +14,13 @@ namespace tradingDefinitions { -struct Strategy { +struct StrategyConfig { std::string UUID; TradingVariables TRADING_VARIABLES; std::vector OHLC_VARIABLES; StrategyVariables STRATEGY_VARIABLES; }; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Strategy, +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(StrategyConfig, UUID, TRADING_VARIABLES, OHLC_VARIABLES, diff --git a/source/shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp b/source/shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp new file mode 100644 index 0000000..6302567 --- /dev/null +++ b/source/shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp @@ -0,0 +1,19 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- + +#pragma once +#include + +namespace tradingDefinitions { +// Parameters for the OHLC breakout strategy. BUFFER_PIPS pads the +// closed-candle high/low breakout levels: price must clear the level by this +// many pips (converted to integer points via symbol_scale::get) before a +// signal fires — a noise filter against marginal pokes through the range. +struct OHLCBreakoutVariables { + int BUFFER_PIPS = 0; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(OHLCBreakoutVariables, BUFFER_PIPS); +} diff --git a/source/shared/tradingDefinitions/variables/strategyVariables.cpp b/source/shared/tradingDefinitions/variables/strategyVariables.cpp index 2c6e0bb..9780e60 100644 --- a/source/shared/tradingDefinitions/variables/strategyVariables.cpp +++ b/source/shared/tradingDefinitions/variables/strategyVariables.cpp @@ -13,6 +13,11 @@ void to_json(nlohmann::json& j, const StrategyVariables& s) { } else { j["OHLC_RSI_VARIABLES"] = nullptr; } + if (s.OHLC_BREAKOUT_VARIABLES) { + j["OHLC_BREAKOUT_VARIABLES"] = *s.OHLC_BREAKOUT_VARIABLES; + } else { + j["OHLC_BREAKOUT_VARIABLES"] = nullptr; + } } void from_json(const nlohmann::json& j, StrategyVariables& s) { @@ -21,6 +26,11 @@ void from_json(const nlohmann::json& j, StrategyVariables& s) { } else { s.OHLC_RSI_VARIABLES = std::nullopt; } + if (j.contains("OHLC_BREAKOUT_VARIABLES") && !j.at("OHLC_BREAKOUT_VARIABLES").is_null()) { + s.OHLC_BREAKOUT_VARIABLES = j.at("OHLC_BREAKOUT_VARIABLES").get(); + } else { + s.OHLC_BREAKOUT_VARIABLES = std::nullopt; + } } } diff --git a/source/shared/tradingDefinitions/variables/strategyVariables.hpp b/source/shared/tradingDefinitions/variables/strategyVariables.hpp index 6e6b6d1..cc655ba 100644 --- a/source/shared/tradingDefinitions/variables/strategyVariables.hpp +++ b/source/shared/tradingDefinitions/variables/strategyVariables.hpp @@ -8,12 +8,14 @@ #include #include #include "shared/tradingDefinitions/variables/ohlcRsiVariables.hpp" +#include "shared/tradingDefinitions/variables/ohlcBreakoutVariables.hpp" namespace tradingDefinitions { // Add in custom parameters struct StrategyVariables { std::optional OHLC_RSI_VARIABLES; + std::optional OHLC_BREAKOUT_VARIABLES; }; void to_json(nlohmann::json& j, const StrategyVariables& s); diff --git a/source/shared/utilities/backtestLog.hpp b/source/shared/utilities/backtestLog.hpp index f1bb3b2..f9313cb 100644 --- a/source/shared/utilities/backtestLog.hpp +++ b/source/shared/utilities/backtestLog.hpp @@ -8,8 +8,12 @@ #define UTILITIES_BACKTEST_LOG_HPP #include +#include +#include // std::snprintf +#include // POSIX gmtime_r, std::strftime #include #include +#include #include // Cross-cutting logging switches for backtest execution. @@ -43,9 +47,29 @@ inline std::mutex& errorMutex() { return m; } +// UTC wall-clock prefix, e.g. "[2026-06-28 15:04:20.785]", to millisecond +// precision. gmtime_r + strftime (rather than C++23 std::format) mirrors the +// classic style in elasticPublisher.cpp and keeps this header light enough to +// stay #includable from module global module fragments. +inline std::string timestamp() { + const auto now = std::chrono::system_clock::now(); + const std::time_t t = std::chrono::system_clock::to_time_t(now); + const long millis = std::chrono::duration_cast( + now.time_since_epoch()) + .count() % + 1000; + std::tm utc{}; + gmtime_r(&t, &utc); + char secs[20]; // "YYYY-MM-DD HH:MM:SS" + NUL + std::strftime(secs, sizeof(secs), "%Y-%m-%d %H:%M:%S", &utc); + char out[28]; + std::snprintf(out, sizeof(out), "[%s.%03ld]", secs, millis); + return out; +} + inline void error(std::string_view message) { std::scoped_lock lock(errorMutex()); - std::cerr << message << std::endl; + std::cerr << timestamp() << ' ' << message << std::endl; } } // namespace backtest_log diff --git a/source/shared/utilities/ema.cppm b/source/shared/utilities/ema.cppm new file mode 100644 index 0000000..331a2ab --- /dev/null +++ b/source/shared/utilities/ema.cppm @@ -0,0 +1,131 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// ema — integer Exponential Moving Average over scaled INT32 price points. +// +// Port of the C# EMA.Calculate(List prices, int period) helper, kept +// entirely in integer arithmetic so it can run on the per-tick hot path (the +// engine's rule: no software-emulated decimal and no floating point there). +// +// Design: the running EMA state lives in an int64 as Q16 fixed point (price +// points x 2^16). The 16 fractional bits are what prevent the classic integer +// EMA "sticking" bug — with whole-point state, small price changes would +// truncate to a zero increment and freeze the indicator; in Q16 the increment +// only vanishes below ~10^-4 of a point, far under the emitted resolution. +// +// The smoothing factor alpha = 2/(period+1) is precomputed once per call as a +// Q16 multiplier (round-to-nearest), so the per-element step is a single +// multiply plus shifts — no division instruction in the loop: +// +// emaQ += (alphaQ * (priceQ - emaQ) + 2^15) >> 16 +// +// This delta form needs one multiply (vs two for the alpha/beta convex form), +// and a constant series is exactly stationary by construction (delta = 0). +// Quantizing alpha to Q16 trades the previous exact rational step for speed: +// |alpha error| <= 2^-17, which perturbs the EMA by at most ~(2^-17/alpha) of +// the price-to-EMA gap — sub-milli-point for any realistic period. +// +// Rounding happens at two independent sites, both round-to-nearest via the +// add-half-then-shift idiom (floor-based, so correct for negative deltas too): +// once folding the Q32 product back into the Q16 state, and once emitting a +// whole-point value. The emit rounding never feeds back into the state, so +// nothing compounds; the state rounding contracts by (1-alpha) each step and +// stays bounded well below one point. +// +// Headroom: the largest scaled price (~4.5M for indices) is < 2^23, so the +// Q16 price is < 2^39 and the Q32 product < 2^55 — comfortably inside int64. +// alphaQ only rounds to zero (a frozen EMA) for periods above ~262k; the +// period here is bounded by the caller's bar-list length, orders of magnitude +// smaller. +export module ema; + +import std; // replaces , , , , + +namespace { + +// Q16 fixed point: 16 fractional bits, i.e. value_in_points * 65536. +constexpr int kFractionBits = 16; +constexpr std::int64_t kHalf = std::int64_t{1} << (kFractionBits - 1); + +constexpr std::int64_t toQ(std::int32_t points) { + return static_cast(points) << kFractionBits; +} + +// Round-to-nearest back to whole integer points. +constexpr std::int32_t roundToPoints(std::int64_t q) { + return static_cast((q + kHalf) >> kFractionBits); +} + +} // namespace + +export namespace ema { + +// C# analogue: List EMA.Calculate(List prices, int period). +// +// `prices` MUST be chronological (index 0 = oldest, last = newest) and holds +// the engine's non-negative scaled points. The output has the same length as +// the input: the first period-1 entries are 0 (not enough data yet), the entry +// at index period-1 is the SMA seed, and every entry after that follows the +// EMA recurrence. Values are rounded to the nearest integer point; the +// recurrence itself always iterates on the full Q16 state, never on the +// rounded outputs. +// +// This overload writes into a caller-owned buffer so per-tick callers can +// reuse capacity instead of allocating — C# analogy: refilling a List you +// keep around instead of newing one up per call. +// +// period < 1 throws std::invalid_argument. Fewer prices than the period yields +// all zeros (defined here; the C# original's behaviour was accidental). +void calculate(std::span prices, int period, + std::vector& out) { + if (period < 1) { + throw std::invalid_argument("ema::calculate: period must be >= 1"); + } + + // resize + targeted fills instead of clear/push_back: reused buffers keep + // their capacity, nothing is zeroed twice, and the loop writes by index. + out.resize(prices.size()); + const auto n = static_cast(period); + if (prices.size() < n) { + std::ranges::fill(out, 0); + return; + } + std::fill(out.begin(), out.begin() + static_cast(n) - 1, 0); + + // 1. Seed with the SMA of the first `period` prices, landing at index + // period-1 exactly like the C#. int64 sum: even absurd periods of maximal + // prices cannot overflow. Prices are non-negative, so the simple + // add-half-the-divisor rounding is enough. + std::int64_t sum = 0; + for (std::size_t i = 0; i < n; ++i) { + sum += prices[i]; + } + std::int64_t emaQ = ((sum << kFractionBits) + period / 2) / period; + out[n - 1] = roundToPoints(emaQ); + + // 2. Precompute alpha = 2/(period+1) as a Q16 multiplier, round-to-nearest. + // The only division left after this line is amortised across the call. + const std::int64_t alphaQ = + ((std::int64_t{2} << kFractionBits) + (period + 1) / 2) / (period + 1); + + // 3. Hot path: one multiply, two shifts, no division. + for (std::size_t i = n; i < prices.size(); ++i) { + const std::int64_t deltaQ = toQ(prices[i]) - emaQ; + emaQ += (alphaQ * deltaQ + kHalf) >> kFractionBits; + out[i] = roundToPoints(emaQ); + } +} + +// Convenience overload returning a fresh vector — matches the C# signature +// shape. Prefer the out-param overload on the per-tick path. +[[nodiscard]] std::vector calculate(std::span prices, + int period) { + std::vector out; + calculate(prices, period, out); + return out; +} + +} // namespace ema diff --git a/source/shared/utilities/env.cpp b/source/shared/utilities/env.cpp index b66eb4b..53903ee 100644 --- a/source/shared/utilities/env.cpp +++ b/source/shared/utilities/env.cpp @@ -6,8 +6,22 @@ #include "shared/utilities/env.hpp" +#include +#include +#include #include +#include +#include +#include #include +#include + +#if defined(__APPLE__) +#include +#define environ (*_NSGetEnviron()) +#else +extern char** environ; +#endif namespace env { @@ -16,4 +30,67 @@ std::string getOr(const char* name, std::string fallback) { return (val && *val) ? std::string{val} : std::move(fallback); } +namespace { + +// True when the variable name suggests its value is a secret that should not be +// printed verbatim. Matching is case-insensitive on the substrings below. +bool looksSensitive(std::string_view name) { + static constexpr std::array markers{ + std::string_view{"PASSWORD"}, std::string_view{"PASSWD"}, + std::string_view{"SECRET"}, std::string_view{"TOKEN"}, + std::string_view{"CREDENTIAL"}, std::string_view{"KEY"}, + std::string_view{"AUTH"}, + }; + + std::string upper(name); + std::transform(upper.begin(), upper.end(), upper.begin(), + [](unsigned char c) { return static_cast(std::toupper(c)); }); + + return std::ranges::any_of(markers, [&](std::string_view m) { + return upper.find(m) != std::string_view::npos; + }); +} + +// Masks a secret value, keeping a short prefix so it stays recognisable in logs. +// Short values are masked whole — 3 chars of a 5-char secret is most of it. +std::string maskValue(std::string_view value) { + constexpr std::size_t reveal = 3; + if (value.size() <= reveal * 2) return "********"; + return std::string{value.substr(0, reveal)} + "********"; +} + +} // namespace + +void printDiagnostics(int argc, const char* argv[]) { + std::cerr << "=== diagnostics ===\n"; + std::cerr << "argv:"; + for (int i = 0; i < argc; ++i) std::cerr << ' ' << argv[i]; + std::cerr << '\n'; + + // Collect and sort so the dump is stable run-to-run. + std::vector> vars; + for (char** ep = environ; ep && *ep; ++ep) { + std::string_view entry{*ep}; + const auto eq = entry.find('='); + const auto name = entry.substr(0, eq); + const auto value = eq == std::string_view::npos + ? std::string_view{} + : entry.substr(eq + 1); + vars.emplace_back(name, value); + } + std::ranges::sort(vars, {}, &std::pair::first); + + std::cerr << "env (" << vars.size() << " vars):\n"; + for (const auto& [name, value] : vars) { + std::cerr << " " << name << '='; + if (looksSensitive(name)) { + std::cerr << maskValue(value); + } else { + std::cerr << value; + } + std::cerr << '\n'; + } + std::cerr << "===================" << std::endl; +} + } // namespace env diff --git a/source/shared/utilities/env.hpp b/source/shared/utilities/env.hpp index e782a1b..5fc5c40 100644 --- a/source/shared/utilities/env.hpp +++ b/source/shared/utilities/env.hpp @@ -13,4 +13,9 @@ namespace env { // Returns $name, or `fallback` when the variable is unset or empty. std::string getOr(const char* name, std::string fallback); +// Prints process diagnostics (argv and the full environment) to stderr. +// Values of variables whose name looks sensitive (password/secret/token/key/ +// credential/auth) are masked so they don't leak into logs. +void printDiagnostics(int argc, const char* argv[]); + } // namespace env diff --git a/source/shared/utilities/jsonParser.cpp b/source/shared/utilities/jsonParser.cpp index 9d6c5c7..7322526 100644 --- a/source/shared/utilities/jsonParser.cpp +++ b/source/shared/utilities/jsonParser.cpp @@ -21,6 +21,6 @@ tradingDefinitions::RunConfiguration JsonParser::parseRunConfigurationFromBase64 return json::parse(Base64::b64decode(input)).get(); } -tradingDefinitions::Strategy JsonParser::parseStrategyFromBase64(const std::string& input) { - return json::parse(Base64::b64decode(input)).get(); +tradingDefinitions::StrategyConfig JsonParser::parseStrategyFromBase64(const std::string& input) { + return json::parse(Base64::b64decode(input)).get(); } diff --git a/source/shared/utilities/jsonParser.hpp b/source/shared/utilities/jsonParser.hpp index ea2798e..7ada8db 100644 --- a/source/shared/utilities/jsonParser.hpp +++ b/source/shared/utilities/jsonParser.hpp @@ -14,5 +14,5 @@ class JsonParser { public: static tradingDefinitions::Configuration parseConfigurationFromBase64(const std::string& input); static tradingDefinitions::RunConfiguration parseRunConfigurationFromBase64(const std::string& input); - static tradingDefinitions::Strategy parseStrategyFromBase64(const std::string& input); + static tradingDefinitions::StrategyConfig parseStrategyFromBase64(const std::string& input); }; diff --git a/source/shared/utilities/ohlcBuilder.cppm b/source/shared/utilities/ohlcBuilder.cppm new file mode 100644 index 0000000..5910ecb --- /dev/null +++ b/source/shared/utilities/ohlcBuilder.cppm @@ -0,0 +1,69 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// ohlcBuilder — incremental tick -> OHLC bar aggregation. +// +// Port of the C# CalculateOHLC(PriceObj, decimal price, TimeSpan duration, +// List) helper. Called once per tick, it appends to / updates a +// caller-owned vector of bars: the last element is always the in-progress bar, +// everything before it is complete. Shared so both the backtester (run) and a +// future live path can build bars the same way. +// +// Type mapping from the C# original: +// - decimal price -> std::int32_t scaled points (engine convention; +// the caller picks ask/bid/mid, exactly as the C# +// passed price separately from priceObj) +// - TimeSpan duration -> std::chrono::system_clock::duration +// (std::chrono::minutes etc. convert implicitly) +// - List -> std::vector&, mutated in place +// (the C# returned the same list it mutated) + +export module ohlcBuilder; + +import std; // replaces , , +import priceData; // PriceData tick (timestamp source) +import ohlcObject; // the bar record being built + +export namespace ohlc { + +void calculateOHLC(const PriceData& tick, std::int32_t price, + std::chrono::system_clock::duration duration, + std::vector& bars) { + // First tick ever: seed the initial bar from it. + if (bars.empty()) { + bars.push_back({.date = tick.timestamp, + .open = price, + .close = price, + .high = price, + .low = price}); + } + + // Strictly greater-than, matching the C# `diff > duration.TotalMinutes`: + // a tick landing exactly on the boundary still belongs to the open bar. + if (tick.timestamp - bars.back().date > duration) { + // C# behaviour, ported as-is: the completed bar's close is overwritten + // with the first price of the NEXT bucket, so consecutive bars join up. + bars.back().complete = true; + bars.back().close = price; + + bars.push_back({.date = tick.timestamp, + .open = price, + .close = price, + .high = price, + .low = price}); + } + + OhlcObject& bar = bars.back(); + if (price > bar.high) { + bar.high = price; + } + if (price < bar.low) { + bar.low = price; + } + bar.close = price; +} + +} // namespace ohlc diff --git a/source/shared/utilities/parameterSweep.hpp b/source/shared/utilities/parameterSweep.hpp deleted file mode 100644 index 1108a9b..0000000 --- a/source/shared/utilities/parameterSweep.hpp +++ /dev/null @@ -1,182 +0,0 @@ -// Backtesting Engine in C++ -// -// (c) 2026 Ryan McCaffery | https://mccaffers.com -// This code is licensed under MIT license (see LICENSE.txt for details) -// --------------------------------------- - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -// Parameter sweep generator: declare named numeric ranges and expand them into -// every combination (the cartesian product), so a strategy can be backtested -// across a grid of parameters in a single `load`. -// -// This is the C++ analogue of the C# StrategyParameterGenerator. C++ has no -// reflection, so a combination is returned keyed by parameter name -// (sweep::Combination) and the caller maps those values onto the strongly-typed -// Configuration fields (see source/commands/loadCommand.cpp). -namespace sweep { - -// Abstract numeric range, the analogue of C#'s IParameterRange. A range knows -// how to enumerate the discrete values a single parameter should take. -class ParameterRange { -public: - virtual ~ParameterRange() = default; - virtual std::vector generateValues() const = 0; -}; - -// Inclusive [start, end] stepped by `step`, e.g. {80, 20, 140} -> 80,100,120,140. -// The epsilon absorbs floating point drift so the upper bound stays inclusive. -class LinearRange final : public ParameterRange { -public: - LinearRange(double start, double step, double end) - : start_(start), step_(step), end_(end) {} - - std::vector generateValues() const override { - std::vector values; - if (step_ <= 0.0) { // defensive: a non-positive step cannot advance - values.push_back(start_); - return values; - } - for (double value = start_; value <= end_ + 1e-9; value += step_) { - values.push_back(value); - } - return values; - } - -private: - double start_; - double step_; - double end_; -}; - -// An explicit set of values, e.g. {1, 3, 5, 8}, analogue of C#'s -// ExplicitParameterRange / AddParameterList. -class ExplicitRange final : public ParameterRange { -public: - explicit ExplicitRange(std::vector values) - : values_(std::move(values)) {} - - std::vector generateValues() const override { return values_; } - -private: - std::vector values_; -}; - -// base * multiplier^i for `iterations` values, e.g. {10, 2, 5} -> 10,20,40,80,160. -class ExponentialRange final : public ParameterRange { -public: - ExponentialRange(double base, double multiplier, int iterations) - : base_(base), multiplier_(multiplier), iterations_(iterations) {} - - std::vector generateValues() const override { - std::vector values; - if (iterations_ > 0) { - values.reserve(static_cast(iterations_)); - } - double current = base_; - for (int i = 0; i < iterations_; ++i) { - values.push_back(current); - current *= multiplier_; - } - return values; - } - -private: - double base_; - double multiplier_; - int iterations_; -}; - -// One point in the parameter grid: parameter name -> chosen value. Values are -// stored as doubles; integer parameters round-trip exactly, so getInt() is safe. -class Combination { -public: - bool has(const std::string& name) const { return values_.count(name) > 0; } - - double get(const std::string& name) const { - const auto it = values_.find(name); - if (it == values_.end()) { - throw std::out_of_range( - "sweep::Combination: unknown parameter '" + name + "'"); - } - return it->second; - } - - int getInt(const std::string& name) const { - return static_cast(std::lround(get(name))); - } - - const std::map& values() const { return values_; } - -private: - friend class ParameterGenerator; - std::map values_; -}; - -// Holds the registered ranges and expands them into every combination. -class ParameterGenerator { -public: - // Register an arbitrary range under `name`. The convenience overloads below - // mirror the C# Add* helpers and cover the common cases. - ParameterGenerator& addRange(std::string name, - std::unique_ptr range) { - ranges_.emplace_back(std::move(name), std::move(range)); - return *this; - } - - ParameterGenerator& addRange(std::string name, double start, double step, - double end) { - return addRange(std::move(name), - std::make_unique(start, step, end)); - } - - ParameterGenerator& addList(std::string name, std::vector values) { - return addRange(std::move(name), - std::make_unique(std::move(values))); - } - - ParameterGenerator& addExponential(std::string name, double base, - double multiplier, int iterations) { - return addRange( - std::move(name), - std::make_unique(base, multiplier, iterations)); - } - - // Expand every registered range into the full cartesian product. Ranges are - // expanded in registration order, so the last-registered parameter varies - // fastest, keeping the output order stable and predictable. - std::vector generateAllCombinations() const { - std::vector result(1); // seed with one empty combination - for (const auto& [name, range] : ranges_) { - const std::vector values = range->generateValues(); - std::vector expanded; - expanded.reserve(result.size() * values.size()); - for (const Combination& base : result) { - for (const double value : values) { - Combination next = base; - next.values_[name] = value; - expanded.push_back(std::move(next)); - } - } - result = std::move(expanded); - } - return result; - } - - std::size_t parameterCount() const { return ranges_.size(); } - -private: - // A vector (not a map) preserves registration order for reproducible output. - std::vector>> ranges_; -}; - -} // namespace sweep diff --git a/source/shared/utilities/queueKeys.hpp b/source/shared/utilities/queueKeys.hpp index e98d1d2..686ce9b 100644 --- a/source/shared/utilities/queueKeys.hpp +++ b/source/shared/utilities/queueKeys.hpp @@ -8,15 +8,30 @@ #include -// Redis keys for the two-tier work queue. A sweep produces one run descriptor on -// RUN and N strategy payloads on STRATEGY_PREFIX + RUN_ID, all linked by RUN_ID. +// Redis keys for the work queue. A sweep produces, linked by RUN_ID: +// - one run descriptor on RUN; +// - one strategy payload per combination, each under its own +// STRATEGY_PAYLOAD_PREFIX + RUN_ID + ":" + UUID string key (spreading the +// sweep across many small keys instead of one unbounded list); +// - the STRATEGY_PREFIX + RUN_ID list, holding just those payload KEY NAMES. +// Workers RPOP a key name (each name goes to exactly one consumer) and GETDEL +// the payload (consumed exactly once, nothing left behind). namespace queue_keys { inline constexpr const char* RUN = "BACKTESTING_QUEUE_RUN"; inline constexpr const char* STRATEGY_PREFIX = "BACKTESTING_QUEUE_STRATEGY:"; +inline constexpr const char* STRATEGY_PAYLOAD_PREFIX = + "BACKTESTING_QUEUE_STRATEGY_PAYLOAD:"; inline std::string strategyKey(const std::string& runId) { return std::string(STRATEGY_PREFIX) + runId; } +// The RUN_ID segment groups a run's payload keys under one SCAN-able pattern +// (orphan cleanup); the UUID is the strategy's own, minted by the factory. +inline std::string strategyPayloadKey(const std::string& runId, + const std::string& strategyUuid) { + return std::string(STRATEGY_PAYLOAD_PREFIX) + runId + ":" + strategyUuid; +} + } // namespace queue_keys diff --git a/source/shared/utilities/threadPool.hpp b/source/shared/utilities/threadPool.hpp index f47da35..43e2cf9 100644 --- a/source/shared/utilities/threadPool.hpp +++ b/source/shared/utilities/threadPool.hpp @@ -7,8 +7,10 @@ #ifndef UTILITIES_THREAD_POOL_HPP #define UTILITIES_THREAD_POOL_HPP +#include #include #include +#include #include #include #include @@ -36,7 +38,12 @@ // exception thrown by any task is captured and surfaced via takeError(). class ThreadPool { public: - explicit ThreadPool(std::size_t threads) : capacity_(threads * 2) { + // `inFlightGauge`, if non-null, is updated (under the lock) every time the + // in-flight count changes, so an observer — e.g. a shared-memory monitor — + // sees the live queued+executing total without coupling the pool to it. + explicit ThreadPool(std::size_t threads, + std::atomic* inFlightGauge = nullptr) + : capacity_(threads * 2), inFlightGauge_(inFlightGauge) { workers_.reserve(threads); for (std::size_t i = 0; i < threads; ++i) { workers_.emplace_back([this](std::stop_token st) { workerLoop(st); }); @@ -52,6 +59,7 @@ class ThreadPool { slotFree_.wait(lock, [this] { return inFlight_ < capacity_; }); tasks_.push(std::move(task)); ++inFlight_; + publishInFlight(); workAvailable_.notify_one(); } @@ -68,6 +76,15 @@ class ThreadPool { } private: + // Mirror the current in-flight count to the observer gauge, if one was given. + // Always called with mutex_ held. + void publishInFlight() { + if (inFlightGauge_) { + inFlightGauge_->store(static_cast(inFlight_), + std::memory_order_release); + } + } + void workerLoop(std::stop_token st) { for (;;) { std::function task; @@ -94,6 +111,7 @@ class ThreadPool { firstError_ = err; } --inFlight_; + publishInFlight(); if (inFlight_ == 0) { idle_.notify_all(); } @@ -109,6 +127,7 @@ class ThreadPool { std::queue> tasks_; std::size_t inFlight_ = 0; std::size_t capacity_; + std::atomic* inFlightGauge_ = nullptr; std::exception_ptr firstError_; // Declared last so the jthreads stop and join before the synchronisation diff --git a/source/strategies/ohlcBreakout/ohlcBreakoutStrategy.cppm b/source/strategies/ohlcBreakout/ohlcBreakoutStrategy.cppm new file mode 100644 index 0000000..7175d0b --- /dev/null +++ b/source/strategies/ohlcBreakout/ohlcBreakoutStrategy.cppm @@ -0,0 +1,203 @@ +// Backtesting Engine in C++ +// +// (c) 2026 Ryan McCaffery | https://mccaffers.com +// This code is licensed under MIT license (see LICENSE.txt for details) +// --------------------------------------- +// +// OhlcBreakoutStrategy — range breakout with an EMA trend filter. +// +// Two OHLC timeframes are built per symbol from the tick stream (both from the +// ask, matching the C# original): +// +// OHLC_VARIABLES[0] — the breakout timeframe. The highest high / lowest low +// of its CLOSED candles (the in-progress bar is excluded) +// define the range, padded by BUFFER_PIPS. +// OHLC_VARIABLES[1] — the trend timeframe. An EMA over its closes (period = +// half the candle count) is the macro trend filter. +// +// Entry: bid breaks above the range top in a macro uptrend -> LONG; ask breaks +// below the range bottom in a macro downtrend -> SHORT. Exits stay central +// (SL/TP pip distances enforced by Operations), so during() only builds bars. +// +// Two structural notes that differ from the C# framework: +// - Bars are built in during(), not decide(): the run loop skips decide() for +// a symbol while it has an open trade, but during() runs on every tick, so +// the bar history never gaps. decide() therefore sees bars as of the +// previous tick — immaterial, because the range comes from closed candles +// only and is compared against the current tick's bid/ask. +// - One strategy instance sees every symbol's ticks interleaved by timestamp +// (SYMBOLS = "EURUSD,AUDUSD"), so all bar state is per-symbol, keyed by +// tick.symbol — the C# "persistent list per instrument" requirement. + +module; + +#include "shared/tradingDefinitions/strategyConfig.hpp" + +export module ohlcBreakoutStrategy; + +import std; // replaces , , , , + // , , , , + // , +import strategy; // IStrategy base class +import priceData; // PriceData +import trade; // Direction +import tradeManager; // TradeManager +import ohlcObject; // OhlcObject bar record +import ohlcBuilder; // ohlc::calculateOHLC +import ema; // ema::calculate (integer EMA) +import symbolScale; // symbol_scale::get — points-per-pip for the buffer + +export class OhlcBreakoutStrategy : public IStrategy { +public: + // Validates the config up front and throws std::invalid_argument on a + // malformed one (missing timeframes / BUFFER_PIPS) — a misconfigured run + // should die loudly at construction, not trade silently wrong. + explicit OhlcBreakoutStrategy(const tradingDefinitions::StrategyConfig& strategyConfig); + + std::optional decide(const PriceData& tick) override; + + // Builds the per-symbol bars every tick. TradeManager is unused — exits + // are driven centrally by the configured SL/TP distances. + void during(const PriceData& price, TradeManager& tradeManager) override; + +private: + struct SymbolState { + std::vector breakoutBars; + std::vector trendBars; + }; + + // Transparent hasher (same pattern as TradeManager::activeTrades) so the + // per-tick find() takes a string_view and never allocates a temporary key. + struct SymbolHash { + using is_transparent = void; + std::size_t operator()(std::string_view symbol) const noexcept { + return std::hash{}(symbol); + } + }; + + tradingDefinitions::OHLCVariables breakoutCfg; + tradingDefinitions::OHLCVariables trendCfg; + std::int32_t bufferPips; + + std::unordered_map> stateBySymbol; + + // Scratch buffers reused every decide() — cleared, never shrunk, so the + // per-tick path stops allocating once their capacity settles. + std::vector closesScratch; + std::vector emaScratch; +}; + +namespace { + +// Feed the tick into one timeframe's bars, then trim the history from the +// front to OHLC_COUNT — a rolling window, oldest bars dropped first. The last +// element is always the in-progress bar (see ohlcBuilder). +void updateBars(const PriceData& tick, const tradingDefinitions::OHLCVariables& cfg, + std::vector& bars) { + ohlc::calculateOHLC(tick, tick.ask, std::chrono::minutes{cfg.OHLC_MINUTES}, bars); + const auto cap = static_cast(cfg.OHLC_COUNT); + if (bars.size() > cap) { + bars.erase(bars.begin(), bars.end() - static_cast(cap)); + } +} + +} // namespace + +OhlcBreakoutStrategy::OhlcBreakoutStrategy( + const tradingDefinitions::StrategyConfig& strategyConfig) { + const auto& ohlcVars = strategyConfig.OHLC_VARIABLES; + if (ohlcVars.size() < 2) { + throw std::invalid_argument( + "OhlcBreakoutStrategy: OHLC_VARIABLES needs two entries " + "(breakout timeframe, trend timeframe)"); + } + breakoutCfg = ohlcVars[0]; + trendCfg = ohlcVars[1]; + for (const auto& cfg : {breakoutCfg, trendCfg}) { + // COUNT >= 2 so the breakout list always has a closed candle besides + // the in-progress one, and the trend EMA period (count/2) is >= 1. + if (cfg.OHLC_COUNT < 2 || cfg.OHLC_MINUTES < 1) { + throw std::invalid_argument( + "OhlcBreakoutStrategy: OHLC_COUNT must be >= 2 and OHLC_MINUTES >= 1"); + } + } + const auto& breakoutVars = strategyConfig.STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES; + if (!breakoutVars) { + throw std::invalid_argument( + "OhlcBreakoutStrategy: STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES is required " + "(BUFFER_PIPS)"); + } + bufferPips = breakoutVars->BUFFER_PIPS; +} + +void OhlcBreakoutStrategy::during(const PriceData& price, TradeManager& /*tradeManager*/) { + // This symbol's entry in stateBySymbol (its two bar histories), inserted + // empty on the symbol's first tick. Heterogeneous find first: only that + // first tick pays for the std::string key construction. + auto stateIt = stateBySymbol.find(std::string_view{price.symbol}); + if (stateIt == stateBySymbol.end()) { + stateIt = stateBySymbol.emplace(price.symbol, SymbolState{}).first; + } + updateBars(price, breakoutCfg, stateIt->second.breakoutBars); + updateBars(price, trendCfg, stateIt->second.trendBars); +} + +std::optional OhlcBreakoutStrategy::decide(const PriceData& tick) { + // This symbol's bar histories; absent means during() has not seen a tick + // for the symbol yet, so there is nothing to decide on. + const auto stateIt = stateBySymbol.find(std::string_view{tick.symbol}); + if (stateIt == stateBySymbol.end()) { + return std::nullopt; + } + const SymbolState& state = stateIt->second; + + // Warm-up gate: no signals until both timeframes have a full window + // (C#: `if (ohlcList.Count < totalOHLCCount) return`). + if (state.breakoutBars.size() < static_cast(breakoutCfg.OHLC_COUNT) || + state.trendBars.size() < static_cast(trendCfg.OHLC_COUNT)) { + return std::nullopt; + } + + // No upper-bound check to pair with the gate above: updateBars trims each + // history to OHLC_COUNT (oldest bars dropped from the front) on every + // tick, so past the gate both windows hold exactly OHLC_COUNT bars. + + // --- 1. THE BREAKOUT LOGIC --- + // Range from the CLOSED breakout candles (drop the in-progress last bar — + // C#: closedCandles = Take(Count - 1)), padded by the pip buffer. Pips + // scale UP to points here (pips * pointsPerPip); the C# divided because + // its prices were raw decimals. An unknown symbol returns scale 0, which + // just disables the buffer rather than corrupting the levels. + std::int32_t highestHigh = std::numeric_limits::min(); + std::int32_t lowestLow = std::numeric_limits::max(); + for (auto bar = state.breakoutBars.begin(); bar != state.breakoutBars.end() - 1; ++bar) { + highestHigh = std::max(highestHigh, bar->high); + lowestLow = std::min(lowestLow, bar->low); + } + const std::int32_t bufferPoints = bufferPips * symbol_scale::get(tick.symbol); + + // --- 2. THE TREND FILTER LOGIC --- + // EMA over the trend timeframe's closes (chronological, in-progress bar + // included, like the C#). Period = half the window (C#: count * 0.5m, + // truncated). Both the close and the EMA are read at the last index so + // today's price is compared against today's moving average. + closesScratch.clear(); + for (const auto& bar : state.trendBars) { + closesScratch.push_back(bar.close); + } + const int emaPeriod = static_cast(closesScratch.size() / 2); + ema::calculate(closesScratch, emaPeriod, emaScratch); + const std::int32_t currentClose = closesScratch.back(); + const std::int32_t currentEma = emaScratch.back(); + + // --- 3. EXECUTION LOGIC --- + // Re-fires while the condition holds once the position is closed; the run + // loop's one-trade-per-symbol gate prevents stacking entries. + if (tick.bid > highestHigh + bufferPoints && currentClose > currentEma) { + return Direction::LONG; + } + if (tick.ask < lowestLow - bufferPoints && currentClose < currentEma) { + return Direction::SHORT; + } + return std::nullopt; +} diff --git a/source/strategies/randomStrategy.cppm b/source/strategies/randomStrategy/randomStrategy.cppm similarity index 95% rename from source/strategies/randomStrategy.cppm rename to source/strategies/randomStrategy/randomStrategy.cppm index 983c074..2c90b90 100644 --- a/source/strategies/randomStrategy.cppm +++ b/source/strategies/randomStrategy/randomStrategy.cppm @@ -6,7 +6,7 @@ module; -#include "shared/tradingDefinitions/strategy.hpp" +#include "shared/tradingDefinitions/strategyConfig.hpp" export module randomStrategy; @@ -39,7 +39,7 @@ import tradeManager; // TradeManager // inheritance, so the `public` keyword has no analogue there. export class RandomStrategy : public IStrategy { public: - explicit RandomStrategy(const tradingDefinitions::Strategy& strategyConfig); + explicit RandomStrategy(const tradingDefinitions::StrategyConfig& strategyConfig); // Returns `std::nullopt` to mean "no signal — don't trade". The // random strategy always returns a direction, but the interface @@ -62,7 +62,7 @@ public: TradeManager& tradeManager) override; private: - tradingDefinitions::Strategy config; + tradingDefinitions::StrategyConfig config; std::mt19937 rng; std::bernoulli_distribution coin; // fair coin flip for entry direction std::bernoulli_distribution closeProb; // per-tick probability of closing all trades @@ -73,7 +73,7 @@ private: // trade.hpp. The C# equivalent would be field initialisers plus a // constructor body, but C++ prefers the initialiser list because it // constructs members directly rather than default-then-assign. -RandomStrategy::RandomStrategy(const tradingDefinitions::Strategy& strategyConfig) +RandomStrategy::RandomStrategy(const tradingDefinitions::StrategyConfig& strategyConfig) : config(strategyConfig), rng(std::random_device{}()), // seed once from the OS entropy source coin(0.5), diff --git a/source/strategies/strategy.cppm b/source/strategies/strategy.cppm index b11f761..f078780 100644 --- a/source/strategies/strategy.cppm +++ b/source/strategies/strategy.cppm @@ -47,8 +47,9 @@ public: // Called every tick. Receives the TradeManager by mutable // reference so strategies can both inspect open positions - // (`tradeManager.getActiveTrades()`) and act on them - // (`closeTrade`, future scale/adjust hooks). A reference signals + // (`tradeManager.getActiveTrades()`, keyed by symbol) and act on + // them (`closeTrade(symbol, ...)`, future scale/adjust hooks). A + // reference signals // "borrow, don't own", the strategy must not delete it. The C# // analogue is just passing the manager as a parameter; C# has no // distinction between reference and pointer so the by-ref nature diff --git a/source/strategies/strategyErrors.cppm b/source/strategies/strategyErrors.cppm index bf31cd3..83a27a6 100644 --- a/source/strategies/strategyErrors.cppm +++ b/source/strategies/strategyErrors.cppm @@ -19,7 +19,7 @@ public: : std::runtime_error("Unknown strategy: '" + name + "'"), name_(std::move(name)) {} - const std::string& name() const noexcept { return name_; } + [[nodiscard]] const std::string& name() const noexcept { return name_; } private: std::string name_; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e5181a4..bba6641 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,6 +5,9 @@ add_executable(unit_tests symbolScale.cpp jsonParser.cpp + ohlc.cpp + ema.cpp + ohlcBreakout.cpp db.cpp sweep.cpp tradeManager.cpp diff --git a/tests/ema.cpp b/tests/ema.cpp new file mode 100644 index 0000000..5c81156 --- /dev/null +++ b/tests/ema.cpp @@ -0,0 +1,147 @@ +#include + +#include +#include +#include +#include + +import ema; + +TEST_CASE("ema::calculate matches the C# reference shape", "[ema]") { + SECTION("pads with zeros, seeds with the SMA, then follows the recurrence") { + // period 3: multiplier 2/(3+1) = 0.5, so each step is the simple + // midpoint of the new price and the previous EMA — exact integers. + const std::vector prices{10, 20, 30, 40, 50}; + const auto out = ema::calculate(prices, 3); + + REQUIRE(out.size() == prices.size()); + CHECK(out[0] == 0); + CHECK(out[1] == 0); + CHECK(out[2] == 20); // SMA(10, 20, 30) + CHECK(out[3] == 30); // 0.5*40 + 0.5*20 + CHECK(out[4] == 40); // 0.5*50 + 0.5*30 + } + + SECTION("fractional values round to the nearest point") { + // period 2: seed SMA(10, 11) = 10.5 -> 11; then + // ema = (2*12 + 1*10.5) / 3 = 11.5 -> 12. + const std::vector prices{10, 11, 12}; + const auto out = ema::calculate(prices, 2); + + REQUIRE(out.size() == 3); + CHECK(out[0] == 0); + CHECK(out[1] == 11); + CHECK(out[2] == 12); + } + + SECTION("a constant series settles on the constant") { + const std::vector prices(6, 110001); + const auto out = ema::calculate(prices, 3); + + REQUIRE(out.size() == 6); + CHECK(out[0] == 0); + CHECK(out[1] == 0); + for (std::size_t i = 2; i < out.size(); ++i) { + CHECK(out[i] == 110001); + } + } + + SECTION("largest scaled prices don't overflow the Q16 state") { + // ~4.5M points is the biggest value the engine stores (indices). + const std::vector prices(4, 4'500'000); + const auto out = ema::calculate(prices, 3); + + REQUIRE(out.size() == 4); + CHECK(out[2] == 4'500'000); + CHECK(out[3] == 4'500'000); + } +} + +TEST_CASE("ema::calculate stays within one point of exact arithmetic", "[ema]") { + // Reference EMA in double (fine in a test; the production path is what + // must stay integer). The integer Q16 recurrence should track it to <= 1 + // point at every index, even over a long, drifting series. + std::vector prices; + for (int i = 0; i < 60; ++i) { + prices.push_back(110000 + i * 7 + (i % 5) * 13); + } + const int period = 9; + const auto out = ema::calculate(prices, period); + REQUIRE(out.size() == prices.size()); + + double emaD = 0.0; + for (int i = 0; i < period; ++i) { + emaD += prices[static_cast(i)]; + } + emaD /= period; + CHECK(std::llabs(out[period - 1] - std::llround(emaD)) <= 1); + + for (std::size_t i = static_cast(period); i < prices.size(); ++i) { + emaD = (2.0 * prices[i] + (period - 1) * emaD) / (period + 1); + INFO("index " << i); + CHECK(std::llabs(out[i] - std::llround(emaD)) <= 1); + } +} + +TEST_CASE("ema::calculate never freezes on small moves (no integer sticking)", "[ema]") { + SECTION("a 1-point-per-step ramp keeps the EMA moving") { + // With whole-point state, alpha * 1 point would truncate to zero and + // the indicator would freeze at the seed. The Q16 state must keep + // climbing the whole way. + std::vector prices; + for (int i = 0; i < 40; ++i) { + prices.push_back(110000 + i); + } + const int period = 9; + const auto out = ema::calculate(prices, period); + + for (std::size_t i = static_cast(period); i < out.size(); ++i) { + INFO("index " << i); + CHECK(out[i] >= out[i - 1]); + } + CHECK(out.back() > out[static_cast(period) - 1]); + } + + SECTION("after the input flattens, the EMA converges onto the constant") { + std::vector prices{110100, 110080, 110060, 110040, 110020}; + for (int i = 0; i < 30; ++i) { + prices.push_back(110000); + } + const auto out = ema::calculate(prices, 5); + + CHECK(out.back() == 110000); + // ...and holds there rather than oscillating or drifting. + CHECK(out[out.size() - 2] == 110000); + CHECK(out[out.size() - 3] == 110000); + } +} + +TEST_CASE("ema::calculate edge cases", "[ema]") { + SECTION("fewer prices than the period yields all zeros") { + const std::vector prices{100, 200}; + const auto out = ema::calculate(prices, 5); + CHECK(out == std::vector{0, 0}); + } + + SECTION("empty input yields empty output") { + CHECK(ema::calculate(std::vector{}, 3).empty()); + } + + SECTION("period below 1 throws") { + const std::vector prices{1, 2, 3}; + CHECK_THROWS_AS(ema::calculate(prices, 0), std::invalid_argument); + CHECK_THROWS_AS(ema::calculate(prices, -2), std::invalid_argument); + } + + SECTION("the out-param overload reuses the buffer and agrees with the returning one") { + const std::vector first{10, 20, 30, 40, 50}; + const std::vector second{5, 6, 7}; + + std::vector out; + ema::calculate(first, 3, out); + CHECK(out == ema::calculate(first, 3)); + + ema::calculate(second, 2, out); // same buffer, shorter input + CHECK(out == ema::calculate(second, 2)); + } +} diff --git a/tests/ohlc.cpp b/tests/ohlc.cpp new file mode 100644 index 0000000..60add33 --- /dev/null +++ b/tests/ohlc.cpp @@ -0,0 +1,149 @@ +#include + +#include +#include +#include +#include + +import ohlcBuilder; +import ohlcObject; +import priceData; + +namespace { + +using std::chrono::minutes; +using std::chrono::seconds; + +// Fixed simulation start; only the offsets between ticks matter to the builder. +const std::chrono::system_clock::time_point t0 = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; + +// The builder reads only the timestamp from the tick — the price is passed +// separately (caller picks ask/bid/mid), so ask/bid here are placeholders. +PriceData tickAt(std::chrono::system_clock::duration offset) { + return PriceData(110002, 110001, t0 + offset, "EURUSD"); +} + +} // namespace + +TEST_CASE("OhlcObject defaults mirror the C# sentinels", "[ohlc]") { + const OhlcObject bar{}; + CHECK(bar.date == std::chrono::system_clock::time_point{}); + CHECK(bar.open == 0); + CHECK(bar.close == 0); + CHECK(bar.high == std::numeric_limits::min()); + CHECK(bar.low == std::numeric_limits::max()); + CHECK_FALSE(bar.complete); +} + +TEST_CASE("first tick seeds a single in-progress bar", "[ohlc]") { + std::vector bars; + + ohlc::calculateOHLC(tickAt(minutes{0}), 110001, minutes{5}, bars); + + REQUIRE(bars.size() == 1); + CHECK(bars[0].date == t0); + CHECK(bars[0].open == 110001); + CHECK(bars[0].high == 110001); + CHECK(bars[0].low == 110001); + CHECK(bars[0].close == 110001); + CHECK_FALSE(bars[0].complete); +} + +TEST_CASE("ticks within the duration update the open bar in place", "[ohlc]") { + std::vector bars; + ohlc::calculateOHLC(tickAt(minutes{0}), 110001, minutes{5}, bars); + + SECTION("higher price raises high and moves close") { + ohlc::calculateOHLC(tickAt(minutes{1}), 110010, minutes{5}, bars); + + REQUIRE(bars.size() == 1); + CHECK(bars[0].open == 110001); + CHECK(bars[0].high == 110010); + CHECK(bars[0].low == 110001); + CHECK(bars[0].close == 110010); + CHECK_FALSE(bars[0].complete); + } + + SECTION("lower price drops low and moves close") { + ohlc::calculateOHLC(tickAt(minutes{2}), 109990, minutes{5}, bars); + + REQUIRE(bars.size() == 1); + CHECK(bars[0].high == 110001); + CHECK(bars[0].low == 109990); + CHECK(bars[0].close == 109990); + } + + SECTION("open and date never move after the seed tick") { + ohlc::calculateOHLC(tickAt(minutes{1}), 110020, minutes{5}, bars); + ohlc::calculateOHLC(tickAt(minutes{3}), 109980, minutes{5}, bars); + + REQUIRE(bars.size() == 1); + CHECK(bars[0].date == t0); + CHECK(bars[0].open == 110001); + CHECK(bars[0].high == 110020); + CHECK(bars[0].low == 109980); + CHECK(bars[0].close == 109980); + } +} + +TEST_CASE("a tick exactly on the boundary stays in the open bar", "[ohlc]") { + std::vector bars; + ohlc::calculateOHLC(tickAt(minutes{0}), 110001, minutes{5}, bars); + + // Rollover requires strictly greater than the duration, matching the C#. + ohlc::calculateOHLC(tickAt(minutes{5}), 110005, minutes{5}, bars); + + REQUIRE(bars.size() == 1); + CHECK(bars[0].close == 110005); + CHECK_FALSE(bars[0].complete); +} + +TEST_CASE("a tick past the duration completes the bar and opens a new one", "[ohlc]") { + std::vector bars; + ohlc::calculateOHLC(tickAt(minutes{0}), 110001, minutes{5}, bars); + ohlc::calculateOHLC(tickAt(minutes{2}), 110010, minutes{5}, bars); + + ohlc::calculateOHLC(tickAt(minutes{5} + seconds{1}), 110020, minutes{5}, bars); + + REQUIRE(bars.size() == 2); + + // Completed bar keeps its extremes; its close is overwritten with the new + // bucket's first price (ported C# behaviour, bars join up). + CHECK(bars[0].complete); + CHECK(bars[0].open == 110001); + CHECK(bars[0].high == 110010); + CHECK(bars[0].low == 110001); + CHECK(bars[0].close == 110020); + + // New bar is seeded entirely from the rollover tick. + CHECK_FALSE(bars[1].complete); + CHECK(bars[1].date == t0 + minutes{5} + seconds{1}); + CHECK(bars[1].open == 110020); + CHECK(bars[1].high == 110020); + CHECK(bars[1].low == 110020); + CHECK(bars[1].close == 110020); +} + +TEST_CASE("a stream spanning several buckets yields one bar per bucket", "[ohlc]") { + std::vector bars; + + ohlc::calculateOHLC(tickAt(minutes{0}), 110001, minutes{5}, bars); + ohlc::calculateOHLC(tickAt(minutes{6}), 110010, minutes{5}, bars); + ohlc::calculateOHLC(tickAt(minutes{12}), 110020, minutes{5}, bars); + ohlc::calculateOHLC(tickAt(minutes{13}), 110015, minutes{5}, bars); + + REQUIRE(bars.size() == 3); + CHECK(bars[0].complete); + CHECK(bars[1].complete); + CHECK_FALSE(bars[2].complete); + + CHECK(bars[0].date == t0); + CHECK(bars[1].date == t0 + minutes{6}); + CHECK(bars[2].date == t0 + minutes{12}); + + CHECK(bars[2].open == 110020); + CHECK(bars[2].high == 110020); + CHECK(bars[2].low == 110015); + CHECK(bars[2].close == 110015); +} diff --git a/tests/ohlcBreakout.cpp b/tests/ohlcBreakout.cpp new file mode 100644 index 0000000..b01626c --- /dev/null +++ b/tests/ohlcBreakout.cpp @@ -0,0 +1,210 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include "shared/tradingDefinitions/strategyConfig.hpp" + +import ohlcBreakoutStrategy; +import priceData; +import trade; +import tradeManager; + +namespace { + +using std::chrono::minutes; +using std::chrono::seconds; + +const std::chrono::system_clock::time_point t0 = + std::chrono::sys_days{std::chrono::year{2026} / 1 / 5} + std::chrono::hours{9}; + +// Breakout timeframe: 3 one-minute candles; trend timeframe: 4 one-minute +// candles (EMA period = 4/2 = 2). Ticks in these tests are spaced 2 minutes +// apart so every tick rolls a fresh bar — warm-up completes after 4 ticks and +// tick 5 is the first that can signal. +tradingDefinitions::StrategyConfig makeConfig(int bufferPips) { + tradingDefinitions::StrategyConfig config; + config.UUID = "test-ohlc-breakout"; + config.TRADING_VARIABLES.STRATEGY = "OhlcBreakoutStrategy"; + config.TRADING_VARIABLES.STOP_DISTANCE_IN_PIPS = 10; + config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_PIPS = 10; + config.TRADING_VARIABLES.TRADING_SIZE = 1; + config.OHLC_VARIABLES = { + tradingDefinitions::OHLCVariables{.OHLC_COUNT = 3, .OHLC_MINUTES = 1}, // breakout + tradingDefinitions::OHLCVariables{.OHLC_COUNT = 4, .OHLC_MINUTES = 1}, // trend + }; + config.STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES = + tradingDefinitions::OHLCBreakoutVariables{.BUFFER_PIPS = bufferPips}; + return config; +} + +PriceData tickAt(std::chrono::system_clock::duration offset, std::int32_t ask, + std::int32_t bid, const std::string& symbol = "EURUSD") { + return PriceData(ask, bid, t0 + offset, symbol); +} + +// One tick in run-loop order: decide first, then during (runTicks calls +// decide before the management hook on each tick). +std::optional step(OhlcBreakoutStrategy& strategy, TradeManager& tm, + const PriceData& tick) { + const auto signal = strategy.decide(tick); + strategy.during(tick, tm); + return signal; +} + +// Feeds the four warm-up ticks (each starts its own bar) and asserts none of +// them may signal. Bars are built from the ask; bid is ask minus a 2-point +// spread and irrelevant until the signal tick. +void warmUp(OhlcBreakoutStrategy& strategy, TradeManager& tm, + const std::vector& asks, + const std::string& symbol = "EURUSD") { + for (std::size_t i = 0; i < asks.size(); ++i) { + const auto tick = tickAt(minutes{2 * static_cast(i)}, asks[i], asks[i] - 2, symbol); + CHECK_FALSE(step(strategy, tm, tick).has_value()); + } +} + +// Rising EURUSD asks: bars close at 110010/110020/110030, closed breakout +// highs top out at 110020, and the trend EMA sits below the last close +// (macro uptrend). +const std::vector kRisingAsks{110000, 110010, 110020, 110030}; + +// Falling series: closed breakout candles span 110060..110080 and the trend +// EMA sits above the last close (macro downtrend). +const std::vector kFallingAsks{110100, 110080, 110060, 110040}; + +} // namespace + +TEST_CASE("OhlcBreakoutStrategy signals LONG on a buffered breakout in an uptrend", + "[ohlcBreakout]") { + OhlcBreakoutStrategy strategy{makeConfig(2)}; // buffer = 2 pips = 20 points + TradeManager tm; + warmUp(strategy, tm, kRisingAsks); + + // Closed-candle highest high is 110020, buffer 20 -> level 110040. + const auto signal = step(strategy, tm, tickAt(minutes{8}, 110100, 110090)); + CHECK(signal == Direction::LONG); +} + +TEST_CASE("OhlcBreakoutStrategy respects the pip buffer", "[ohlcBreakout]") { + OhlcBreakoutStrategy strategy{makeConfig(2)}; + TradeManager tm; + warmUp(strategy, tm, kRisingAsks); + + SECTION("above the high but inside the buffer: no signal") { + // bid 110035 clears the 110020 high but not the 110040 buffered level. + CHECK_FALSE(step(strategy, tm, tickAt(minutes{8}, 110045, 110035)).has_value()); + } + + SECTION("exactly on the buffered level: no signal (strictly greater)") { + CHECK_FALSE(step(strategy, tm, tickAt(minutes{8}, 110050, 110040)).has_value()); + } + + SECTION("one point beyond the buffered level: LONG") { + CHECK(step(strategy, tm, tickAt(minutes{8}, 110051, 110041)) == Direction::LONG); + } +} + +TEST_CASE("OhlcBreakoutStrategy trend filter blocks counter-trend breakouts", + "[ohlcBreakout]") { + OhlcBreakoutStrategy strategy{makeConfig(2)}; + TradeManager tm; + warmUp(strategy, tm, kFallingAsks); + + // bid 110200 is far above the closed-candle high (110080) + buffer, but + // the macro trend is down (last close 110040 < EMA ~110043) — no LONG. + CHECK_FALSE(step(strategy, tm, tickAt(minutes{8}, 110210, 110200)).has_value()); +} + +TEST_CASE("OhlcBreakoutStrategy signals SHORT on a buffered breakdown in a downtrend", + "[ohlcBreakout]") { + OhlcBreakoutStrategy strategy{makeConfig(2)}; + TradeManager tm; + warmUp(strategy, tm, kFallingAsks); + + // Closed-candle lowest low is 110060, buffer 20 -> level 110040. + SECTION("ask below the buffered low: SHORT") { + CHECK(step(strategy, tm, tickAt(minutes{8}, 110030, 110020)) == Direction::SHORT); + } + + SECTION("exactly on the buffered level: no signal (strictly less)") { + CHECK_FALSE(step(strategy, tm, tickAt(minutes{8}, 110040, 110030)).has_value()); + } +} + +TEST_CASE("OhlcBreakoutStrategy keeps per-symbol state isolated", "[ohlcBreakout]") { + OhlcBreakoutStrategy strategy{makeConfig(2)}; + TradeManager tm; + + // Interleave a rising EURUSD with a falling AUDUSD at a very different + // price level. If either symbol's ticks leaked into the other's bars, the + // ranges and trend filters below would be wildly wrong. + const std::vector audAsks{65100, 65080, 65060, 65040}; + for (std::size_t i = 0; i < 4; ++i) { + const auto offset = minutes{2 * static_cast(i)}; + CHECK_FALSE(step(strategy, tm, + tickAt(offset, kRisingAsks[i], kRisingAsks[i] - 2)).has_value()); + CHECK_FALSE(step(strategy, tm, + tickAt(offset + seconds{30}, audAsks[i], audAsks[i] - 2, "AUDUSD")) + .has_value()); + } + + CHECK(step(strategy, tm, tickAt(minutes{8}, 110100, 110090)) == Direction::LONG); + CHECK(step(strategy, tm, tickAt(minutes{8} + seconds{30}, 65030, 65020, "AUDUSD")) == + Direction::SHORT); +} + +TEST_CASE("OhlcBreakoutStrategy rejects malformed configuration", "[ohlcBreakout]") { + SECTION("fewer than two OHLC timeframes") { + auto config = makeConfig(2); + config.OHLC_VARIABLES.resize(1); + CHECK_THROWS_AS(OhlcBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("missing OHLC_BREAKOUT_VARIABLES") { + auto config = makeConfig(2); + config.STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES = std::nullopt; + CHECK_THROWS_AS(OhlcBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("OHLC_COUNT below 2") { + auto config = makeConfig(2); + config.OHLC_VARIABLES[0].OHLC_COUNT = 1; + CHECK_THROWS_AS(OhlcBreakoutStrategy{config}, std::invalid_argument); + } + + SECTION("OHLC_MINUTES below 1") { + auto config = makeConfig(2); + config.OHLC_VARIABLES[1].OHLC_MINUTES = 0; + CHECK_THROWS_AS(OhlcBreakoutStrategy{config}, std::invalid_argument); + } +} + +TEST_CASE("StrategyVariables round-trips OHLC_BREAKOUT_VARIABLES through JSON", + "[ohlcBreakout]") { + SECTION("present group survives the round-trip") { + tradingDefinitions::StrategyVariables vars; + vars.OHLC_BREAKOUT_VARIABLES = + tradingDefinitions::OHLCBreakoutVariables{.BUFFER_PIPS = 7}; + + const nlohmann::json j = vars; + const auto back = j.get(); + + REQUIRE(back.OHLC_BREAKOUT_VARIABLES.has_value()); + CHECK(back.OHLC_BREAKOUT_VARIABLES->BUFFER_PIPS == 7); + } + + SECTION("absent group serialises as null and stays absent") { + const tradingDefinitions::StrategyVariables vars; + const nlohmann::json j = vars; + + CHECK(j.at("OHLC_BREAKOUT_VARIABLES").is_null()); + CHECK_FALSE(j.get() + .OHLC_BREAKOUT_VARIABLES.has_value()); + } +} diff --git a/tests/sweep.cpp b/tests/sweep.cpp index e38d065..5ef5f23 100644 --- a/tests/sweep.cpp +++ b/tests/sweep.cpp @@ -6,22 +6,34 @@ #include +#include #include #include +#include #include #include +#include #include +#include #include #include -#include "shared/utilities/parameterSweep.hpp" #include "shared/tradingDefinitions/config/runConfiguration.hpp" -#include "shared/tradingDefinitions/strategy.hpp" +#include "shared/tradingDefinitions/strategyConfig.hpp" +#include "shared/utilities/queueKeys.hpp" +#include "load/redisLoader.hpp" +import loadCommand; // sweep::buildStrategyChunk, sweep::StrategyFactory +import makeStrategy; // sweep::makeStrategy +import parameterGenerator; // sweep::ParameterGenerator, sweep::Combination import randomStrategySweep; // sweep::buildRandomStrategySweep -import runConfigurationBuilder; // sweep::makeRunConfiguration +import ohlcBreakoutStrategySweep; // sweep::buildOhlcBreakoutStrategySweep +import makeOhlcBreakoutStrategy; // sweep::makeOhlcBreakoutStrategy +import runConfigurationBuilder; // sweep::makeRunConfiguration, sweep::resolveSymbolGroups +import symbolGroups; // sweep::cleanSymbols, sweep::allSymbolsKnown import randomStrategy; // RandomStrategy +import ohlcBreakoutStrategy; // OhlcBreakoutStrategy import tradeManager; // TradeManager import priceData; // PriceData import trade; // Direction @@ -101,12 +113,12 @@ TEST_CASE("buildRandomStrategySweep expansion order is deterministic", "[sweep]" } TEST_CASE("makeRunConfiguration carries the run id", "[sweep]") { - const auto config = sweep::makeRunConfiguration("test-run-id"); + const auto config = sweep::makeRunConfiguration("test-run-id", "EURUSD"); CHECK(config.RUN_ID == std::string("test-run-id")); } TEST_CASE("makeRunConfiguration sets the run descriptor values", "[sweep]") { - const auto config = sweep::makeRunConfiguration("test-run-id"); + const auto config = sweep::makeRunConfiguration("test-run-id", "EURUSD"); CHECK(config.SYMBOLS == std::string("EURUSD")); CHECK(config.LAST_MONTHS == 6); CHECK(config.STARTING_BALANCE == tradingDefinitions::DEFAULT_STARTING_BALANCE); @@ -119,7 +131,7 @@ TEST_CASE("makeRunConfiguration sets the run descriptor values", "[sweep]") { // runner parses it back), so the round-trip must preserve every field — // including the decimal ones, which move as strings via decimal_json.hpp. TEST_CASE("makeRunConfiguration survives a JSON round-trip", "[sweep]") { - const auto original = sweep::makeRunConfiguration("round-trip-id"); + const auto original = sweep::makeRunConfiguration("round-trip-id", "EURUSD"); const nlohmann::json j = original; const auto restored = j.get(); @@ -133,11 +145,59 @@ TEST_CASE("makeRunConfiguration survives a JSON round-trip", "[sweep]") { CHECK(restored.REPORT_FAILURES == original.REPORT_FAILURES); } +// cleanSymbols normalises one symbol group for the run side, which splits +// SYMBOLS on ',' WITHOUT trimming — so surrounding whitespace and empty fields +// must be stripped here, while a multi-instrument group stays comma-joined. +TEST_CASE("cleanSymbols trims whitespace and drops empty fields", "[sweep]") { + CHECK(sweep::cleanSymbols("EURUSD") == std::string("EURUSD")); + CHECK(sweep::cleanSymbols("EURUSD,AUDUSD") == std::string("EURUSD,AUDUSD")); + CHECK(sweep::cleanSymbols("EURUSD, AUDUSD") == std::string("EURUSD,AUDUSD")); + CHECK(sweep::cleanSymbols(" EURUSD ,\tAUDUSD ") == std::string("EURUSD,AUDUSD")); + CHECK(sweep::cleanSymbols("EURUSD,,AUDUSD,") == std::string("EURUSD,AUDUSD")); + CHECK(sweep::cleanSymbols(" ").empty()); +} + +// allSymbolsKnown backs the static_asserts guarding kSymbolGroups and the +// per-sweep overrides: it must tokenise groups exactly like cleanSymbols +// (trim, drop empties) before the symbol_scale lookup, and reject any symbol +// missing from the table. +TEST_CASE("allSymbolsKnown validates groups against symbol_scale", "[sweep]") { + constexpr auto known = std::to_array( + {"EURUSD", " GBPUSD ,\tAUDUSD", "XAUUSD,,"}); + constexpr auto unknown = std::to_array( + {"EURUSD", "EURUSD,NOTASYMBOL"}); + CHECK(sweep::allSymbolsKnown(known)); + CHECK_FALSE(sweep::allSymbolsKnown(unknown)); + CHECK(sweep::allSymbolsKnown({})); // no groups = nothing to reject +} + +// resolveSymbolGroups picks the sweep's own symbol list when it set one and +// falls back to the full kSymbolGroups default otherwise — the load command +// fans out one run per returned group, so this decides what a `load` targets. +TEST_CASE("resolveSymbolGroups prefers the sweep override", "[sweep]") { + const std::vector overrideGroups{"EURUSD", "GBPUSD,AUDUSD"}; + CHECK(sweep::resolveSymbolGroups(overrideGroups) == overrideGroups); + + const auto defaults = sweep::resolveSymbolGroups({}); + REQUIRE(defaults.size() == sweep::kSymbolGroups.size()); + for (std::size_t i = 0; i < defaults.size(); ++i) { + CHECK(defaults[i] == sweep::kSymbolGroups[i]); + } +} + +// The shipped kSymbolGroupsOverride narrows the sweep to EURUSD — widening it +// back to the full kSymbolGroups set is a deliberate source edit in +// randomStrategySweep (empty the override), never a silent default. +TEST_CASE("buildRandomStrategySweep narrows the symbols to EURUSD", "[sweep]") { + CHECK(sweep::buildRandomStrategySweep().symbolGroups() == + std::vector{"EURUSD"}); +} + // RandomStrategy ignores everything in its config, so default-constructed -// Strategy is enough — selectStrategy only needs the name, and these tests +// StrategyConfig is enough — selectStrategy only needs the name, and these tests // construct the class directly. TEST_CASE("RandomStrategy always returns a signal", "[sweep]") { - RandomStrategy strategy{tradingDefinitions::Strategy{}}; + RandomStrategy strategy{tradingDefinitions::StrategyConfig{}}; PriceData tick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); for (int i = 0; i < 100; ++i) { @@ -150,7 +210,7 @@ TEST_CASE("RandomStrategy always returns a signal", "[sweep]") { // A fair coin over 1000 flips produces both directions with probability // 1 - 2^-999 — a single-sided run means the distribution (or seeding) broke. TEST_CASE("RandomStrategy produces both directions", "[sweep]") { - RandomStrategy strategy{tradingDefinitions::Strategy{}}; + RandomStrategy strategy{tradingDefinitions::StrategyConfig{}}; PriceData tick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); bool sawLong = false; @@ -168,7 +228,7 @@ TEST_CASE("RandomStrategy produces both directions", "[sweep]") { // positions, otherwise the sweep's stop/limit parameters stop being the only // exit mechanism under test. TEST_CASE("RandomStrategy::during leaves open trades alone", "[sweep]") { - RandomStrategy strategy{tradingDefinitions::Strategy{}}; + RandomStrategy strategy{tradingDefinitions::StrategyConfig{}}; TradeManager manager; PriceData tick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); manager.openTrade(tick, 1, Direction::LONG); @@ -178,3 +238,220 @@ TEST_CASE("RandomStrategy::during leaves open trades alone", "[sweep]") { CHECK(manager.getActiveTrades().size() == 1); CHECK(manager.getClosedTrades().size() == 0); } + +// combinationCount/combinationAt are the lazy mirror of generateAllCombinations +// — the breakout tests below lean on them because that grid is billions of +// combinations, far too large to materialise. Pin the lazy decode to the eager +// expansion on a small synthetic grid: same size, index-for-index identical. +TEST_CASE("ParameterGenerator lazy accessors mirror generateAllCombinations", "[sweep]") { + sweep::ParameterGenerator generator; + generator.addList("A", {1, 3, 5}); + generator.addRange("B", 10, 10, 20); // 10, 20 + generator.addExponential("C", 2, 3, 2); // 2, 6 + + const auto combinations = generator.generateAllCombinations(); + REQUIRE(combinations.size() == 3 * 2 * 2); + REQUIRE(generator.combinationCount() == combinations.size()); + for (std::size_t i = 0; i < combinations.size(); ++i) { + INFO("Lazy decode diverges from eager expansion at index " << i); + CHECK(generator.combinationAt(i).values() == combinations[i].values()); + } + + const auto counts = generator.rangeValueCounts(); + REQUIRE(counts.size() == 3); + CHECK(counts[0] == std::pair{"A", 3}); + CHECK(counts[1] == std::pair{"B", 2}); + CHECK(counts[2] == std::pair{"C", 2}); +} + +// Key shapes for the split queue: the per-run list carries payload key names, +// each payload lives under its own key grouped by RUN_ID for SCAN-able cleanup. +TEST_CASE("queue keys embed run id and strategy uuid", "[sweep]") { + CHECK(queue_keys::strategyKey("run-1") == + std::string("BACKTESTING_QUEUE_STRATEGY:run-1")); + CHECK(queue_keys::strategyPayloadKey("run-1", "uuid-2") == + std::string("BACKTESTING_QUEUE_STRATEGY_PAYLOAD:run-1:uuid-2")); +} + +// buildStrategyChunk is the producer's streaming seam: it maps grid indices +// [begin, end) onto (payload key, strategy JSON) pairs. Contract pinned here: +// 1:1 with the lazily-decoded combinations in index order, every key derived +// from the run id and the payload's OWN freshly-minted UUID, and the JSON +// round-trips to a StrategyConfig carrying that combination's swept values. +TEST_CASE("buildStrategyChunk keys each payload by run id and its UUID", "[sweep]") { + const auto generator = sweep::buildRandomStrategySweep(); + const std::string runId = "test-run-id"; + + const std::size_t begin = 5; + const std::size_t end = 12; + const auto chunk = + sweep::buildStrategyChunk(generator, sweep::makeStrategy, runId, begin, end); + REQUIRE(chunk.size() == end - begin); + + for (std::size_t offset = 0; offset < chunk.size(); ++offset) { + const auto config = nlohmann::json::parse(chunk[offset].rawJson) + .get(); + CHECK_FALSE(config.UUID.empty()); + CHECK(chunk[offset].key == + queue_keys::strategyPayloadKey(runId, config.UUID)); + + const auto combo = generator.combinationAt(begin + offset); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_PIPS == + static_cast(combo.get("STOP_DISTANCE_IN_PIPS"))); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_PIPS == + static_cast(combo.get("LIMIT_DISTANCE_IN_PIPS"))); + } + + // An empty range (how the stream terminates) yields an empty chunk. + CHECK(sweep::buildStrategyChunk(generator, sweep::makeStrategy, runId, end, end) + .empty()); +} + +// Names the OhlcBreakout sweep registers and makeOhlcBreakoutStrategy reads +// back; the two must stay in lockstep or the mapper throws at load time. +namespace { +const std::array kBreakoutParameterNames = { + "BREAKOUT_OHLC_MINUTES", "BREAKOUT_OHLC_COUNT", + "TREND_OHLC_MINUTES", "TREND_OHLC_COUNT", + "BUFFER_PIPS", "STOP_DISTANCE_IN_PIPS", + "LIMIT_DISTANCE_IN_PIPS"}; + +// Stride of each parameter's axis in expansion order (later-registered ranges +// vary fastest): stepping combinationAt's index by strides[k] walks parameter +// k through its values while every other parameter holds still. +std::vector axisStrides( + const std::vector>& counts) { + std::vector strides(counts.size(), 1); + for (std::size_t k = counts.size(); k >= 2; --k) { + strides[k - 2] = strides[k - 1] * counts[k - 1].second; + } + return strides; +} +} + +TEST_CASE("buildOhlcBreakoutStrategySweep registers all seven parameters", "[sweep]") { + const auto generator = sweep::buildOhlcBreakoutStrategySweep(); + CHECK(generator.parameterCount() == kBreakoutParameterNames.size()); + + // Every combination carries every registered range by construction, so + // probing the two ends of the grid proves the names are registered without + // materialising the billions of combinations in between. + REQUIRE(generator.combinationCount() > 0); + for (const auto& combo : {generator.combinationAt(0), + generator.combinationAt(generator.combinationCount() - 1)}) { + for (const auto& name : kBreakoutParameterNames) { + INFO("Combination is missing " << name); + CHECK(combo.has(name)); + } + } +} + +// Config-agnostic (same idea as the random product test), but checked through +// the lazy API because the breakout grid is too large to materialise: walking +// each parameter's axis must yield that range's declared number of DISTINCT +// values (a duplicate inside a range would shrink the true product), their +// product must equal combinationCount(), and a strided sample of full +// combinations must contain no duplicates. +TEST_CASE("buildOhlcBreakoutStrategySweep generates the full cartesian product", "[sweep]") { + const auto generator = sweep::buildOhlcBreakoutStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + std::size_t product = 1; + for (std::size_t k = 0; k < counts.size(); ++k) { + std::set axisValues; + for (std::size_t j = 0; j < counts[k].second; ++j) { + axisValues.insert(generator.combinationAt(j * strides[k]).get(counts[k].first)); + } + INFO(counts[k].first << " expands to duplicate values"); + CHECK(axisValues.size() == counts[k].second); + product *= axisValues.size(); + } + CHECK(product == total); + + constexpr std::size_t kSamples = 1000; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + std::set> uniqueCombinations; + std::size_t sampled = 0; + for (std::size_t i = 0; i < total; i += sampleStride, ++sampled) { + uniqueCombinations.insert(generator.combinationAt(i).values()); + } + CHECK(uniqueCombinations.size() == sampled); +} + +// The breakout sweep narrows itself to EURUSD while the strategy is being +// validated (kOhlcBreakoutSymbolGroupsOverride) — widening it is a deliberate +// source edit, never a silent default. +TEST_CASE("buildOhlcBreakoutStrategySweep narrows the symbols to EURUSD", "[sweep]") { + const auto generator = sweep::buildOhlcBreakoutStrategySweep(); + CHECK(generator.symbolGroups() == std::vector{"EURUSD"}); +} + +TEST_CASE("makeOhlcBreakoutStrategy maps a combination onto the config", "[sweep]") { + sweep::Combination combo; + combo.set("BREAKOUT_OHLC_MINUTES", 15); + combo.set("BREAKOUT_OHLC_COUNT", 24); + combo.set("TREND_OHLC_MINUTES", 60); + combo.set("TREND_OHLC_COUNT", 50); + combo.set("BUFFER_PIPS", 5); + combo.set("STOP_DISTANCE_IN_PIPS", 40); + combo.set("LIMIT_DISTANCE_IN_PIPS", 60); + + const auto config = sweep::makeOhlcBreakoutStrategy(combo); + + // Must match the dispatch string in run/operations.cppm. + CHECK(config.TRADING_VARIABLES.STRATEGY == "OhlcBreakoutStrategy"); + CHECK(config.TRADING_VARIABLES.STOP_DISTANCE_IN_PIPS == 40); + CHECK(config.TRADING_VARIABLES.LIMIT_DISTANCE_IN_PIPS == 60); + CHECK(config.TRADING_VARIABLES.TRADING_SIZE == 1); + CHECK_FALSE(config.UUID.empty()); + + // Positional contract: [0] breakout timeframe, [1] trend timeframe. + REQUIRE(config.OHLC_VARIABLES.size() == 2); + CHECK(config.OHLC_VARIABLES[0].OHLC_COUNT == 24); + CHECK(config.OHLC_VARIABLES[0].OHLC_MINUTES == 15); + CHECK(config.OHLC_VARIABLES[1].OHLC_COUNT == 50); + CHECK(config.OHLC_VARIABLES[1].OHLC_MINUTES == 60); + + REQUIRE(config.STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES.has_value()); + CHECK(config.STRATEGY_VARIABLES.OHLC_BREAKOUT_VARIABLES->BUFFER_PIPS == 5); +} + +// End-to-end contract between the builder, the mapper and the strategy's +// fail-fast ctor: every combination the sweep queues must produce a config an +// OhlcBreakoutStrategy accepts (names line up, OHLC_COUNT >= 2, +// OHLC_MINUTES >= 1, BUFFER_PIPS present) — otherwise the worker throws at +// construction instead of the grid being caught here at test time. The grid is +// too large to construct literally every combination, but the ctor's checks +// are all per-field, so sweeping each parameter's axis exercises every +// distinct value a field can take; a strided sample of full combinations +// guards the mapper against cross-field surprises. +TEST_CASE("makeOhlcBreakoutStrategy accepts every swept parameter value", "[sweep]") { + const auto generator = sweep::buildOhlcBreakoutStrategySweep(); + const auto counts = generator.rangeValueCounts(); + const auto total = generator.combinationCount(); + REQUIRE(total > 0); + REQUIRE(!counts.empty()); + + const auto strides = axisStrides(counts); + for (std::size_t k = 0; k < counts.size(); ++k) { + for (std::size_t j = 0; j < counts[k].second; ++j) { + const auto combo = generator.combinationAt(j * strides[k]); + INFO(counts[k].first << " = " << combo.get(counts[k].first) + << " produced a config the ctor rejects"); + const auto config = sweep::makeOhlcBreakoutStrategy(combo); + CHECK_NOTHROW(OhlcBreakoutStrategy{config}); + } + } + + constexpr std::size_t kSamples = 500; + const std::size_t sampleStride = total > kSamples ? total / kSamples : 1; + for (std::size_t i = 0; i < total; i += sampleStride) { + const auto config = + sweep::makeOhlcBreakoutStrategy(generator.combinationAt(i)); + CHECK_NOTHROW(OhlcBreakoutStrategy{config}); + } +} diff --git a/tests/tradeManager.cpp b/tests/tradeManager.cpp index 9dc9e7f..741abe5 100644 --- a/tests/tradeManager.cpp +++ b/tests/tradeManager.cpp @@ -108,8 +108,8 @@ struct AlwaysLongStrategy : IStrategy { // Trades close in call order, which is the order collect() walks for drawdown. void seedClosedTrade(TradeManager& manager, std::int32_t pnlPoints) { PriceData tick(110000, 110000, std::chrono::system_clock::now(), "EURUSD"); - std::string id = manager.openTrade(tick, 1, Direction::LONG); - manager.closeTrade(id, 110000 + pnlPoints, tick); + manager.openTrade(tick, 1, Direction::LONG); + manager.closeTrade(tick.symbol, 110000 + pnlPoints, tick); } } // namespace @@ -125,17 +125,19 @@ TEST_CASE("TradeManager opens a trade", "[tradeManager]") { TEST_CASE("TradeManager closes a trade", "[tradeManager]") { TradeManager manager; PriceData tick(10000000, 9900000, std::chrono::system_clock::now(), "EURUSD"); - std::string tradeId = manager.openTrade(tick, 1, Direction::LONG); - bool closed = manager.closeTrade(tradeId, 11000000, tick); + manager.openTrade(tick, 1, Direction::LONG); + bool closed = manager.closeTrade(tick.symbol, 11000000, tick); CHECK(closed); CHECK(manager.reviewAccount() == 0); } -TEST_CASE("TradeManager tracks multiple trades", "[tradeManager]") { +// One open trade per symbol is a TradeManager invariant now (activeTrades is +// keyed by symbol), so multiple concurrent trades means multiple symbols. +TEST_CASE("TradeManager tracks multiple trades across symbols", "[tradeManager]") { TradeManager manager; PriceData tick1(10000000, 9900000, std::chrono::system_clock::now(), "EURUSD"); - PriceData tick2(20000000, 19900000, std::chrono::system_clock::now(), "EURUSD"); - PriceData tick3(30000000, 29900000, std::chrono::system_clock::now(), "EURUSD"); + PriceData tick2(20000000, 19900000, std::chrono::system_clock::now(), "GBPUSD"); + PriceData tick3(30000000, 29900000, std::chrono::system_clock::now(), "USDJPY"); manager.openTrade(tick1, 1, Direction::LONG); manager.openTrade(tick2, 2, Direction::SHORT); manager.openTrade(tick3, 3, Direction::LONG); @@ -148,9 +150,10 @@ TEST_CASE("TradeManager records trade details", "[tradeManager]") { PriceData tick(10000000, 9900000, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(tick, 1, Direction::LONG); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(tick.symbol); REQUIRE(trade != trades.end()); + CHECK(trade->second.id == tradeId); CHECK(trade->second.entryPrice == 10000000); CHECK(trade->second.size == 1); CHECK(trade->second.direction == Direction::LONG); @@ -165,7 +168,7 @@ TEST_CASE("LONG does not exit on entry tick with a one-pip spread", "[tradeManag PriceData entryTick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(entryTick, 1, Direction::LONG, 1, 1); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(entryTick.symbol); REQUIRE(trade != trades.end()); CHECK(trade->second.entryPrice == 110010); CHECK(trade->second.exitReferencePrice == 110000); @@ -180,7 +183,7 @@ TEST_CASE("SHORT does not exit on entry tick with a one-pip spread", "[tradeMana PriceData entryTick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(entryTick, 1, Direction::SHORT, 1, 1); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(entryTick.symbol); REQUIRE(trade != trades.end()); CHECK(trade->second.entryPrice == 110000); CHECK(trade->second.exitReferencePrice == 110010); @@ -196,7 +199,7 @@ TEST_CASE("LONG stops out when bid drops one pip below entry bid", "[tradeManage PriceData entryTick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(entryTick, 1, Direction::LONG, 1, 1); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(entryTick.symbol); PriceData laterTick(110000, 109990, std::chrono::system_clock::now(), "EURUSD"); auto exit = trading::exit_rules::checkExit(trade->second, laterTick); @@ -213,7 +216,7 @@ TEST_CASE("checkExit: LONG flat tick does not fire", "[tradeManager]") { PriceData entryTick(110011, 110001, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(entryTick, 1, Direction::LONG, 1, 1); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(entryTick.symbol); REQUIRE(trade != trades.end()); auto exit = trading::exit_rules::checkExit(trade->second, entryTick); @@ -225,7 +228,7 @@ TEST_CASE("checkExit: LONG SL fires at bid", "[tradeManager]") { PriceData entryTick(110011, 110001, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(entryTick, 1, Direction::LONG, 1, 0); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(entryTick.symbol); REQUIRE(trade != trades.end()); // The SL distance is 1 pip (10 points), so entry-bid 110001 minus 10 is @@ -242,7 +245,7 @@ TEST_CASE("checkExit: SHORT SL fires at ask", "[tradeManager]") { PriceData entryTick(110011, 110001, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(entryTick, 1, Direction::SHORT, 1, 0); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(entryTick.symbol); REQUIRE(trade != trades.end()); PriceData nextTick(110021, 110011, std::chrono::system_clock::now(), "EURUSD"); @@ -257,7 +260,7 @@ TEST_CASE("checkExit: LONG TP fires at bid", "[tradeManager]") { PriceData entryTick(110011, 110001, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(entryTick, 1, Direction::LONG, 0, 1); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(entryTick.symbol); REQUIRE(trade != trades.end()); PriceData nextTick(110021, 110011, std::chrono::system_clock::now(), "EURUSD"); @@ -272,7 +275,7 @@ TEST_CASE("checkExit: SHORT TP fires at ask", "[tradeManager]") { PriceData entryTick(110011, 110001, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(entryTick, 1, Direction::SHORT, 0, 1); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(entryTick.symbol); REQUIRE(trade != trades.end()); PriceData nextTick(110001, 109991, std::chrono::system_clock::now(), "EURUSD"); @@ -296,7 +299,7 @@ TEST_CASE("reviewStopAndLimit skips trades for other symbols", "[tradeManager]") CHECK(manager.reviewAccount() == 1); auto trades = manager.getActiveTrades(); - CHECK(trades.find(tradeId) != trades.end()); + CHECK(trades.find(entryTick.symbol) != trades.end()); } // Matching-symbol tick still closes — the filter must not over-block exits @@ -310,7 +313,7 @@ TEST_CASE("reviewStopAndLimit closes a trade on a matching-symbol tick", "[trade trading::reviewStopAndLimit(manager, stopTick); auto active = manager.getActiveTrades(); - CHECK(active.find(tradeId) == active.end()); + CHECK(active.find(entryTick.symbol) == active.end()); const auto& closed = manager.getClosedTrades(); REQUIRE(closed.size() == 1); @@ -331,7 +334,7 @@ TEST_CASE("reviewStopAndLimit: AUSIDXAUD trade not closed by EURUSD tick", "[tra CHECK(manager.reviewAccount() == 1); auto trades = manager.getActiveTrades(); - CHECK(trades.find(tradeId) != trades.end()); + CHECK(trades.find(entryTick.symbol) != trades.end()); } TEST_CASE("hasActiveTradeForSymbol: empty manager returns false", "[tradeManager]") { @@ -351,10 +354,10 @@ TEST_CASE("hasActiveTradeForSymbol: true for opened symbol, false for other", "[ TEST_CASE("hasActiveTradeForSymbol: false after closeTrade", "[tradeManager]") { TradeManager manager; PriceData tick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); - std::string tradeId = manager.openTrade(tick, 1, Direction::LONG); + manager.openTrade(tick, 1, Direction::LONG); REQUIRE(manager.hasActiveTradeForSymbol("EURUSD")); - bool closed = manager.closeTrade(tradeId, 110000, tick); + bool closed = manager.closeTrade(tick.symbol, 110000, tick); CHECK(closed); CHECK_FALSE(manager.hasActiveTradeForSymbol("EURUSD")); } @@ -374,14 +377,21 @@ TEST_CASE("Can open trades on different symbols simultaneously", "[tradeManager] CHECK(manager.hasActiveTradeForSymbol("AUSIDXAUD")); } -// openTrade does not enforce same-symbol uniqueness, so callers rely on -// hasActiveTradeForSymbol to gate same-symbol re-entry. This pins the invariant -// that a single open is enough to flip the helper to true. -TEST_CASE("Helper reports symbol active after first open", "[tradeManager]") { +// activeTrades is keyed by symbol, so openTrade refuses a same-symbol +// double-open: the second call returns the existing trade's id and leaves the +// account untouched. Callers still gate re-entry on hasActiveTradeForSymbol; +// this pins the fallback behaviour if that gate is ever bypassed. +TEST_CASE("openTrade refuses a second open on an active symbol", "[tradeManager]") { TradeManager manager; PriceData tick(110010, 110000, std::chrono::system_clock::now(), "EURUSD"); - manager.openTrade(tick, 1, Direction::LONG); + std::string firstId = manager.openTrade(tick, 1, Direction::LONG); CHECK(manager.hasActiveTradeForSymbol("EURUSD")); + + std::string secondId = manager.openTrade(tick, 1, Direction::SHORT); + CHECK(secondId == firstId); + CHECK(manager.reviewAccount() == 1); + // The original LONG survives; the refused SHORT never entered the book. + CHECK(manager.getActiveTrades().begin()->second.direction == Direction::LONG); } // --- Entry-side bid/ask handling --- @@ -394,7 +404,7 @@ TEST_CASE("openTrade: LONG enters at ask and records the spread", "[tradeManager PriceData tick(10000000, 9900000, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(tick, 1, Direction::LONG); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(tick.symbol); REQUIRE(trade != trades.end()); CHECK(trade->second.entryPrice == 10000000); @@ -409,7 +419,7 @@ TEST_CASE("openTrade: SHORT enters at bid and records the spread", "[tradeManage PriceData tick(10000000, 9900000, std::chrono::system_clock::now(), "EURUSD"); std::string tradeId = manager.openTrade(tick, 1, Direction::SHORT); auto trades = manager.getActiveTrades(); - auto trade = trades.find(tradeId); + auto trade = trades.find(tick.symbol); REQUIRE(trade != trades.end()); CHECK(trade->second.entryPrice == 9900000);