From 5be0108c9a866e4745b287b446659ff375e41c54 Mon Sep 17 00:00:00 2001 From: oglego Date: Fri, 14 Aug 2026 18:39:27 -0500 Subject: [PATCH 1/2] feat: add support for GBNF grammar fix: update comment on GBNF grammar fix: update comments in model.h for GBNF grammar test: update test suite for GBNF grammar support refactor: clean up grammar test formatting feat: update GBNF grammar support with example and reset fix Update GBNF grammar support to ModelConfig via a `grammar` string and `grammar_root` rule name, applied to the sampler chain in initialize_context(). load_grammar_file() reads a .gbnf file into ModelConfig::grammar. The grammar sampler retains its parse position across calls to generate_from_tokens(), so a completed grammar from one turn forces EOS immediately on the next. Reset only the grammar sampler (not the whole chain) at the start of each turn, via a non-owning pointer kept on Model, so the dist sampler's RNG and other stateful samplers are left untouched when an explicit seed is configured. Add examples/grammar, a minimal sentiment classifier constrained to {"sentiment": "positive"|"negative"|"neutral"} via sentiment.gbnf, demonstrating the feature end-to-end. Document grammar-constrained output in the README and register the example in CMakeLists.txt. fix: allow custom grammar roots in grammar example Add a -r flag to the grammar demo, update the README usage examples, and polish the grammar documentation wording. fix: comment formatting in model.cpp fix: update formatting for comments in model.h --- CMakeLists.txt | 28 +++++++- README.md | 14 ++++ examples/README.md | 4 ++ examples/grammar/CMakeLists.txt | 24 +++++++ examples/grammar/README.md | 78 ++++++++++++++++++++++ examples/grammar/grammar.cpp | 110 ++++++++++++++++++++++++++++++++ examples/grammar/sentiment.gbnf | 3 + src/model.cpp | 44 +++++++++++++ src/model.h | 10 +++ tests/test_grammar.cpp | 72 +++++++++++++++++++++ 10 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 examples/grammar/CMakeLists.txt create mode 100644 examples/grammar/README.md create mode 100644 examples/grammar/grammar.cpp create mode 100644 examples/grammar/sentiment.gbnf create mode 100644 tests/test_grammar.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0603d3e..6636064 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,6 +131,18 @@ if(AGENT_CPP_BUILD_TESTS) target_link_libraries(test_tool PRIVATE common llama) target_compile_features(test_tool PRIVATE cxx_std_17) + add_executable(test_grammar tests/test_grammar.cpp) + target_include_directories(test_grammar PRIVATE + src + tests + ${LLAMA_SOURCE_DIR}/common + ${LLAMA_SOURCE_DIR}/ggml/include + ${LLAMA_SOURCE_DIR}/include + ${LLAMA_SOURCE_DIR}/vendor + ) + target_link_libraries(test_grammar PRIVATE model common llama) + target_compile_features(test_grammar PRIVATE cxx_std_17) + add_executable(test_callbacks tests/test_callbacks.cpp) target_include_directories(test_callbacks PRIVATE src @@ -145,6 +157,7 @@ if(AGENT_CPP_BUILD_TESTS) add_test(NAME ToolTests COMMAND test_tool) add_test(NAME CallbacksTests COMMAND test_callbacks) + add_test(NAME GrammarTests COMMAND test_grammar) if(AGENT_CPP_BUILD_MCP) add_executable(test_mcp_client tests/test_mcp_client.cpp) @@ -162,7 +175,7 @@ if(AGENT_CPP_BUILD_TESTS) # On Windows, DLLs are placed in the bin/ directory by llama.cpp # We need to add this directory to PATH so tests can find the DLLs if(WIN32) - set_tests_properties(ToolTests CallbacksTests PROPERTIES + set_tests_properties(ToolTests CallbacksTests GrammarTests PROPERTIES ENVIRONMENT "PATH=${CMAKE_BINARY_DIR}/bin\;$ENV{PATH}" ) endif() @@ -223,6 +236,19 @@ if(AGENT_CPP_BUILD_EXAMPLES) target_link_libraries(context-engineering-example PRIVATE agent model common llama) target_compile_features(context-engineering-example PRIVATE cxx_std_17) + # Grammar example + add_executable(grammar-example examples/grammar/grammar.cpp) + target_include_directories(grammar-example PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/examples/shared + ${LLAMA_SOURCE_DIR}/common + ${LLAMA_SOURCE_DIR}/ggml/include + ${LLAMA_SOURCE_DIR}/include + ${LLAMA_SOURCE_DIR}/vendor + ) + target_link_libraries(grammar-example PRIVATE agent model common llama) + target_compile_features(grammar-example PRIVATE cxx_std_17) + # MCP client example (requires MCP support) if(AGENT_CPP_BUILD_MCP) add_executable(mcp-example examples/mcp/mcp.cpp) diff --git a/README.md b/README.md index aca6d28..5fc747d 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ Building blocks for **local** agents in C++. - **[Context Engineering](./examples/context-engineering/README.md)** - Use callbacks to manipulate the context between iterations of the agent loop. +- **[Grammar](./examples/grammar/README.md)** - Constrain model output to a GBNF grammar so every response matches a fixed structure. + - **[Memory](./examples/memory/README.md)** - Use tools that allow an agent to store and retrieve relevant information across conversations. - **[Multi-Agent](./examples/multi-agent/README.md)** - Build a multi-agent system with weight sharing where a main agent delegates to specialized sub-agents. @@ -70,6 +72,18 @@ Handles: - Text generation with configurable sampling (temperature, top_p, top_k, etc.) - KV cache management for efficient prompt caching +### Grammar-constrained output + +Model output can be constrained to a [GBNF grammar](https://github.com/ggml-org/llama.cpp/blob/master/grammars/README.md) to enforce structured formats (e.g. JSON, yes/no answers): + +```cpp +ModelConfig config; +config.grammar = agent_cpp::load_grammar_file("path/to/grammar.gbnf"); +config.grammar_root = "root"; // optional, defaults to "root" +``` + +See the [Grammar example](./examples/grammar/README.md) for a full working demo. + ## Tools Tools extend the agent's capabilities beyond text generation. Each tool defines: diff --git a/examples/README.md b/examples/README.md index c5132c7..a060ab7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,6 +2,10 @@ This directory contains example applications demonstrating agent.cpp capabilities. +## Grammar + +The [grammar](./grammar) example demonstrates constraining model output to a GBNF grammar, so every response is guaranteed to match a fixed structure. You can also point it at a custom grammar file and root rule. + ## Shared Utilities The [shared](./shared) directory contains reusable helper components used across multiple examples. These are **not part of the public API** but can be useful as reference implementations. diff --git a/examples/grammar/CMakeLists.txt b/examples/grammar/CMakeLists.txt new file mode 100644 index 0000000..799e40f --- /dev/null +++ b/examples/grammar/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.14) +project(grammar-example VERSION 0.1.0) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../.. ${CMAKE_CURRENT_BINARY_DIR}/agent-cpp) + +add_executable(grammar-example grammar.cpp) + +target_include_directories(grammar-example PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../../src + ${CMAKE_CURRENT_SOURCE_DIR}/../shared + ${LLAMA_SOURCE_DIR}/common + ${LLAMA_SOURCE_DIR}/ggml/include + ${LLAMA_SOURCE_DIR}/include + ${LLAMA_SOURCE_DIR}/vendor +) + +target_link_libraries(grammar-example PRIVATE agent-cpp::agent common llama) +target_compile_features(grammar-example PRIVATE cxx_std_17) + +message(STATUS "Grammar example configured.") diff --git a/examples/grammar/README.md b/examples/grammar/README.md new file mode 100644 index 0000000..127df2d --- /dev/null +++ b/examples/grammar/README.md @@ -0,0 +1,78 @@ +# Grammar Example + +This example shows how to constrain model output with a [GBNF grammar](https://github.com/ggml-org/llama.cpp/blob/master/grammars/README.md). Instead of relying on prompt instructions alone, the grammar is enforced at the sampler level, so every response matches it. + +The demo is a minimal sentiment classifier: whatever the user types, the model replies with a single well-formed JSON object shaped like `{"sentiment": "positive" | "negative" | "neutral"}`. + +## Building Blocks + +### Grammar + +[`sentiment.gbnf`](./sentiment.gbnf) defines the grammar used in this example: + +```gbnf +root ::= "{" ws "\"sentiment\":" ws sentiment ws "}" +sentiment ::= "\"positive\"" | "\"negative\"" | "\"neutral\"" +ws ::= [ \t\n]* +``` + +Load it with `agent_cpp::load_grammar_file` and set it on `ModelConfig::grammar`: + +```cpp +auto model_config = agent_cpp::ModelConfig{}; +model_config.grammar = agent_cpp::load_grammar_file("sentiment.gbnf"); +model_config.grammar_root = "root"; // optional, this is the default +``` + +No tools are used in this example. The grammar alone is enough to constrain every response. + +## Building + +> [!IMPORTANT] +> Check the [llama.cpp build documentation](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md) to find +> CMake flags you might want to pass depending on your available hardware. + +```bash +cd examples/grammar + +git -C ../.. submodule update --init --recursive + +cmake -B build +cmake --build build -j$(nproc) +``` + +### Using a custom llama.cpp + +If you have llama.cpp already downloaded: + +```bash +cmake -B build -DLLAMA_CPP_DIR=/path/to/your/llama.cpp +cmake --build build -j$(nproc) +``` + +## Usage + +```bash +./build/grammar-example -m "path-to-model.gguf" + +# Use a different grammar file +./build/grammar-example -m "path-to-model.gguf" -g "path-to-grammar.gbnf" + +# Use a different grammar root rule +./build/grammar-example -m "path-to-model.gguf" -g "path-to-grammar.gbnf" -r "start" +``` + +## Example + +```console +$ ./build/grammar-example -m ../../granite-4.0-micro-Q8_0.gguf +> This library is exactly what I needed, thank you! +{"sentiment": "positive"} + +> The build failed three times before I figured out the flag I was missing. +{"sentiment": "negative"} + +> The package arrived on Tuesday. +{"sentiment": "neutral"} +> +``` diff --git a/examples/grammar/grammar.cpp b/examples/grammar/grammar.cpp new file mode 100644 index 0000000..d67c5d3 --- /dev/null +++ b/examples/grammar/grammar.cpp @@ -0,0 +1,110 @@ +#include "agent.h" +#include "chat_loop.h" +#include "error.h" +#include "model.h" +#include +#include +#include +#include +#include + +static constexpr const char* DEFAULT_GRAMMAR_PATH = "sentiment.gbnf"; +static constexpr const char* DEFAULT_GRAMMAR_ROOT = "root"; + +static void +print_usage(int /*unused*/, char** argv) +{ + printf("\nexample usage:\n"); + printf("\n %s -m model.gguf\n", argv[0]); + printf("\n"); + printf("options:\n"); + printf(" -m Path to the GGUF model file (required)\n"); + printf(" -g Path to a GBNF grammar file (default: %s)\n", + DEFAULT_GRAMMAR_PATH); + printf(" -r Grammar root rule (default: %s)\n", + DEFAULT_GRAMMAR_ROOT); + printf("\n"); +} + +int +main(int argc, char** argv) +{ + std::string model_path; + std::string grammar_path = DEFAULT_GRAMMAR_PATH; + std::string grammar_root = DEFAULT_GRAMMAR_ROOT; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-m") == 0) { + if (i + 1 < argc) { + model_path = argv[++i]; + } else { + print_usage(argc, argv); + return 1; + } + } else if (strcmp(argv[i], "-g") == 0) { + if (i + 1 < argc) { + grammar_path = argv[++i]; + } else { + print_usage(argc, argv); + return 1; + } + } else if (strcmp(argv[i], "-r") == 0) { + if (i + 1 < argc) { + grammar_root = argv[++i]; + } else { + print_usage(argc, argv); + return 1; + } + } else { + print_usage(argc, argv); + return 1; + } + } + + if (model_path.empty()) { + print_usage(argc, argv); + return 1; + } + + printf("Loading grammar from '%s'...\n", grammar_path.c_str()); + auto model_config = agent_cpp::ModelConfig{}; + try { + model_config.grammar = agent_cpp::load_grammar_file(grammar_path); + } catch (const agent_cpp::ModelError& e) { + fprintf(stderr, "error: %s\n", e.what()); + return 1; + } + model_config.grammar_root = grammar_root; + model_config.n_ctx = 4096; + model_config.temp = 0.0F; + + printf("Loading model...\n"); + std::shared_ptr model; + try { + model = agent_cpp::Model::create(model_path, model_config); + } catch (const agent_cpp::ModelError& e) { + fprintf(stderr, "error: %s\n", e.what()); + return 1; + } + printf("Model loaded successfully\n"); + + // No tools are needed: the grammar alone constrains the output. + std::vector> tools; + + const std::string instructions = + "You are a sentiment classifier. Given a message from the user, " + "respond with your assessment of its sentiment. Do not explain your " + "reasoning, just answer directly."; + + agent_cpp::Agent agent( + std::move(model), std::move(tools), {}, instructions); + + printf("\nGrammar Demo ready!\n"); + printf(" Every response is constrained by sentiment.gbnf, so the " + "model can only ever reply with {\"sentiment\": \"positive\" | " + "\"negative\" | \"neutral\"}, regardless of what you type.\n"); + printf(" Type an empty line to quit.\n\n"); + + run_chat_loop(agent); + return 0; +} diff --git a/examples/grammar/sentiment.gbnf b/examples/grammar/sentiment.gbnf new file mode 100644 index 0000000..555b3e3 --- /dev/null +++ b/examples/grammar/sentiment.gbnf @@ -0,0 +1,3 @@ +root ::= "{" ws "\"sentiment\":" ws sentiment ws "}" +sentiment ::= "\"positive\"" | "\"negative\"" | "\"neutral\"" +ws ::= [ \t\n]* diff --git a/src/model.cpp b/src/model.cpp index 1186767..213aff5 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -3,9 +3,25 @@ #include "error.h" #include #include +#include +#include namespace agent_cpp { +std::string +load_grammar_file(const std::string& grammar_path) +{ + std::ifstream file(grammar_path); + if (!file) { + throw ModelError("failed to open grammar file '" + grammar_path + + "'"); + } + + std::ostringstream contents; + contents << file.rdbuf(); + return contents.str(); +} + std::shared_ptr ModelWeights::create(const std::string& model_path) { @@ -70,12 +86,14 @@ Model::Model(Model&& other) noexcept : weights_(std::move(other.weights_)) , ctx_(other.ctx_) , sampler_(other.sampler_) + , grammar_sampler_(other.grammar_sampler_) , processed_tokens_(std::move(other.processed_tokens_)) , n_past_(other.n_past_) , config_(other.config_) { other.ctx_ = nullptr; other.sampler_ = nullptr; + other.grammar_sampler_ = nullptr; other.n_past_ = 0; } @@ -93,12 +111,14 @@ Model::operator=(Model&& other) noexcept weights_ = std::move(other.weights_); ctx_ = other.ctx_; sampler_ = other.sampler_; + grammar_sampler_ = other.grammar_sampler_; processed_tokens_ = std::move(other.processed_tokens_); n_past_ = other.n_past_; config_ = other.config_; other.ctx_ = nullptr; other.sampler_ = nullptr; + other.grammar_sampler_ = nullptr; other.n_past_ = 0; } return *this; @@ -123,6 +143,24 @@ Model::initialize_context(const ModelConfig& model_config) } sampler_ = llama_sampler_chain_init(llama_sampler_chain_default_params()); + + if (!model_config.grammar.empty()) { + llama_sampler* grammar_sampler = + llama_sampler_init_grammar(weights_->get_vocab(), + model_config.grammar.c_str(), + model_config.grammar_root.c_str()); + if (grammar_sampler == nullptr) { + llama_sampler_free(sampler_); + sampler_ = nullptr; + throw ModelError("failed to parse GBNF grammar (root rule '" + + model_config.grammar_root + "')"); + } + // Add grammar before the rest of the sampler chain + llama_sampler_chain_add(sampler_, grammar_sampler); + // Keep a non-owning reference so we can reset this sampler between turns + grammar_sampler_ = grammar_sampler; + } + llama_sampler_chain_add(sampler_, llama_sampler_init_top_k(model_config.top_k)); llama_sampler_chain_add(sampler_, @@ -243,6 +281,12 @@ Model::generate_from_tokens(const std::vector& all_tokens, i += batch_size; } + // Reset the grammar sampler before each turn so a finished grammar does + // not force EOS on the next call. + if (grammar_sampler_ != nullptr) { + llama_sampler_reset(grammar_sampler_); + } + llama_token new_token_id{}; while (true) { new_token_id = llama_sampler_sample(sampler_, ctx_, -1); diff --git a/src/model.h b/src/model.h index 5f75771..27515ad 100644 --- a/src/model.h +++ b/src/model.h @@ -33,8 +33,15 @@ struct ModelConfig static_cast(std::max(1u, std::thread::hardware_concurrency() - 1)); ggml_type cache_type_k = GGML_TYPE_F16; ggml_type cache_type_v = GGML_TYPE_F16; + // Optional GBNF grammar and root rule name + std::string grammar; + std::string grammar_root = "root"; }; +/// Reads a GBNF file into a string for ModelConfig::grammar +/// @throws ModelError if the file cannot be opened +std::string load_grammar_file(const std::string& grammar_path); + // Forward declaration class Model; @@ -184,6 +191,9 @@ class Model std::shared_ptr weights_; llama_context* ctx_ = nullptr; llama_sampler* sampler_ = nullptr; + // Non-owning pointer to the grammar sampler in sampler_'s chain + // Reset this one between turns without resetting the rest of the chain + llama_sampler* grammar_sampler_ = nullptr; std::vector processed_tokens_; // Track tokens in KV cache int n_past_ = 0; // Track position in KV cache ModelConfig config_; diff --git a/tests/test_grammar.cpp b/tests/test_grammar.cpp new file mode 100644 index 0000000..f5a417e --- /dev/null +++ b/tests/test_grammar.cpp @@ -0,0 +1,72 @@ +#include "error.h" +#include "model.h" +#include "test_utils.h" +#include +#include +#include + +namespace { + +std::string +write_temp_file(const std::string& filename, const std::string& contents) +{ + std::ofstream file(filename); + file << contents; + return filename; +} + +} + +TEST(test_model_config_grammar_defaults) +{ + agent_cpp::ModelConfig config; + + ASSERT_TRUE(config.grammar.empty()); + ASSERT_EQ(config.grammar_root, "root"); +} + +TEST(test_load_grammar_file_reads_contents) +{ + const std::string filename = "test_grammar_tmp.gbnf"; + const std::string grammar = "root ::= \"yes\" | \"no\"\n"; + + write_temp_file(filename, grammar); + + std::string loaded = agent_cpp::load_grammar_file(filename); + std::remove(filename.c_str()); + + ASSERT_EQ(loaded, grammar); +} + +TEST(test_load_grammar_file_missing_file_throws) +{ + bool threw = false; + try { + agent_cpp::load_grammar_file("nonexistent_file_12345.gbnf"); + } catch (const agent_cpp::ModelError&) { + threw = true; + } + ASSERT_TRUE(threw); +} + +// This needs a loaded GGUF model and a real llama_context, which this test +// file does not exercise. Verify the turn-to-turn grammar reset manually in +// examples/grammar. + +int +main() +{ + std::cout << "\n=== Running Grammar Unit Tests ===\n" << std::endl; + + try { + RUN_TEST(test_model_config_grammar_defaults); + RUN_TEST(test_load_grammar_file_reads_contents); + RUN_TEST(test_load_grammar_file_missing_file_throws); + + std::cout << "\n=== All tests passed! āœ“ ===\n" << std::endl; + return 0; + } catch (const std::exception& e) { + std::cerr << "\nāœ— TEST FAILED: " << e.what() << std::endl; + return 1; + } +} From cabf6846b23be11a0ead16969be00a5ef8195910 Mon Sep 17 00:00:00 2001 From: oglego Date: Fri, 21 Aug 2026 16:07:34 -0500 Subject: [PATCH 2/2] fix: apply clang-format --- src/model.cpp | 6 +++--- src/model.h | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/model.cpp b/src/model.cpp index 213aff5..1250f42 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -13,8 +13,7 @@ load_grammar_file(const std::string& grammar_path) { std::ifstream file(grammar_path); if (!file) { - throw ModelError("failed to open grammar file '" + grammar_path + - "'"); + throw ModelError("failed to open grammar file '" + grammar_path + "'"); } std::ostringstream contents; @@ -157,7 +156,8 @@ Model::initialize_context(const ModelConfig& model_config) } // Add grammar before the rest of the sampler chain llama_sampler_chain_add(sampler_, grammar_sampler); - // Keep a non-owning reference so we can reset this sampler between turns + // Keep a non-owning reference so we can reset this sampler between + // turns grammar_sampler_ = grammar_sampler; } diff --git a/src/model.h b/src/model.h index 27515ad..873a52a 100644 --- a/src/model.h +++ b/src/model.h @@ -40,7 +40,8 @@ struct ModelConfig /// Reads a GBNF file into a string for ModelConfig::grammar /// @throws ModelError if the file cannot be opened -std::string load_grammar_file(const std::string& grammar_path); +std::string +load_grammar_file(const std::string& grammar_path); // Forward declaration class Model;