diff --git a/docs/models/breeze_tts.md b/docs/models/breeze_tts.md index 082a77220..94ddbaafb 100644 --- a/docs/models/breeze_tts.md +++ b/docs/models/breeze_tts.md @@ -97,6 +97,7 @@ audiocpp_cli \ |---|---|---:|---| | `breeze_tts.reference_cache_slots` | integer >= 0 | `1` | Prepared reference-audio cache slots. | | `breeze_tts.attention` | `auto`, `flash`, `eager` | `auto` | Attention kernel. `auto` uses flash except on Volta/Turing GPUs (e.g. V100), where it falls back to eager to avoid missing MMA kernels. | +| `breeze_tts.bf16_activations` | `auto`, `on`, `off` | `auto` | Reference bf16 activation rounding (and, on Metal, the bf16 KV cache). `auto` is on for CUDA/HIP/Vulkan and off on Metal; see [Metal and the reference bf16 path](#metal-and-the-reference-bf16-path). | | `weight_type` | `native`, `f32`, `f16`, `bf16`, `q8_0`, `q4_0`, `q4_k` | `native` | Weight storage type; quantized types convert at load time from the BF16 package. | BreezeTTS streaming is incremental by default. It emits audio events from the @@ -126,3 +127,36 @@ and HIP alike: `q8_0` cut the fixed 100-token regression case from RTF ~1.5 to voice-design regression cases. Counter to intuition, fp32 is the one configuration known to be *worse* for this model (mispronunciations and runaway repetition), because the model is trained and tuned in bf16. + +## Metal and the reference bf16 path + +The official BreezeTTS 2 inference rounds activations to bf16 at every decoder +stage and keeps a bf16 KV cache. CUDA, HIP and Vulkan match that by default; the +casts are cheap enough there. On Metal the same rounding costs a visible share +of the AR loop — even after adding the fused round-to-bf16 unary op for Metal +(which removes the f32 -> bf16 -> f32 cast pair at every rounding point) the +reference path measured roughly **20% slower on the AR component** on a Mac +mini M4, and about the same on an M4 MacBook Air — so on Metal it is **opt-in**: + +```bash +# Reference bf16 parity on Metal (slower, matches the official implementation) +audiocpp_cli --task tts --family breeze_tts \ + --model models/Breeze-TTS-2-GGUF/breeze-tts-2-q8_0.gguf \ + --backend metal \ + --text "Welcome to the local voice demo." \ + --session-option breeze_tts.bf16_activations=on \ + --out breeze_tts_bf16.wav +``` + +| Value | Behavior | +|---|---| +| `auto` (default) | on for CUDA/HIP/Vulkan, off on Metal | +| `on` | bf16 activation rounding on every GPU backend; on Metal it also switches the KV cache to bf16 | +| `off` | f32 activations; Metal/CUDA/Vulkan keep an f16 KV cache (HIP keeps bf16) | + +The trade-off is real in both directions: the Metal default (`auto` -> off) is +the faster path but is the same f32 configuration this page warns about above, +so if a prompt mispronounces or collapses into repetition, re-run it with +`bf16_activations=on` before changing anything else. On non-Metal GPUs the +default already matches the reference, and `off` is only useful for A/B +measurements. diff --git a/external/ggml/src/ggml-metal/ggml-metal-device.cpp b/external/ggml/src/ggml-metal/ggml-metal-device.cpp index 8f11f92a2..a47145ff1 100644 --- a/external/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/external/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -320,6 +320,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary(ggml_metal case GGML_UNARY_OP_ROUND: op_num = OP_UNARY_NUM_ROUND; break; case GGML_UNARY_OP_TRUNC: op_num = OP_UNARY_NUM_TRUNC; break; case GGML_UNARY_OP_XIELU: op_num = OP_UNARY_NUM_XIELU; break; + case GGML_UNARY_OP_ROUND_BF16: op_num = OP_UNARY_NUM_ROUND_BF16; break; default: GGML_ABORT("fatal error"); } break; default: GGML_ABORT("fatal error"); diff --git a/external/ggml/src/ggml-metal/ggml-metal-device.m b/external/ggml/src/ggml-metal/ggml-metal-device.m index dca96bc5c..f588860ba 100644 --- a/external/ggml/src/ggml-metal/ggml-metal-device.m +++ b/external/ggml/src/ggml-metal/ggml-metal-device.m @@ -1091,6 +1091,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_UNARY_OP_TRUNC: case GGML_UNARY_OP_XIELU: return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); + case GGML_UNARY_OP_ROUND_BF16: + // Fused round-to-bf16: f32/f16/bf16 source, f32 result. The rounding + // itself is integer math, so only reading a bf16 source needs + // hardware support; other devices keep the cast round trip. + return op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]) && + (op->src[0]->type == GGML_TYPE_F32 || + op->src[0]->type == GGML_TYPE_F16 || + (op->src[0]->type == GGML_TYPE_BF16 && has_bfloat)); default: return false; } @@ -1279,14 +1287,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te switch (op->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: - return true; + case GGML_TYPE_BF16: return true; default: return false; } case GGML_TYPE_BF16: switch (op->type) { case GGML_TYPE_F32: - case GGML_TYPE_BF16: + case GGML_TYPE_F16: case GGML_TYPE_BF16: return true; default: return false; diff --git a/external/ggml/src/ggml-metal/ggml-metal-impl.h b/external/ggml/src/ggml-metal/ggml-metal-impl.h index a6ad1eec5..af3d64b7c 100644 --- a/external/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/external/ggml/src/ggml-metal/ggml-metal-impl.h @@ -141,6 +141,7 @@ #define OP_UNARY_NUM_ROUND 119 #define OP_UNARY_NUM_TRUNC 120 #define OP_UNARY_NUM_XIELU 121 +#define OP_UNARY_NUM_ROUND_BF16 122 #define OP_SUM_ROWS_NUM_SUM_ROWS 10 #define OP_SUM_ROWS_NUM_MEAN 11 diff --git a/external/ggml/src/ggml-metal/ggml-metal.metal b/external/ggml/src/ggml-metal/ggml-metal.metal index b71b83c68..4655626f4 100644 --- a/external/ggml/src/ggml-metal/ggml-metal.metal +++ b/external/ggml/src/ggml-metal/ggml-metal.metal @@ -1013,6 +1013,28 @@ template<> inline float4 elu_approx(float4 x) { constant short FC_unary_op [[function_constant(FC_UNARY + 0)]]; constant bool FC_unary_cnt[[function_constant(FC_UNARY + 1)]]; +// GGML_UNARY_OP_ROUND_BF16 semantics: round-to-nearest-even f32 -> bf16 -> f32, +// mirroring ggml's ggml_compute_fp32_to_bf16 (NaNs are forced quiet). Doing it with +// integer math keeps the op available on Metal devices without bf16 hardware support. +static inline uint bf16_round_bits(uint u) { + if ((u & 0x7fffffffu) > 0x7f800000u) { + return ((u >> 16) | 64u) << 16; + } + return (u + (0x7fffu + ((u >> 16) & 1u))) & 0xffff0000u; +} + +static inline uint4 bf16_round_bits(uint4 u) { + const uint4 sign_mask = uint4(0x7fffffffu); + const uint4 inf_bits = uint4(0x7f800000u); + const uint4 high = u >> 16; + const uint4 rounded = (u + (0x7fffu + (high & 1u))) & 0xffff0000u; + const uint4 quiet = (high | 64u) << 16; + return select(rounded, quiet, (u & sign_mask) > inf_bits); +} + +static inline float bf16_round_f32(float x) { return as_type(bf16_round_bits(as_type(x))); } +static inline float4 bf16_round_f32(float4 x) { return as_type(bf16_round_bits(as_type(x))); } + template kernel void kernel_unary_impl( constant ggml_metal_kargs_unary & args, @@ -1186,16 +1208,28 @@ kernel void kernel_unary_impl( const TC y_neg = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi; dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg); } + + if (FC_OP == OP_UNARY_NUM_ROUND_BF16) { + dst_ptr[i0] = (T) bf16_round_f32(x); + } } #undef FC_OP #undef FC_CNT } -typedef decltype(kernel_unary_impl) kernel_unary_t; - -template [[host_name("kernel_unary_f32_f32")]] kernel kernel_unary_t kernel_unary_impl; -template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl; +typedef decltype(kernel_unary_impl) kernel_unary_t; + +template [[host_name("kernel_unary_f32_f32")]] kernel kernel_unary_t kernel_unary_impl; +template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl; +// GGML_UNARY_OP_ROUND_BF16 widens f16/bf16 inputs to f32 while rounding, so the +// fused op needs the narrower source types as well. +template [[host_name("kernel_unary_f16_f32")]] kernel kernel_unary_t kernel_unary_impl; +template [[host_name("kernel_unary_f16_f32_4")]] kernel kernel_unary_t kernel_unary_impl; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_unary_bf16_f32")]] kernel kernel_unary_t kernel_unary_impl; +template [[host_name("kernel_unary_bf16_f32_4")]] kernel kernel_unary_t kernel_unary_impl; +#endif template [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl; template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl; @@ -7932,6 +7966,8 @@ template [[host_name("kernel_cpy_contig_f16_f16")]] kernel kernel_cpy_contig_t k #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_cpy_contig_bf16_f32")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t; template [[host_name("kernel_cpy_contig_bf16_bf16")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t; +template [[host_name("kernel_cpy_contig_f16_bf16")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t; +template [[host_name("kernel_cpy_contig_bf16_f16")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t; #endif template @@ -8064,6 +8100,8 @@ template [[host_name("kernel_cpy_f16_f16")]] kernel kernel_cpy_t kernel_cpy_t_ #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_cpy_bf16_f32")]] kernel kernel_cpy_t kernel_cpy_t_t; template [[host_name("kernel_cpy_bf16_bf16")]] kernel kernel_cpy_t kernel_cpy_t_t; +template [[host_name("kernel_cpy_f16_bf16")]] kernel kernel_cpy_t kernel_cpy_t_t; +template [[host_name("kernel_cpy_bf16_f16")]] kernel kernel_cpy_t kernel_cpy_t_t; #endif templateassets), @@ -757,8 +785,12 @@ struct BreezeGeneratorRuntime::Impl { execution.backend(), config.depth_head_dim, attention_preference); engine::debug::trace_log_scalar("breeze_tts.attention.allow_backbone_flash", allow_backbone_flash); engine::debug::trace_log_scalar("breeze_tts.attention.allow_depth_flash", allow_depth_flash); - backbone_runtime_config = backbone_config(config, execution.backend_type(), graph_arena_bytes, allow_backbone_flash); - depth_runtime_config = depth_config(config, execution.backend_type(), graph_arena_bytes, allow_depth_flash); + const bool bf16_reference = bf16_reference_enabled(bf16_activations, execution.backend_type()); + engine::debug::trace_log_scalar("breeze_tts.bf16_activations", bf16_reference); + backbone_runtime_config = backbone_config( + config, execution.backend_type(), graph_arena_bytes, allow_backbone_flash, bf16_reference); + depth_runtime_config = depth_config( + config, execution.backend_type(), graph_arena_bytes, allow_depth_flash, bf16_reference); weights = load_weights(*this->assets, execution, weight_context_bytes, storage_type, backbone_runtime_config); backbone_cond = std::make_unique(execution, backbone_runtime_config, weights->backbone); backbone_uncond = std::make_unique(execution, backbone_runtime_config, weights->backbone); @@ -1437,9 +1469,16 @@ BreezeGeneratorRuntime::BreezeGeneratorRuntime( size_t graph_arena_bytes, size_t weight_context_bytes, engine::assets::TensorStorageType storage_type, - engine::core::AttentionPreference attention_preference) + engine::core::AttentionPreference attention_preference, + Bf16ActivationMode bf16_activations) : impl_(std::make_unique( - std::move(assets), execution, graph_arena_bytes, weight_context_bytes, storage_type, attention_preference)) {} + std::move(assets), + execution, + graph_arena_bytes, + weight_context_bytes, + storage_type, + attention_preference, + bf16_activations)) {} BreezeGeneratorRuntime::~BreezeGeneratorRuntime() = default; diff --git a/src/models/breeze_tts/session.cpp b/src/models/breeze_tts/session.cpp index 6fe4e983b..c13688fc0 100644 --- a/src/models/breeze_tts/session.cpp +++ b/src/models/breeze_tts/session.cpp @@ -58,6 +58,23 @@ core::AttentionPreference attention_preference_from_options(const runtime::Sessi return core::AttentionPreference::Auto; } +Bf16ActivationMode bf16_activation_mode_from_options(const runtime::SessionOptions & options) { + const auto value = runtime::find_option(options.options, {"bf16_activations", "breeze_tts.bf16_activations"}); + if (!value.has_value()) { + return Bf16ActivationMode::Auto; + } + if (*value == "auto") { + return Bf16ActivationMode::Auto; + } + if (*value == "on") { + return Bf16ActivationMode::On; + } + if (*value == "off") { + return Bf16ActivationMode::Off; + } + throw std::runtime_error("BreezeTTS bf16_activations must be auto, on or off (got '" + *value + "')"); +} + void trace_attention_preference(core::AttentionPreference preference) { const char * name = "auto"; if (preference == core::AttentionPreference::Flash) { @@ -75,9 +92,11 @@ void validate_session_options( // Older standalone GGUF packages embed a v1 contract that predates this // backend-compatibility option; keep them usable while still validating // the option value in attention_preference_from_options(). - if (contract.session_option_keys.find("breeze_tts.attention") == - contract.session_option_keys.end()) { - validation_options.options.erase("breeze_tts.attention"); + // bf16_activations is handled the same way. + for (const char * key : {"breeze_tts.attention", "bf16_activations", "breeze_tts.bf16_activations"}) { + if (contract.session_option_keys.find(key) == contract.session_option_keys.end()) { + validation_options.options.erase(key); + } } runtime::validate_spec_backed_session_options(validation_options, contract, kFamily, kModelName); } @@ -173,13 +192,15 @@ BreezeTTSSession::BreezeTTSSession( 2048ull * 1024ull * 1024ull); const auto attention_preference = attention_preference_from_options(options); trace_attention_preference(attention_preference); + const auto bf16_activations = bf16_activation_mode_from_options(options); generator_ = std::make_unique( assets_, execution_context(), graph_arena_bytes, weight_context_bytes, storage_type, - attention_preference); + attention_preference, + bf16_activations); } BreezeTTSSession::~BreezeTTSSession() = default;