breeze: enable bf16 activation rounding on Metal

docs/models/breeze_tts.md

@@ -97,6 +97,7 @@ audiocpp_cli \
 |---|---|---:|---|
 | `breeze_tts.reference_cache_slots` | integer >= 0 | `1` | Prepared reference-audio cache slots. |
 | `breeze_tts.attention` | `auto`, `flash`, `eager` | `auto` | Attention kernel. `auto` uses flash except on Volta/Turing GPUs (e.g. V100), where it falls back to eager to avoid missing MMA kernels. |
+| `breeze_tts.bf16_activations` | `auto`, `on`, `off` | `auto` | Reference bf16 activation rounding (and, on Metal, the bf16 KV cache). `auto` is on for CUDA/HIP/Vulkan and off on Metal; see [Metal and the reference bf16 path](#metal-and-the-reference-bf16-path). |
 | `weight_type` | `native`, `f32`, `f16`, `bf16`, `q8_0`, `q4_0`, `q4_k` | `native` | Weight storage type; quantized types convert at load time from the BF16 package. |
 
 BreezeTTS streaming is incremental by default. It emits audio events from the
@@ -126,3 +127,36 @@ and HIP alike: `q8_0` cut the fixed 100-token regression case from RTF ~1.5 to
 voice-design regression cases. Counter to intuition, fp32 is the one
 configuration known to be *worse* for this model (mispronunciations and
 runaway repetition), because the model is trained and tuned in bf16.
+
+## Metal and the reference bf16 path
+
+The official BreezeTTS 2 inference rounds activations to bf16 at every decoder
+stage and keeps a bf16 KV cache. CUDA, HIP and Vulkan match that by default; the
+casts are cheap enough there. On Metal the same rounding costs a visible share
+of the AR loop — even after adding the fused round-to-bf16 unary op for Metal
+(which removes the f32 -> bf16 -> f32 cast pair at every rounding point) the
+reference path measured roughly **20% slower on the AR component** on a Mac
+mini M4, and about the same on an M4 MacBook Air — so on Metal it is **opt-in**:
+
+```bash
+# Reference bf16 parity on Metal (slower, matches the official implementation)
+audiocpp_cli --task tts --family breeze_tts \
+  --model models/Breeze-TTS-2-GGUF/breeze-tts-2-q8_0.gguf \
+  --backend metal \
+  --text "Welcome to the local voice demo." \
+  --session-option breeze_tts.bf16_activations=on \
+  --out breeze_tts_bf16.wav
+```
+
+| Value | Behavior |
+|---|---|
+| `auto` (default) | on for CUDA/HIP/Vulkan, off on Metal |
+| `on` | bf16 activation rounding on every GPU backend; on Metal it also switches the KV cache to bf16 |
+| `off` | f32 activations; Metal/CUDA/Vulkan keep an f16 KV cache (HIP keeps bf16) |
+
+The trade-off is real in both directions: the Metal default (`auto` -> off) is
+the faster path but is the same f32 configuration this page warns about above,
+so if a prompt mispronounces or collapses into repetition, re-run it with
+`bf16_activations=on` before changing anything else. On non-Metal GPUs the
+default already matches the reference, and `off` is only useful for A/B
+measurements.

external/ggml/src/ggml-metal/ggml-metal-device.cpp

@@ -320,6 +320,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary(ggml_metal
                 case GGML_UNARY_OP_ROUND:       op_num = OP_UNARY_NUM_ROUND;       break;
                 case GGML_UNARY_OP_TRUNC:       op_num = OP_UNARY_NUM_TRUNC;       break;
                 case GGML_UNARY_OP_XIELU:       op_num = OP_UNARY_NUM_XIELU;       break;
+                case GGML_UNARY_OP_ROUND_BF16:  op_num = OP_UNARY_NUM_ROUND_BF16;  break;
                 default: GGML_ABORT("fatal error");
             } break;
         default: GGML_ABORT("fatal error");

external/ggml/src/ggml-metal/ggml-metal-device.m

@@ -1091,6 +1091,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                 case GGML_UNARY_OP_TRUNC:
                 case GGML_UNARY_OP_XIELU:
                     return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
+                case GGML_UNARY_OP_ROUND_BF16:
+                    // Fused round-to-bf16: f32/f16/bf16 source, f32 result. The rounding
+                    // itself is integer math, so only reading a bf16 source needs
+                    // hardware support; other devices keep the cast round trip.
+                    return op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]) &&
+                           (op->src[0]->type == GGML_TYPE_F32 ||
+                            op->src[0]->type == GGML_TYPE_F16 ||
+                            (op->src[0]->type == GGML_TYPE_BF16 && has_bfloat));
                 default:
                     return false;
             }
@@ -1279,14 +1287,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                         switch (op->type) {
                             case GGML_TYPE_F32:
                             case GGML_TYPE_F16:
-                                return true;
+                            case GGML_TYPE_BF16:                                return true;
                             default:
                                 return false;
                         }
                     case GGML_TYPE_BF16:
                         switch (op->type) {
                             case GGML_TYPE_F32:
-                            case GGML_TYPE_BF16:
+                            case GGML_TYPE_F16:                            case GGML_TYPE_BF16:
                                 return true;
                             default:
                                 return false;

external/ggml/src/ggml-metal/ggml-metal-impl.h

@@ -141,6 +141,7 @@
 #define OP_UNARY_NUM_ROUND       119
 #define OP_UNARY_NUM_TRUNC       120
 #define OP_UNARY_NUM_XIELU       121
+#define OP_UNARY_NUM_ROUND_BF16  122
 
 #define OP_SUM_ROWS_NUM_SUM_ROWS 10
 #define OP_SUM_ROWS_NUM_MEAN     11

external/ggml/src/ggml-metal/ggml-metal.metal

@@ -1013,6 +1013,28 @@ template<> inline float4 elu_approx<float4>(float4 x) {
 constant short FC_unary_op [[function_constant(FC_UNARY + 0)]];
 constant bool  FC_unary_cnt[[function_constant(FC_UNARY + 1)]];
 
+// GGML_UNARY_OP_ROUND_BF16 semantics: round-to-nearest-even f32 -> bf16 -> f32,
+// mirroring ggml's ggml_compute_fp32_to_bf16 (NaNs are forced quiet). Doing it with
+// integer math keeps the op available on Metal devices without bf16 hardware support.
+static inline uint bf16_round_bits(uint u) {
+    if ((u & 0x7fffffffu) > 0x7f800000u) {
+        return ((u >> 16) | 64u) << 16;
+    }
+    return (u + (0x7fffu + ((u >> 16) & 1u))) & 0xffff0000u;
+}
+
+static inline uint4 bf16_round_bits(uint4 u) {
+    const uint4 sign_mask = uint4(0x7fffffffu);
+    const uint4 inf_bits  = uint4(0x7f800000u);
+    const uint4 high      = u >> 16;
+    const uint4 rounded   = (u + (0x7fffu + (high & 1u))) & 0xffff0000u;
+    const uint4 quiet     = (high | 64u) << 16;
+    return select(rounded, quiet, (u & sign_mask) > inf_bits);
+}
+
+static inline float  bf16_round_f32(float  x) { return as_type<float>(bf16_round_bits(as_type<uint>(x))); }
+static inline float4 bf16_round_f32(float4 x) { return as_type<float4>(bf16_round_bits(as_type<uint4>(x))); }
+
 template <typename T0, typename T, typename TC>
 kernel void kernel_unary_impl(
         constant ggml_metal_kargs_unary & args,
@@ -1186,16 +1208,28 @@ kernel void kernel_unary_impl(
             const TC y_neg   = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi;
             dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg);
         }
+
+        if (FC_OP == OP_UNARY_NUM_ROUND_BF16) {
+            dst_ptr[i0] = (T) bf16_round_f32(x);
+        }
     }
 
 #undef FC_OP
 #undef FC_CNT
 }
 
-typedef decltype(kernel_unary_impl<float, float, float>) kernel_unary_t;
-
-template [[host_name("kernel_unary_f32_f32")]]   kernel kernel_unary_t kernel_unary_impl<float,  float,  float>;
-template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl<float4, float4, float4>;
+typedef decltype(kernel_unary_impl<float, float, float>) kernel_unary_t;
+
+template [[host_name("kernel_unary_f32_f32")]]   kernel kernel_unary_t kernel_unary_impl<float,  float,  float>;
+template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl<float4, float4, float4>;
+// GGML_UNARY_OP_ROUND_BF16 widens f16/bf16 inputs to f32 while rounding, so the
+// fused op needs the narrower source types as well.
+template [[host_name("kernel_unary_f16_f32")]]   kernel kernel_unary_t kernel_unary_impl<half,   float,  float>;
+template [[host_name("kernel_unary_f16_f32_4")]] kernel kernel_unary_t kernel_unary_impl<half4,  float4, float4>;
+#if defined(GGML_METAL_HAS_BF16)
+template [[host_name("kernel_unary_bf16_f32")]]   kernel kernel_unary_t kernel_unary_impl<bfloat,  float,  float>;
+template [[host_name("kernel_unary_bf16_f32_4")]] kernel kernel_unary_t kernel_unary_impl<bfloat4, float4, float4>;
+#endif
 template [[host_name("kernel_unary_f16_f16")]]   kernel kernel_unary_t kernel_unary_impl<half,   half,   float>;
 template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl<half4,  half4,  float4>;
 
@@ -7932,6 +7966,8 @@ template [[host_name("kernel_cpy_contig_f16_f16")]] kernel kernel_cpy_contig_t k
 #if defined(GGML_METAL_HAS_BF16)
 template [[host_name("kernel_cpy_contig_bf16_f32")]]  kernel kernel_cpy_contig_t kernel_cpy_contig_t_t<bfloat, float>;
 template [[host_name("kernel_cpy_contig_bf16_bf16")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t<bfloat, bfloat>;
+template [[host_name("kernel_cpy_contig_f16_bf16")]]  kernel kernel_cpy_contig_t kernel_cpy_contig_t_t<half,   bfloat>;
+template [[host_name("kernel_cpy_contig_bf16_f16")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t<bfloat, half>;
 #endif
 
 template<typename T>
@@ -8064,6 +8100,8 @@ template [[host_name("kernel_cpy_f16_f16")]]   kernel kernel_cpy_t kernel_cpy_t_
 #if defined(GGML_METAL_HAS_BF16)
 template [[host_name("kernel_cpy_bf16_f32")]]  kernel kernel_cpy_t kernel_cpy_t_t<bfloat,  float>;
 template [[host_name("kernel_cpy_bf16_bf16")]] kernel kernel_cpy_t kernel_cpy_t_t<bfloat,  bfloat>;
+template [[host_name("kernel_cpy_f16_bf16")]]  kernel kernel_cpy_t kernel_cpy_t_t<half,    bfloat>;
+template [[host_name("kernel_cpy_bf16_f16")]]  kernel kernel_cpy_t kernel_cpy_t_t<bfloat,  half>;
 #endif
 
 template<short QK,

include/engine/models/breeze_tts/generator.h

@@ -34,6 +34,17 @@ struct BreezeStreamEvent {
     bool done = false;
 };
 
+// BreezeTTS 2's reference inference rounds activations to bf16 (and keeps a bf16
+// KV cache). That is what the model was trained with, but on backends without a
+// cheap fused cast it costs a visible share of the AR loop, so the choice is
+// explicit: 'auto' keeps the reference behavior on CUDA/HIP/Vulkan and stays on
+// the faster f32 path on Metal, 'on'/'off' force it either way.
+enum class Bf16ActivationMode {
+    Auto,
+    On,
+    Off,
+};
+
 class BreezeGeneratorRuntime {
 public:
     BreezeGeneratorRuntime(
@@ -42,7 +53,8 @@ class BreezeGeneratorRuntime {
         size_t graph_arena_bytes,
         size_t weight_context_bytes,
         engine::assets::TensorStorageType storage_type,
-        engine::core::AttentionPreference attention_preference = engine::core::AttentionPreference::Auto);
+        engine::core::AttentionPreference attention_preference = engine::core::AttentionPreference::Auto,
+        Bf16ActivationMode bf16_activations = Bf16ActivationMode::Auto);
     ~BreezeGeneratorRuntime();
 
     engine::runtime::AudioBuffer generate(const BreezeGenerationRequest & request);

model_specs/breeze_tts.json

@@ -185,6 +185,14 @@
         "required": false,
         "values": ["auto", "flash", "eager"],
         "default": "auto"
+      },
+      {
+        "name": "bf16_activations",
+        "type": "enum",
+        "description": "Reference bf16 activation rounding as the official BreezeTTS 2 inference does it. auto = on for CUDA/HIP/Vulkan and off on Metal, where the cast cost is visible even with the fused round-to-bf16 op (~20% slower AR loop); on/off force it either way. On Metal, on also switches the KV cache to bf16 to match the reference implementation.",
+        "required": false,
+        "values": ["auto", "on", "off"],
+        "default": "auto"
       }
     ],
     "load": []