llama : disable FA if KV head size do not match

2026-03-05 14:33:24 +02:00 · 2024-06-17 19:20:24 +03:00
1 changed files with 5 additions and 0 deletions
--- a/llama.cpp
+++ b/llama.cpp
@@ -16260,6 +16260,11 @@ struct llama_context * llama_new_context_with_model(
        params.flash_attn = false;
    }

+    if (params.flash_attn && model->hparams.n_embd_head_k != model->hparams.n_embd_head_v) {
+        LLAMA_LOG_WARN("%s: flash_attn requires n_embd_head_k == n_embd_head_v - forcing off\n", __func__);
+        params.flash_attn = false;
+    }
+
    if (params.type_v != GGML_TYPE_F16 && !params.flash_attn) {
        LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__);
        return nullptr;