From 852f567444cf8c206219edb7b2c42aec55fc41cf Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Fri, 15 May 2026 20:15:52 -0400 Subject: [PATCH] [Bugfix] Respect explicit --kv-cache-dtype over checkpoint kv_cache_scheme (#42782) Signed-off-by: mgoin --- vllm/model_executor/layers/attention/attention.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index b993de0501..2e17a55ce7 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -228,9 +228,14 @@ class Attention(nn.Module, AttentionLayerBase): kv_cache_dtype = "auto" calculate_kv_scales = False - # llm-compressor mdls need to set cache_dtype to "fp8" manually. + # llm-compressor models declare an FP8 KV-cache scheme in their + # checkpoint config. Honor it only when the user did not explicitly + # pick a kv_cache_dtype; an explicit choice (e.g. bfloat16) must win. + # The "auto" case is normally resolved upstream in + # resolve_kv_cache_dtype_string, but we re-apply here defensively in + # case anything bypassed that path. kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None) - if kv_cache_scheme is not None: + if kv_cache_scheme is not None and kv_cache_dtype == "auto": kv_cache_dtype = "fp8" calculate_kv_scales = False if cache_config is not None: