[Bugfix] Respect explicit --kv-cache-dtype over checkpoint kv_cache_scheme (#42782)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-05-15 17:15:52 -07:00
committed by GitHub
parent b2a27b82d9
commit 852f567444
@@ -228,9 +228,14 @@ class Attention(nn.Module, AttentionLayerBase):
kv_cache_dtype = "auto"
calculate_kv_scales = False
# llm-compressor mdls need to set cache_dtype to "fp8" manually.
# llm-compressor models declare an FP8 KV-cache scheme in their
# checkpoint config. Honor it only when the user did not explicitly
# pick a kv_cache_dtype; an explicit choice (e.g. bfloat16) must win.
# The "auto" case is normally resolved upstream in
# resolve_kv_cache_dtype_string, but we re-apply here defensively in
# case anything bypassed that path.
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
if kv_cache_scheme is not None:
if kv_cache_scheme is not None and kv_cache_dtype == "auto":
kv_cache_dtype = "fp8"
calculate_kv_scales = False
if cache_config is not None: