From a982c81e3ed337f76b116338192c83f776a3225c Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Wed, 7 Oct 2026 16:37:13 -0400 Subject: [PATCH] [Bugfix] Limit GlmMoeDsaForCausalLM fp8 KV cache default to SM100 (#60281) Signed-off-by: mgoin Co-authored-by: Claude Opus 5.5 --- vllm/model_executor/models/config.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 97b9bfa0d3..1a1bf92061 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -46,14 +46,22 @@ class GlmMoeDsaForCausalLM(VerifyAndUpdateConfig): from vllm.platforms import current_platform cache_config = vllm_config.cache_config - if cache_config.cache_dtype == "auto": - if current_platform.is_xpu(): - cache_config.cache_dtype = "bfloat16" - elif current_platform.is_cuda_alike(): - capability = current_platform.get_device_capability() - cache_config.cache_dtype = ( - "fp8_e4m3" if capability and capability.major >= 10 else "bfloat16" - ) + if ( + cache_config.cache_dtype == "auto" + and current_platform.is_cuda() + and current_platform.is_device_capability_family(100) + ): + cache_config.cache_dtype = "fp8_e4m3" + logger.info_once("Using fp8 kv-cache for GlmMoeDsaForCausalLM on SM10x") + # MTP shares the target's KV cache layout; a separate draft model + # must not inherit a default it was never validated with. + spec_config = vllm_config.speculative_config + if ( + spec_config is not None + and spec_config.method != "mtp" + and spec_config.kv_cache_dtype is None + ): + spec_config.kv_cache_dtype = "auto" # For Glm-Moe-DSA, qrep + a2a is better than the default all-gather + ag-rs # in most cases.