[Bugfix] Limit GlmMoeDsaForCausalLM fp8 KV cache default to SM100 (#60281)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Misha Goin
2026-10-07 13:37:13 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 08567505b0
commit a982c81e3e
+16 -8
View File
@@ -46,14 +46,22 @@ class GlmMoeDsaForCausalLM(VerifyAndUpdateConfig):
from vllm.platforms import current_platform from vllm.platforms import current_platform
cache_config = vllm_config.cache_config cache_config = vllm_config.cache_config
if cache_config.cache_dtype == "auto": if (
if current_platform.is_xpu(): cache_config.cache_dtype == "auto"
cache_config.cache_dtype = "bfloat16" and current_platform.is_cuda()
elif current_platform.is_cuda_alike(): and current_platform.is_device_capability_family(100)
capability = current_platform.get_device_capability() ):
cache_config.cache_dtype = ( cache_config.cache_dtype = "fp8_e4m3"
"fp8_e4m3" if capability and capability.major >= 10 else "bfloat16" logger.info_once("Using fp8 kv-cache for GlmMoeDsaForCausalLM on SM10x")
) # MTP shares the target's KV cache layout; a separate draft model
# must not inherit a default it was never validated with.
spec_config = vllm_config.speculative_config
if (
spec_config is not None
and spec_config.method != "mtp"
and spec_config.kv_cache_dtype is None
):
spec_config.kv_cache_dtype = "auto"
# For Glm-Moe-DSA, qrep + a2a is better than the default all-gather + ag-rs # For Glm-Moe-DSA, qrep + a2a is better than the default all-gather + ag-rs
# in most cases. # in most cases.