From fba31f31a9d14a8fbe52e4246ba5b3fcd1a9ba63 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:54:20 -0400 Subject: [PATCH] [GLM5.3 Perf] Switch to fp8 kv cache by default for GLM, 2.3%~5.5% E2E Throughput Improvement (#60140) Signed-off-by: yewentao256 --- vllm/model_executor/models/config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 8831c5f6c4..d80663731d 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -43,6 +43,18 @@ class DeepseekV32ForCausalLM(VerifyAndUpdateConfig): class GlmMoeDsaForCausalLM(VerifyAndUpdateConfig): @staticmethod def verify_and_update_config(vllm_config: "VllmConfig") -> None: + from vllm.platforms import current_platform + + cache_config = vllm_config.cache_config + if cache_config.cache_dtype == "auto": + if current_platform.is_xpu(): + cache_config.cache_dtype = "bfloat16" + elif current_platform.is_cuda_alike(): + capability = current_platform.get_device_capability() + cache_config.cache_dtype = ( + "fp8_e4m3" if capability and capability.major >= 10 else "bfloat16" + ) + # For Glm-Moe-DSA, qrep + a2a is better than the default all-gather + ag-rs # in most cases. vllm_config.parallel_config.set_dcp_defaults(