[GLM5.3 Perf] Switch to fp8 kv cache by default for GLM, 2.3%~5.5% E2E Throughput Improvement (#60140)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-06 12:54:20 -04:00
committed by GitHub
parent f8d92cae41
commit fba31f31a9
+12
View File
@@ -43,6 +43,18 @@ class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
class GlmMoeDsaForCausalLM(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
from vllm.platforms import current_platform
cache_config = vllm_config.cache_config
if cache_config.cache_dtype == "auto":
if current_platform.is_xpu():
cache_config.cache_dtype = "bfloat16"
elif current_platform.is_cuda_alike():
capability = current_platform.get_device_capability()
cache_config.cache_dtype = (
"fp8_e4m3" if capability and capability.major >= 10 else "bfloat16"
)
# For Glm-Moe-DSA, qrep + a2a is better than the default all-gather + ag-rs
# in most cases.
vllm_config.parallel_config.set_dcp_defaults(