From d7607ad2730ff26b5cb8730354179a4d42dc45d1 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:47:06 -0400 Subject: [PATCH] [Bug] Fix deepseek v4 OOM issue (#44914) Signed-off-by: yewentao256 --- vllm/models/deepseek_v4/quant_config.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/vllm/models/deepseek_v4/quant_config.py b/vllm/models/deepseek_v4/quant_config.py index d6d650b517..d6e1619cac 100644 --- a/vllm/models/deepseek_v4/quant_config.py +++ b/vllm/models/deepseek_v4/quant_config.py @@ -7,7 +7,11 @@ from __future__ import annotations from typing import TYPE_CHECKING from vllm.config import get_current_vllm_config -from vllm.model_executor.layers.fused_moe import MoERunner, UnquantizedFusedMoEMethod +from vllm.model_executor.layers.fused_moe import ( + MoERunner, + RoutedExperts, + UnquantizedFusedMoEMethod, +) from vllm.model_executor.layers.quantization import QuantizationMethods from vllm.model_executor.layers.quantization.fp8 import Fp8Config from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod @@ -129,7 +133,7 @@ class DeepseekV4FP8Config(Fp8Config): return None def get_quant_method(self, layer, prefix): - if isinstance(layer, MoERunner): + if isinstance(layer, (MoERunner, RoutedExperts)): if is_layer_skipped( prefix=prefix, ignored_layers=self.ignored_layers, @@ -152,6 +156,9 @@ class DeepseekV4FP8Config(Fp8Config): return super().get_quant_method(layer, prefix) def is_mxfp4_quant(self, prefix, layer): - if not isinstance(layer, MoERunner) or self.expert_dtype != "fp4": + if ( + not isinstance(layer, (MoERunner, RoutedExperts)) + or self.expert_dtype != "fp4" + ): return False return self.moe_quant_algo != "NVFP4"