[Bug] Fix deepseek v4 OOM issue (#44914)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-06-09 15:47:06 -07:00
committed by GitHub
parent d955745d58
commit d7607ad273
+10 -3
View File
@@ -7,7 +7,11 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.fused_moe import MoERunner, UnquantizedFusedMoEMethod
from vllm.model_executor.layers.fused_moe import (
MoERunner,
RoutedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
@@ -129,7 +133,7 @@ class DeepseekV4FP8Config(Fp8Config):
return None
def get_quant_method(self, layer, prefix):
if isinstance(layer, MoERunner):
if isinstance(layer, (MoERunner, RoutedExperts)):
if is_layer_skipped(
prefix=prefix,
ignored_layers=self.ignored_layers,
@@ -152,6 +156,9 @@ class DeepseekV4FP8Config(Fp8Config):
return super().get_quant_method(layer, prefix)
def is_mxfp4_quant(self, prefix, layer):
if not isinstance(layer, MoERunner) or self.expert_dtype != "fp4":
if (
not isinstance(layer, (MoERunner, RoutedExperts))
or self.expert_dtype != "fp4"
):
return False
return self.moe_quant_algo != "NVFP4"