[Bug] Fix deepseek v4 OOM issue (#44914)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -7,7 +7,11 @@ from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner, UnquantizedFusedMoEMethod
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
MoERunner,
|
||||
RoutedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
@@ -129,7 +133,7 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
return None
|
||||
|
||||
def get_quant_method(self, layer, prefix):
|
||||
if isinstance(layer, MoERunner):
|
||||
if isinstance(layer, (MoERunner, RoutedExperts)):
|
||||
if is_layer_skipped(
|
||||
prefix=prefix,
|
||||
ignored_layers=self.ignored_layers,
|
||||
@@ -152,6 +156,9 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
return super().get_quant_method(layer, prefix)
|
||||
|
||||
def is_mxfp4_quant(self, prefix, layer):
|
||||
if not isinstance(layer, MoERunner) or self.expert_dtype != "fp4":
|
||||
if (
|
||||
not isinstance(layer, (MoERunner, RoutedExperts))
|
||||
or self.expert_dtype != "fp4"
|
||||
):
|
||||
return False
|
||||
return self.moe_quant_algo != "NVFP4"
|
||||
|
||||
Reference in New Issue
Block a user