[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50 (#38423)
Signed-off-by: johnnynunez <johnnynuca14@gmail.com> Signed-off-by: Johnny <johnnynuca14@gmail.com>
This commit is contained in:
@@ -406,6 +406,11 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
if self.routing_method_type == RoutingMethodType.DeepSeekV3:
|
||||
router_logits = router_logits.to(torch.float32)
|
||||
|
||||
# Currently FI requires bfloat16 routing bias.
|
||||
# https://github.com/flashinfer-ai/flashinfer/issues/2909
|
||||
if e_score_correction_bias is not None:
|
||||
e_score_correction_bias = e_score_correction_bias.to(torch.bfloat16)
|
||||
|
||||
out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
|
||||
@@ -5,6 +5,7 @@ import flashinfer
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -27,6 +28,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TrtLlmNvFp4ExpertsBase:
|
||||
"""
|
||||
@@ -315,6 +318,11 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
else router_logits
|
||||
)
|
||||
|
||||
# Currently FI requires bfloat16 routing bias.
|
||||
# https://github.com/flashinfer-ai/flashinfer/issues/2909
|
||||
if e_score_correction_bias is not None:
|
||||
e_score_correction_bias = e_score_correction_bias.to(torch.bfloat16)
|
||||
|
||||
# Invoke kernel.
|
||||
return flashinfer.fused_moe.trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
|
||||
@@ -361,7 +361,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
fc1_expert_weights = w1
|
||||
fc2_expert_weights = w2
|
||||
else:
|
||||
quant_scales = None
|
||||
quant_scales = []
|
||||
a1q_scale = None
|
||||
fc1_expert_weights = w1
|
||||
fc2_expert_weights = w2
|
||||
|
||||
@@ -70,7 +70,6 @@ def map_unquantized_backend(runner_backend: MoEBackend) -> UnquantizedMoeBackend
|
||||
|
||||
def select_unquantized_moe_backend(
|
||||
moe_config: FusedMoEConfig,
|
||||
use_ep: bool,
|
||||
use_dp: bool,
|
||||
) -> UnquantizedMoeBackend:
|
||||
"""
|
||||
@@ -96,7 +95,6 @@ def select_unquantized_moe_backend(
|
||||
# FlashInfer CUTLASS MoE is only supported on Hopper and later GPUS
|
||||
flashinfer_cutlass_available = (
|
||||
has_flashinfer_cutlass_fused_moe()
|
||||
and use_ep
|
||||
and (not use_dp)
|
||||
and current_platform.has_device_capability(90)
|
||||
)
|
||||
@@ -161,9 +159,9 @@ def select_unquantized_moe_backend(
|
||||
"to enable it for better performance.",
|
||||
scope="local",
|
||||
)
|
||||
elif use_ep and (not use_dp):
|
||||
elif not use_dp and flashinfer_cutlass_available:
|
||||
logger.info_once(
|
||||
"FlashInfer MoE is available for EP"
|
||||
"FlashInfer CUTLASS MoE is available"
|
||||
" but not enabled, consider setting"
|
||||
" VLLM_USE_FLASHINFER_MOE_FP16=1 to enable it.",
|
||||
scope="local",
|
||||
|
||||
@@ -61,7 +61,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
super().__init__(moe)
|
||||
self.unquantized_backend = select_unquantized_moe_backend(
|
||||
moe_config=self.moe,
|
||||
use_ep=self.moe.moe_parallel_config.use_ep,
|
||||
use_dp=self.moe.moe_parallel_config.dp_size > 1,
|
||||
)
|
||||
|
||||
|
||||
@@ -55,8 +55,16 @@ def select_nvfp4_linear_backend() -> NvFp4LinearBackend:
|
||||
elif envs.VLLM_USE_NVFP4_CT_EMULATIONS:
|
||||
backend = NvFp4LinearBackend.EMULATION
|
||||
elif envs.VLLM_NVFP4_GEMM_BACKEND is None:
|
||||
# Auto-select best available backend
|
||||
if current_platform.has_device_capability(100) and has_flashinfer():
|
||||
# Auto-select best available backend.
|
||||
# cutlass_fp4_supported() checks that the vLLM NVFP4 kernels (both
|
||||
# quantization and GEMM) were compiled for the current SM version.
|
||||
# FlashInfer backends still rely on the vLLM quantization kernels,
|
||||
# so we gate them on the same check.
|
||||
if (
|
||||
cutlass_fp4_supported()
|
||||
and current_platform.has_device_capability(100)
|
||||
and has_flashinfer()
|
||||
):
|
||||
backend = NvFp4LinearBackend.FLASHINFER_CUTLASS
|
||||
elif cutlass_fp4_supported():
|
||||
backend = NvFp4LinearBackend.VLLM_CUTLASS
|
||||
@@ -72,6 +80,10 @@ def select_nvfp4_linear_backend() -> NvFp4LinearBackend:
|
||||
NvFp4LinearBackend.FLASHINFER_CUDNN,
|
||||
):
|
||||
assert has_flashinfer(), f"FlashInfer is required for {backend}"
|
||||
assert cutlass_fp4_supported(), (
|
||||
f"{backend} requires vLLM NVFP4 quantization kernels compiled "
|
||||
f"for the current GPU (SM {current_platform.get_device_capability()})"
|
||||
)
|
||||
elif backend == NvFp4LinearBackend.VLLM_CUTLASS:
|
||||
assert cutlass_fp4_supported(), f"Cutlass is required for {backend}"
|
||||
elif backend == NvFp4LinearBackend.MARLIN:
|
||||
|
||||
Reference in New Issue
Block a user