nixl_ep: Skip post-receive quantization for NVFP4 (#45606)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -49,6 +49,9 @@ class FlashInferCuteDSLBatchedExperts(mk.FusedMoEExpertsModular):
|
||||
"Only nvfp4 quantization are currently supported."
|
||||
)
|
||||
self.out_dtype = moe_config.in_dtype
|
||||
self.use_deep_ep_ll_nvfp4_dispatch = (
|
||||
envs.VLLM_DEEPEPLL_NVFP4_DISPATCH and moe_config.use_deepep_ll_kernels
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
|
||||
@@ -123,7 +126,7 @@ class FlashInferCuteDSLBatchedExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
# We use global_num_experts due to how moe_align_block_size handles
|
||||
# expert_maps.
|
||||
K_dim = K * 2 if envs.VLLM_DEEPEPLL_NVFP4_DISPATCH else K
|
||||
K_dim = K * 2 if self.use_deep_ep_ll_nvfp4_dispatch else K
|
||||
output_shape = (local_num_experts, M, K_dim)
|
||||
workspace2 = (local_num_experts, M, N)
|
||||
workspace1 = output_shape
|
||||
@@ -161,11 +164,11 @@ class FlashInferCuteDSLBatchedExperts(mk.FusedMoEExpertsModular):
|
||||
assert self.w2_scale.ndim == 3
|
||||
|
||||
input_global_scale = (
|
||||
None if envs.VLLM_DEEPEPLL_NVFP4_DISPATCH else self.a1_gscale
|
||||
None if self.use_deep_ep_ll_nvfp4_dispatch else self.a1_gscale
|
||||
)
|
||||
flashinfer_hidden_states = (
|
||||
(hidden_states, a1q_scale)
|
||||
if envs.VLLM_DEEPEPLL_NVFP4_DISPATCH
|
||||
if self.use_deep_ep_ll_nvfp4_dispatch
|
||||
else hidden_states
|
||||
)
|
||||
flashinfer_cutedsl_moe_masked(
|
||||
|
||||
@@ -6,7 +6,6 @@ import nixl_ep
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
|
||||
from vllm.logger import init_logger
|
||||
@@ -192,13 +191,9 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
x = x.view((-1, hidden_dim))
|
||||
q_dtype = quant_config.quant_dtype
|
||||
|
||||
moe_backend = get_current_vllm_config().kernel_config.moe_backend
|
||||
if moe_backend == "flashinfer_cutedsl":
|
||||
logger.info_once(
|
||||
"Skip quantization when using FlashInfer CUTEDSL "
|
||||
"(--moe-backend flashinfer_cutedsl) for ModelOptNvFp4FusedMoE."
|
||||
)
|
||||
if q_dtype == "nvfp4":
|
||||
q_dtype = None
|
||||
logger.debug_once("Using NIXL EP bfloat16 dispatch for NVFP4 MoE.")
|
||||
|
||||
x, x_scales = moe_kernel_quantize_input(
|
||||
x,
|
||||
|
||||
Reference in New Issue
Block a user