Support mnnvl all2allv from Flashinfer (#21003)
Signed-off-by: Shu Wang <shuw@nvidia.com> Signed-off-by: Shu Wang. <shuw@nvidia.com> Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
Tyler Michael Smith
parent
2dda3e35d0
commit
54e42b72db
@@ -8,7 +8,7 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_prepare_finalize import ( # noqa: E501
|
||||
FlashInferCutlassMoEPrepareAndFinalize)
|
||||
create_flashinfer_prepare_finalize)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP)
|
||||
from vllm.utils.flashinfer import (flashinfer_cutlass_fused_moe,
|
||||
@@ -108,7 +108,7 @@ class FlashInferExperts(mk.FusedMoEPermuteExpertsUnpermute):
|
||||
of each tuple must be the number of tokens.
|
||||
"""
|
||||
aq_m, aq_n = aq.shape
|
||||
workspace2 = ()
|
||||
workspace2 = (0, )
|
||||
output_shape = (aq_m, aq_n * 2) if self.quant_dtype != \
|
||||
torch.float8_e4m3fn else (aq_m, aq_n)
|
||||
workspace_dtype = a.dtype
|
||||
@@ -192,9 +192,8 @@ def flashinfer_cutlass_moe_fp4(
|
||||
expert_map: Optional[torch.Tensor] = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
) -> torch.Tensor:
|
||||
|
||||
fused_experts = mk.FusedMoEModularKernel(
|
||||
FlashInferCutlassMoEPrepareAndFinalize(use_dp=False),
|
||||
create_flashinfer_prepare_finalize(use_dp=False),
|
||||
FlashInferExperts(
|
||||
out_dtype=hidden_states.dtype,
|
||||
quant_config=quant_config,
|
||||
|
||||
@@ -5,7 +5,9 @@ from typing import Optional
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.distributed import get_dp_group
|
||||
from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.distributed.device_communicators.base_device_communicator import (
|
||||
All2AllManagerBase)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
@@ -18,6 +20,7 @@ def get_local_sizes():
|
||||
|
||||
|
||||
class FlashInferCutlassMoEPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):
|
||||
"""Base class for FlashInfer MoE prepare and finalize operations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -42,6 +45,39 @@ class FlashInferCutlassMoEPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):
|
||||
def num_dispatchers(self) -> int:
|
||||
return self.num_dispatchers_
|
||||
|
||||
def _apply_router_weight_on_input(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
"""Apply router weight on input if needed."""
|
||||
if apply_router_weight_on_input:
|
||||
topk = topk_ids.size(1)
|
||||
assert topk == 1, \
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
a1.mul_(topk_weights.to(a1.dtype))
|
||||
|
||||
|
||||
class FlashInferAllToAllMoEPrepareAndFinalize(
|
||||
FlashInferCutlassMoEPrepareAndFinalize):
|
||||
"""FlashInfer implementation using AllToAll communication."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
use_dp: bool,
|
||||
num_dispatchers: int = 1,
|
||||
):
|
||||
super().__init__(use_dp, num_dispatchers)
|
||||
self.alltoall_info = None
|
||||
|
||||
# Initialize all2all_manager only for DP case
|
||||
self.all2all_manager = None
|
||||
if self.use_dp:
|
||||
self.all2all_manager = get_ep_group(
|
||||
).device_communicator.all2all_manager
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
@@ -53,12 +89,84 @@ class FlashInferCutlassMoEPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> mk.PrepareResultType:
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
topk = topk_ids.size(1)
|
||||
# TODO: this only works for topK=1, will need to update for topK>1
|
||||
assert topk == 1, \
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
a1.mul_(topk_weights.to(a1.dtype))
|
||||
self._apply_router_weight_on_input(a1, topk_weights, topk_ids,
|
||||
apply_router_weight_on_input)
|
||||
|
||||
if not self.use_dp:
|
||||
# Non-DP case: standard quantization
|
||||
a1q, a1q_scale = moe_kernel_quantize_input(
|
||||
a1,
|
||||
quant_config.a1_gscale,
|
||||
quant_config.quant_dtype,
|
||||
quant_config.per_act_token_quant,
|
||||
quant_config.block_shape,
|
||||
is_fp4_scale_swizzled=not self.use_dp,
|
||||
)
|
||||
else:
|
||||
# DP case: use FlashInfer AllToAll
|
||||
global_num_tokens_cpu = get_local_sizes()
|
||||
top_k = topk_ids.size(1)
|
||||
|
||||
(self.alltoall_info, topk_ids, topk_weights, a1q,
|
||||
a1q_scale) = flashinfer_alltoall_dispatch(
|
||||
self.all2all_manager,
|
||||
global_num_tokens_cpu,
|
||||
a1,
|
||||
quant_config.a1_gscale,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
top_k,
|
||||
num_experts,
|
||||
quant_config,
|
||||
)
|
||||
|
||||
return a1q, a1q_scale, None, topk_ids, topk_weights
|
||||
|
||||
def finalize(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
fused_expert_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
) -> None:
|
||||
if self.use_dp:
|
||||
top_k = topk_ids.size(1)
|
||||
token_count = output.shape[0]
|
||||
fused_expert_output = flashinfer_alltoall_combine(
|
||||
self.all2all_manager,
|
||||
fused_expert_output,
|
||||
top_k=top_k,
|
||||
token_count=token_count,
|
||||
alltoall_info=self.alltoall_info,
|
||||
)
|
||||
output.copy_(fused_expert_output)
|
||||
|
||||
|
||||
class FlashInferAllGatherMoEPrepareAndFinalize(
|
||||
FlashInferCutlassMoEPrepareAndFinalize):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
use_dp: bool,
|
||||
num_dispatchers: int = 1,
|
||||
):
|
||||
super().__init__(use_dp, num_dispatchers)
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: Optional[torch.Tensor],
|
||||
apply_router_weight_on_input: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> mk.PrepareResultType:
|
||||
|
||||
self._apply_router_weight_on_input(a1, topk_weights, topk_ids,
|
||||
apply_router_weight_on_input)
|
||||
|
||||
a1q, a1q_scale = moe_kernel_quantize_input(
|
||||
a1,
|
||||
@@ -66,7 +174,6 @@ class FlashInferCutlassMoEPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):
|
||||
quant_config.quant_dtype,
|
||||
quant_config.per_act_token_quant,
|
||||
quant_config.block_shape,
|
||||
# Swizzling after communication
|
||||
is_fp4_scale_swizzled=not self.use_dp,
|
||||
)
|
||||
if self.use_dp:
|
||||
@@ -76,17 +183,117 @@ class FlashInferCutlassMoEPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):
|
||||
dim=0,
|
||||
sizes=get_local_sizes(),
|
||||
)
|
||||
a1_m, a1_n = a1q.shape
|
||||
a1q_scale = nvfp4_block_scale_interleave(a1q_scale)
|
||||
|
||||
return a1q, a1q_scale, None, topk_ids, topk_weights
|
||||
|
||||
def finalize(self, output: torch.Tensor, fused_expert_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor, topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce) -> None:
|
||||
def finalize(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
fused_expert_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
) -> None:
|
||||
|
||||
if self.use_dp:
|
||||
fused_expert_output = get_dp_group().reduce_scatterv(
|
||||
fused_expert_output, dim=0, sizes=get_local_sizes())
|
||||
output.copy_(fused_expert_output)
|
||||
|
||||
|
||||
def flashinfer_alltoall_dispatch(
|
||||
all2all_manager: All2AllManagerBase,
|
||||
global_num_tokens_cpu: list[int],
|
||||
x: torch.Tensor,
|
||||
gs: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
top_k: int,
|
||||
num_experts: int,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
from flashinfer.comm.trtllm_alltoall import MnnvlMoe
|
||||
assert (all2all_manager.ensure_alltoall_workspace_initialized()
|
||||
), "FlashInfer AllToAll workspace not available"
|
||||
|
||||
ep_rank = all2all_manager.rank
|
||||
ep_size = all2all_manager.world_size
|
||||
max_num_token = max(global_num_tokens_cpu
|
||||
) if global_num_tokens_cpu is not None else x.shape[0]
|
||||
alltoall_info, topk_ids, topk_weights, _ = (
|
||||
MnnvlMoe.mnnvl_moe_alltoallv_prepare_without_allgather(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
None,
|
||||
all2all_manager.prepare_workspace,
|
||||
max_num_token,
|
||||
ep_rank,
|
||||
ep_size,
|
||||
num_experts,
|
||||
num_experts,
|
||||
top_k,
|
||||
))
|
||||
|
||||
x, x_sf = moe_kernel_quantize_input(
|
||||
x,
|
||||
gs,
|
||||
quant_config.quant_dtype,
|
||||
quant_config.per_act_token_quant,
|
||||
quant_config.block_shape,
|
||||
is_fp4_scale_swizzled=False, # delay swizzle to after comm
|
||||
)
|
||||
x = MnnvlMoe.mnnvl_moe_alltoallv(
|
||||
x,
|
||||
alltoall_info,
|
||||
all2all_manager.workspace_tensor,
|
||||
ep_rank,
|
||||
ep_size,
|
||||
)
|
||||
|
||||
x_sf = MnnvlMoe.mnnvl_moe_alltoallv(
|
||||
x_sf,
|
||||
alltoall_info,
|
||||
all2all_manager.workspace_tensor,
|
||||
ep_rank,
|
||||
ep_size,
|
||||
)
|
||||
x_sf = nvfp4_block_scale_interleave(x_sf)
|
||||
return alltoall_info, topk_ids, topk_weights, x, x_sf
|
||||
|
||||
|
||||
def flashinfer_alltoall_combine(
|
||||
all2all_manager: All2AllManagerBase,
|
||||
output: torch.Tensor,
|
||||
top_k: int,
|
||||
token_count: int,
|
||||
alltoall_info,
|
||||
):
|
||||
from flashinfer.comm.trtllm_alltoall import MnnvlMoe
|
||||
assert (all2all_manager.ensure_alltoall_workspace_initialized()
|
||||
), "FlashInfer AllToAll workspace not available"
|
||||
return MnnvlMoe.mnnvl_moe_alltoallv_combine(
|
||||
output,
|
||||
alltoall_info,
|
||||
all2all_manager.workspace_tensor,
|
||||
ep_rank=all2all_manager.rank,
|
||||
ep_size=all2all_manager.world_size,
|
||||
top_k=top_k,
|
||||
token_count=token_count,
|
||||
)
|
||||
|
||||
|
||||
def create_flashinfer_prepare_finalize(
|
||||
use_dp: bool,
|
||||
use_nvfp4: bool = False,
|
||||
enable_alltoallv: bool = False,
|
||||
) -> FlashInferCutlassMoEPrepareAndFinalize:
|
||||
"""Factory function to create the appropriate FlashInfer implementation."""
|
||||
if use_nvfp4:
|
||||
if enable_alltoallv:
|
||||
return FlashInferAllToAllMoEPrepareAndFinalize(use_dp)
|
||||
else:
|
||||
return FlashInferAllGatherMoEPrepareAndFinalize(use_dp)
|
||||
# Fp8 only supports AllGather
|
||||
return FlashInferAllGatherMoEPrepareAndFinalize(use_dp)
|
||||
|
||||
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.fused_moe.config import (FusedMoEConfig,
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_prepare_finalize import ( # noqa: E501
|
||||
FlashInferCutlassMoEPrepareAndFinalize)
|
||||
create_flashinfer_prepare_finalize)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
|
||||
|
||||
@@ -51,7 +51,9 @@ def build_flashinfer_fp4_cutlass_moe_prepare_finalize(
|
||||
moe: FusedMoEConfig) -> mk.FusedMoEPrepareAndFinalize:
|
||||
"""Create a FlashInfer CUTLASS fused-MoE prepare finalize kernel"""
|
||||
use_dp = moe.moe_parallel_config.dp_size > 1
|
||||
return FlashInferCutlassMoEPrepareAndFinalize(use_dp)
|
||||
enable_alltoallv = envs.VLLM_ALL2ALL_BACKEND == "flashinfer_all2allv"
|
||||
return create_flashinfer_prepare_finalize(
|
||||
use_dp=use_dp, use_nvfp4=True, enable_alltoallv=enable_alltoallv)
|
||||
|
||||
|
||||
def select_nvfp4_gemm_impl(
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.fused_moe.config import (FusedMoEConfig,
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_prepare_finalize import ( # noqa: E501
|
||||
FlashInferCutlassMoEPrepareAndFinalize)
|
||||
create_flashinfer_prepare_finalize)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -173,7 +173,7 @@ def build_flashinfer_fp8_cutlass_moe_prepare_finalize(
|
||||
moe: Optional[FusedMoEConfig], ) -> mk.FusedMoEPrepareAndFinalize:
|
||||
"""Create a FlashInfer CUTLASS fused-MoE prepare finalize kernel"""
|
||||
use_dp = moe.moe_parallel_config.dp_size > 1 if moe is not None else False
|
||||
return FlashInferCutlassMoEPrepareAndFinalize(use_dp)
|
||||
return create_flashinfer_prepare_finalize(use_dp)
|
||||
|
||||
|
||||
def select_cutlass_fp8_gemm_impl(
|
||||
|
||||
Reference in New Issue
Block a user