[Kernel] Add FlashInfer MoE A2A Kernel (#36022)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: Leo Tian <lctian@nvidia.com> Co-authored-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: root <root@lyris0267.lyris.clusters.nvidia.com>
This commit is contained in:
co-authored by
wzhao18
Stefano Castagnetta
root
parent
2390d44209
commit
2754231ba3
@@ -5,6 +5,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
)
|
||||
@@ -14,8 +15,11 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_a2a_prepare_finalize import (
|
||||
FlashInferA2APrepareAndFinalize,
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_nvlink_one_sided_prepare_finalize import ( # noqa: E501
|
||||
FlashInferNVLinkOneSidedPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_nvlink_two_sided_prepare_finalize import ( # noqa: E501
|
||||
FlashInferNVLinkTwoSidedPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEPrepareAndFinalize,
|
||||
@@ -206,9 +210,22 @@ def maybe_make_prepare_finalize(
|
||||
use_fp8_dispatch=use_fp8_dispatch,
|
||||
)
|
||||
|
||||
elif moe.use_fi_all2allv_kernels:
|
||||
elif moe.use_fi_nvl_two_sided_kernels:
|
||||
assert quant_config is not None
|
||||
prepare_finalize = FlashInferA2APrepareAndFinalize(
|
||||
prepare_finalize = FlashInferNVLinkTwoSidedPrepareAndFinalize(
|
||||
num_dispatchers=all2all_manager.world_size,
|
||||
)
|
||||
|
||||
elif moe.use_fi_nvl_one_sided_kernels:
|
||||
assert quant_config is not None
|
||||
max_num_tokens = (
|
||||
get_current_vllm_config().scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
prepare_finalize = FlashInferNVLinkOneSidedPrepareAndFinalize(
|
||||
max_num_tokens=max_num_tokens,
|
||||
top_k=moe.experts_per_token,
|
||||
num_experts=moe.num_experts,
|
||||
hidden_size=moe.hidden_dim,
|
||||
num_dispatchers=all2all_manager.world_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -957,9 +957,17 @@ class FusedMoEParallelConfig:
|
||||
return self.use_all2all_kernels and self.all2all_backend == "deepep_low_latency"
|
||||
|
||||
@property
|
||||
def use_fi_all2allv_kernels(self):
|
||||
def use_fi_nvl_two_sided_kernels(self):
|
||||
return self.use_all2all_kernels and (
|
||||
self.all2all_backend == "flashinfer_all2allv"
|
||||
or self.all2all_backend == "flashinfer_nvlink_two_sided"
|
||||
)
|
||||
|
||||
@property
|
||||
def use_fi_nvl_one_sided_kernels(self):
|
||||
return (
|
||||
self.use_all2all_kernels and self.all2all_backend == "flashinfer_all2allv"
|
||||
self.use_all2all_kernels
|
||||
and self.all2all_backend == "flashinfer_nvlink_one_sided"
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -1240,8 +1248,12 @@ class FusedMoEConfig:
|
||||
return self.moe_parallel_config.use_mori_kernels
|
||||
|
||||
@property
|
||||
def use_fi_all2allv_kernels(self):
|
||||
return self.moe_parallel_config.use_fi_all2allv_kernels
|
||||
def use_fi_nvl_two_sided_kernels(self):
|
||||
return self.moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
|
||||
@property
|
||||
def use_fi_nvl_one_sided_kernels(self):
|
||||
return self.moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
|
||||
@property
|
||||
def use_naive_all2all_kernels(self):
|
||||
|
||||
@@ -396,8 +396,9 @@ class CutlassExpertsFp8(CutlassExpertsFp8Base):
|
||||
# Note that the BATCHED activation format does not use
|
||||
# the expert map for identifying experts.
|
||||
return not (
|
||||
moe_parallel_config.use_fi_all2allv_kernels
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_deepep_ht_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
|
||||
@@ -152,7 +152,10 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
# NOTE(rob): discovered an IMA with this combination. Needs investigation.
|
||||
return not moe_parallel_config.use_fi_all2allv_kernels
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
|
||||
|
||||
|
||||
def get_local_sizes():
|
||||
return get_forward_context().dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
|
||||
|
||||
class FlashInferNVLinkOneSidedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""FlashInfer implementation using the Moe AlltoAll kernel."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_num_tokens: int,
|
||||
top_k: int,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
num_dispatchers: int = 1,
|
||||
):
|
||||
super().__init__()
|
||||
self.max_num_tokens = max_num_tokens
|
||||
self.top_k = top_k
|
||||
self.num_experts = num_experts
|
||||
self.hidden_size = hidden_size
|
||||
self.num_dispatchers_ = num_dispatchers
|
||||
|
||||
self.all2all_manager = get_ep_group().device_communicator.all2all_manager
|
||||
self.all2all_manager.initialize(
|
||||
max_num_tokens=self.max_num_tokens,
|
||||
top_k=self.top_k,
|
||||
num_experts=self.num_experts,
|
||||
hidden_size=self.hidden_size,
|
||||
)
|
||||
|
||||
@property
|
||||
def activation_format(self) -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
def max_num_tokens_per_rank(self) -> int | None:
|
||||
return None
|
||||
|
||||
def num_dispatchers(self) -> int:
|
||||
return self.num_dispatchers_
|
||||
|
||||
def output_is_reduced(self) -> bool:
|
||||
return False
|
||||
|
||||
def topk_indices_dtype(self) -> torch.dtype | None:
|
||||
return torch.int32
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.PrepareResultType:
|
||||
if apply_router_weight_on_input:
|
||||
topk = topk_ids.size(1)
|
||||
assert topk == 1, (
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
)
|
||||
a1.mul_(topk_weights.to(a1.dtype))
|
||||
|
||||
global_num_tokens_cpu = get_local_sizes()
|
||||
self.runtime_max_tokens_per_rank = (
|
||||
max(global_num_tokens_cpu)
|
||||
if global_num_tokens_cpu is not None
|
||||
else a1.shape[0]
|
||||
)
|
||||
|
||||
a1q, a1q_scale = moe_kernel_quantize_input(
|
||||
a1,
|
||||
quant_config.a1_gscale,
|
||||
quant_config.quant_dtype,
|
||||
quant_config.per_act_token_quant,
|
||||
quant_config.block_shape,
|
||||
is_fp4_scale_swizzled=False, # delay swizzle to after comm
|
||||
)
|
||||
|
||||
payloads = []
|
||||
payloads.append(a1q)
|
||||
if a1q_scale is not None:
|
||||
payloads.append(a1q_scale)
|
||||
payloads.append(topk_ids)
|
||||
payloads.append(topk_weights)
|
||||
|
||||
recv_payloads = self.all2all_manager.moe_alltoall.dispatch(
|
||||
token_selected_experts=topk_ids,
|
||||
input_payloads=payloads,
|
||||
runtime_max_tokens_per_rank=self.runtime_max_tokens_per_rank,
|
||||
)
|
||||
if a1q_scale is not None:
|
||||
a1q_recv, a1q_scale_recv, topk_ids_recv, topk_weights_recv = recv_payloads
|
||||
# Apply scale interleaving only for CUTLASS (not TRT-LLM)
|
||||
if (
|
||||
quant_config.quant_dtype == "nvfp4"
|
||||
and quant_config.is_nvfp4_scale_swizzled
|
||||
):
|
||||
a1q_scale_recv = a1q_scale_recv.view(-1, a1q_scale_recv.shape[-1])
|
||||
a1q_scale_recv = a1q_scale_recv.view(torch.uint8)
|
||||
a1q_scale_recv = nvfp4_block_scale_interleave(a1q_scale_recv)
|
||||
a1q_scale_recv = a1q_scale_recv.view(-1, self.hidden_size // 16)
|
||||
else:
|
||||
a1q_recv, topk_ids_recv, topk_weights_recv = recv_payloads
|
||||
a1q_scale_recv = None
|
||||
a1q_recv = a1q_recv.view(-1, a1q_recv.shape[-1])
|
||||
topk_ids_recv = topk_ids_recv.view(-1, topk_ids_recv.shape[-1])
|
||||
topk_weights_recv = topk_weights_recv.view(-1, topk_weights_recv.shape[-1])
|
||||
|
||||
return a1q_recv, a1q_scale_recv, None, topk_ids_recv, topk_weights_recv
|
||||
|
||||
def finalize(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
fused_expert_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
) -> None:
|
||||
assert self.all2all_manager.moe_alltoall is not None
|
||||
|
||||
ep_size = self.all2all_manager.world_size
|
||||
hidden_size = fused_expert_output.shape[-1]
|
||||
fused_expert_output = fused_expert_output.view(
|
||||
ep_size, self.runtime_max_tokens_per_rank, hidden_size
|
||||
)
|
||||
|
||||
combined_output = self.all2all_manager.moe_alltoall.combine(
|
||||
payload=fused_expert_output,
|
||||
runtime_max_tokens_per_rank=self.runtime_max_tokens_per_rank,
|
||||
)
|
||||
output.copy_(combined_output)
|
||||
+1
-1
@@ -18,7 +18,7 @@ def get_local_sizes():
|
||||
return get_forward_context().dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
|
||||
|
||||
class FlashInferA2APrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
class FlashInferNVLinkTwoSidedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""Base class for FlashInfer MoE prepare and finalize operations."""
|
||||
|
||||
def __init__(
|
||||
@@ -600,7 +600,10 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return not moe_parallel_config.use_fi_all2allv_kernels
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
@property
|
||||
def quant_type_id(self) -> int:
|
||||
|
||||
@@ -1965,7 +1965,10 @@ class TritonExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return not moe_parallel_config.use_fi_all2allv_kernels
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return True
|
||||
|
||||
@@ -638,7 +638,7 @@ class FusedMoE(CustomOp):
|
||||
self.use_overlapped = (
|
||||
not (
|
||||
(self.enable_eplb and backend != "allgather_reducescatter")
|
||||
or self.moe_parallel_config.use_fi_all2allv_kernels
|
||||
or self.moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
)
|
||||
and self._shared_experts is not None
|
||||
)
|
||||
|
||||
@@ -332,7 +332,10 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return not moe_parallel_config.use_fi_all2allv_kernels
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
def supports_expert_map(self):
|
||||
return True
|
||||
|
||||
@@ -233,7 +233,7 @@ class DefaultMoERunner(MoERunner):
|
||||
return (
|
||||
self.moe_config.moe_parallel_config.use_deepep_ll_kernels
|
||||
or self.moe_config.moe_parallel_config.use_mori_kernels
|
||||
or self.moe_config.moe_parallel_config.use_fi_all2allv_kernels
|
||||
or self.moe_config.moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or self.moe_config.moe_parallel_config.use_nixl_ep_kernels
|
||||
) and envs.VLLM_ENABLE_MOE_DP_CHUNK
|
||||
|
||||
|
||||
Reference in New Issue
Block a user