[WideEP] Integrate DeepEP v2 (#41183)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
05cb606cad
commit
e2f993dc41
@@ -29,7 +29,12 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two
|
||||
FlashInferNVLinkTwoSidedPrepareAndFinalize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_deep_ep, has_mori, has_nixl_ep
|
||||
from vllm.utils.import_utils import (
|
||||
has_deep_ep,
|
||||
has_deep_ep_v2,
|
||||
has_mori,
|
||||
has_nixl_ep,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -40,6 +45,8 @@ if current_platform.is_cuda_alike():
|
||||
DEEPEP_QUANT_BLOCK_SHAPE,
|
||||
DeepEPLLPrepareAndFinalize,
|
||||
)
|
||||
if has_deep_ep_v2():
|
||||
from .prepare_finalize.deepep_v2 import DeepEPV2PrepareAndFinalize
|
||||
if has_mori():
|
||||
from .prepare_finalize.mori import MoriPrepareAndFinalize
|
||||
if has_nixl_ep():
|
||||
@@ -94,6 +101,11 @@ def maybe_roundup_layer_hidden_size(
|
||||
hidden_size
|
||||
)
|
||||
|
||||
if moe_parallel_config.use_deepep_v2_kernels:
|
||||
hidden_size = DeepEPV2PrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
||||
hidden_size, act_dtype
|
||||
)
|
||||
|
||||
if moe_parallel_config.use_nixl_ep_kernels:
|
||||
hidden_size = NixlEPPrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
||||
hidden_size
|
||||
@@ -193,6 +205,36 @@ def maybe_make_prepare_finalize(
|
||||
physical_to_global=physical_to_global,
|
||||
local_expert_global_ids=local_expert_global_ids,
|
||||
)
|
||||
elif moe.use_deepep_v2_kernels:
|
||||
assert moe.dp_size == all2all_manager.dp_world_size
|
||||
|
||||
use_fp8_dispatch = (
|
||||
quant_config is not None
|
||||
and quant_config.quant_dtype == current_platform.fp8_dtype()
|
||||
and quant_config.is_block_quantized
|
||||
)
|
||||
all_to_all_args = dict(
|
||||
num_max_tokens_per_rank=moe.max_num_tokens,
|
||||
hidden=moe.hidden_dim,
|
||||
num_topk=moe.experts_per_token,
|
||||
num_experts=moe.num_experts,
|
||||
use_fp8_dispatch=use_fp8_dispatch,
|
||||
)
|
||||
handle = all2all_manager.get_handle(all_to_all_args)
|
||||
vllm_config = get_current_vllm_config()
|
||||
use_cudagraph = not vllm_config.model_config.enforce_eager
|
||||
|
||||
prepare_finalize = DeepEPV2PrepareAndFinalize(
|
||||
buffer=handle,
|
||||
num_dispatchers=all2all_manager.world_size,
|
||||
dp_size=all2all_manager.dp_world_size,
|
||||
rank_expert_offset=all2all_manager.rank * moe.num_local_experts,
|
||||
num_experts=moe.num_experts,
|
||||
num_topk=moe.experts_per_token,
|
||||
use_fp8_dispatch=use_fp8_dispatch,
|
||||
use_cudagraph=use_cudagraph,
|
||||
)
|
||||
|
||||
elif moe.use_mori_kernels:
|
||||
assert quant_config is not None
|
||||
|
||||
|
||||
@@ -1070,6 +1070,10 @@ class FusedMoEParallelConfig:
|
||||
def use_nixl_ep_kernels(self):
|
||||
return self.use_all2all_kernels and self.all2all_backend == "nixl_ep"
|
||||
|
||||
@property
|
||||
def use_deepep_v2_kernels(self):
|
||||
return self.use_all2all_kernels and self.all2all_backend == "deepep_v2"
|
||||
|
||||
@staticmethod
|
||||
def flatten_tp_across_dp_and_pcp(
|
||||
tp_size: int, dp_size: int, dp_rank: int, pcp_size: int, pcp_rank: int
|
||||
@@ -1394,6 +1398,10 @@ class FusedMoEConfig:
|
||||
def use_nixl_ep_kernels(self):
|
||||
return self.moe_parallel_config.use_nixl_ep_kernels
|
||||
|
||||
@property
|
||||
def use_deepep_v2_kernels(self):
|
||||
return self.moe_parallel_config.use_deepep_v2_kernels
|
||||
|
||||
@property
|
||||
def needs_round_robin_routing_tables(self):
|
||||
return self.moe_parallel_config.needs_round_robin_routing_tables
|
||||
|
||||
@@ -94,6 +94,14 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
Fp8 TRTLLM-Gen MoE kernels. Supports modular interface.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return (
|
||||
not moe_parallel_config.use_all2all_kernels
|
||||
or moe_parallel_config.use_ag_rs_all2all_kernels
|
||||
or moe_parallel_config.use_deepep_v2_kernels
|
||||
) and not moe_parallel_config.enable_eplb
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
@@ -193,7 +201,7 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.quant_config.w2_scale,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
top_k=topk_ids.size(1),
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
|
||||
@@ -85,6 +85,18 @@ def _get_priority_backends(
|
||||
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
|
||||
backends.insert(0, backends.pop(backends.index(backend)))
|
||||
|
||||
# With DeepEP v2 contiguous layout (do_expand=False), tensors are
|
||||
# worst-case allocated with padding. TrtLLM's tile-level skipping
|
||||
# avoids wasted compute on padding rows; other backends process all rows.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and moe_config.moe_parallel_config.use_deepep_v2_kernels
|
||||
and activation_key == kFp8Dynamic128Sym
|
||||
and weight_key == kFp8Static128BlockSym
|
||||
):
|
||||
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.FLASHINFER_TRTLLM)
|
||||
|
||||
# On Hopper for Block Fp8, prefer Triton for TP and FI CUTLASS for EP.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
|
||||
@@ -0,0 +1,394 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import deep_ep
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceContiguous,
|
||||
TopKWeightAndReduceDelegate,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.worker.ubatching import (
|
||||
dbo_current_ubatch_id,
|
||||
)
|
||||
|
||||
|
||||
class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""
|
||||
Prepare/Finalize using DeepEP v2 ElasticBuffer (unified API).
|
||||
|
||||
Supports two modes controlled by the `use_cudagraph` constructor arg:
|
||||
|
||||
**Decode mode (use_cudagraph=True):**
|
||||
- do_expand=False, do_cpu_sync=False
|
||||
- Tokens returned in original order with recv_topk_idx (global IDs)
|
||||
- Worst-case tensor allocation; padding rows zeroed via
|
||||
handle.psum_num_recv_tokens_per_scaleup_rank
|
||||
- Fully cudagraph-capturable
|
||||
- Expert kernel sorts internally (expert_tokens_meta=None)
|
||||
|
||||
**Prefill mode (use_cudagraph=False):**
|
||||
- do_expand=True, do_cpu_sync=True
|
||||
- Per-expert-contiguous layout; exact memory allocation
|
||||
- Saves GPU memory (no worst-case allocation)
|
||||
- Not cudagraph-capturable (CPU polling), but prefill doesn't
|
||||
use cudagraphs anyway
|
||||
- Provides expert_tokens_meta for efficient batched expert kernels
|
||||
|
||||
Both modes use async_with_compute_stream=False (synchronous from
|
||||
caller's perspective). The ElasticBuffer handles comm internally.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def maybe_roundup_layer_hidden_size(hidden_size: int, dtype: torch.dtype) -> int:
|
||||
hidden_size_bytes = hidden_size * dtype.itemsize
|
||||
xfer_atom_size = 512 # 32 * 16 (size(int4))
|
||||
if hidden_size_bytes % xfer_atom_size == 0:
|
||||
return hidden_size
|
||||
|
||||
hidden_size_bytes = round_up(hidden_size_bytes, xfer_atom_size)
|
||||
return hidden_size_bytes // dtype.itemsize
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
buffer: deep_ep.ElasticBuffer,
|
||||
num_dispatchers: int,
|
||||
dp_size: int,
|
||||
rank_expert_offset: int,
|
||||
num_experts: int,
|
||||
num_topk: int,
|
||||
use_fp8_dispatch: bool = False,
|
||||
use_cudagraph: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.buffer = buffer
|
||||
self.num_dispatchers_ = num_dispatchers
|
||||
self.dp_size = dp_size
|
||||
self.rank_expert_offset = rank_expert_offset
|
||||
self.num_experts = num_experts
|
||||
self.num_topk = num_topk
|
||||
self.use_fp8_dispatch = use_fp8_dispatch
|
||||
self.use_cudagraph = use_cudagraph
|
||||
|
||||
# DBO microbatching: one handle slot per micro-batch.
|
||||
self.handles: list[deep_ep.EPHandle | None] = [None, None]
|
||||
|
||||
def num_dispatchers(self) -> int:
|
||||
return self.num_dispatchers_
|
||||
|
||||
def output_is_reduced(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def activation_format(self) -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
def max_num_tokens_per_rank(self) -> int | None:
|
||||
return None
|
||||
|
||||
def topk_indices_dtype(self) -> torch.dtype | None:
|
||||
return torch.int64
|
||||
|
||||
def _do_dispatch(
|
||||
self,
|
||||
tokens: torch.Tensor,
|
||||
token_scales: torch.Tensor | None,
|
||||
rank_topk_ids: torch.Tensor,
|
||||
rank_topk_weights: torch.Tensor,
|
||||
num_experts: int,
|
||||
a1_scale: torch.Tensor | None,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool,
|
||||
) -> Callable:
|
||||
has_scales = token_scales is not None
|
||||
|
||||
token_data = tokens
|
||||
if has_scales:
|
||||
token_data = (tokens, token_scales)
|
||||
|
||||
# Decode: do_expand=False + do_cpu_sync=False (cudagraph-safe)
|
||||
# Prefill: do_expand=True + do_cpu_sync=True (memory-efficient)
|
||||
do_expand = not self.use_cudagraph
|
||||
do_cpu_sync = not self.use_cudagraph
|
||||
|
||||
(
|
||||
recv_x,
|
||||
recv_topk_idx,
|
||||
recv_topk_weights,
|
||||
handle,
|
||||
event,
|
||||
) = self.buffer.dispatch(
|
||||
x=token_data,
|
||||
topk_idx=rank_topk_ids,
|
||||
topk_weights=rank_topk_weights,
|
||||
num_experts=num_experts,
|
||||
do_expand=do_expand,
|
||||
do_cpu_sync=do_cpu_sync,
|
||||
async_with_compute_stream=False,
|
||||
)
|
||||
|
||||
a2a_idx = dbo_current_ubatch_id()
|
||||
self.handles[a2a_idx] = handle
|
||||
|
||||
return lambda: self._receiver(
|
||||
event,
|
||||
has_scales,
|
||||
recv_x,
|
||||
recv_topk_idx,
|
||||
num_experts,
|
||||
handle.num_recv_tokens_per_expert_list,
|
||||
recv_topk_weights,
|
||||
handle.psum_num_recv_tokens_per_scaleup_rank,
|
||||
a1_scale,
|
||||
quant_config,
|
||||
defer_input_quant=defer_input_quant,
|
||||
)
|
||||
|
||||
def _receiver(
|
||||
self,
|
||||
event: deep_ep.EventOverlap,
|
||||
has_scales: bool,
|
||||
recv_x: tuple[torch.Tensor, torch.Tensor] | torch.Tensor,
|
||||
recv_topk_idx: torch.Tensor | None,
|
||||
num_experts: int,
|
||||
recv_expert_num_tokens: list[int],
|
||||
recv_topk_weights: torch.Tensor | None,
|
||||
psum_recv_per_rank: torch.Tensor,
|
||||
a1_scale: torch.Tensor | None,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool,
|
||||
) -> mk.PrepareResultType:
|
||||
if event.event is not None:
|
||||
event.current_stream_wait()
|
||||
|
||||
if isinstance(recv_x, tuple):
|
||||
expert_x, expert_x_scale = recv_x
|
||||
else:
|
||||
expert_x, expert_x_scale = recv_x, None
|
||||
|
||||
if recv_topk_idx is None:
|
||||
# do_expand=True (prefill mode): build topk_ids from
|
||||
# per-expert token counts.
|
||||
total_tokens = sum(recv_expert_num_tokens)
|
||||
if total_tokens > 0:
|
||||
recv_topk_idx = torch.empty(
|
||||
total_tokens,
|
||||
dtype=torch.int64,
|
||||
device=expert_x.device,
|
||||
)
|
||||
offset = 0
|
||||
for i, count in enumerate(recv_expert_num_tokens):
|
||||
if count > 0:
|
||||
recv_topk_idx[offset : offset + count].fill_(
|
||||
i + self.rank_expert_offset
|
||||
)
|
||||
offset += count
|
||||
else:
|
||||
recv_topk_idx = torch.empty(
|
||||
0,
|
||||
dtype=torch.int64,
|
||||
device=expert_x.device,
|
||||
)
|
||||
recv_topk_idx = recv_topk_idx.unsqueeze(1)
|
||||
else:
|
||||
# do_expand=False (decode/cudagraph mode): recv_topk_idx has
|
||||
# LOCAL expert IDs (-1 for non-local and padding rows).
|
||||
# Convert valid local IDs to global. Rows with -1 are
|
||||
# skipped by expert kernels (TrtLLM tile-level skipping,
|
||||
# DeepGemm is_computation_valid), so no need to zero
|
||||
# hidden states, scales, or weights for padding rows.
|
||||
valid_mask = recv_topk_idx >= 0
|
||||
recv_topk_idx = torch.where(
|
||||
valid_mask,
|
||||
recv_topk_idx + self.rank_expert_offset,
|
||||
recv_topk_idx,
|
||||
)
|
||||
|
||||
# Reshape recv_topk_weights to match recv_topk_idx shape [N, 1]
|
||||
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
|
||||
recv_topk_weights = recv_topk_weights.unsqueeze(1)
|
||||
|
||||
expert_tokens_meta = mk.ExpertTokensMetadata.make_from_list(
|
||||
recv_expert_num_tokens,
|
||||
device=expert_x.device,
|
||||
)
|
||||
|
||||
if not quant_config.is_block_quantized and not defer_input_quant:
|
||||
expert_x_scale = None
|
||||
if expert_x.numel() != 0:
|
||||
expert_x, expert_x_scale = moe_kernel_quantize_input(
|
||||
expert_x,
|
||||
a1_scale,
|
||||
quant_dtype=quant_config.quant_dtype,
|
||||
per_act_token_quant=False,
|
||||
block_shape=quant_config.block_shape,
|
||||
is_scale_swizzled=quant_config.is_scale_swizzled,
|
||||
)
|
||||
|
||||
return (
|
||||
expert_x,
|
||||
expert_x_scale,
|
||||
expert_tokens_meta,
|
||||
recv_topk_idx,
|
||||
recv_topk_weights,
|
||||
)
|
||||
|
||||
def supports_async(self) -> bool:
|
||||
return True
|
||||
|
||||
def prepare_async(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.ReceiverType:
|
||||
if apply_router_weight_on_input:
|
||||
topk = topk_ids.size(1)
|
||||
assert topk == 1, (
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
)
|
||||
a1 = a1 * topk_weights.to(a1.dtype)
|
||||
|
||||
if quant_config.is_block_quantized and not defer_input_quant:
|
||||
a1q, a1q_scale = moe_kernel_quantize_input(
|
||||
a1,
|
||||
quant_config.a1_scale,
|
||||
quant_dtype=quant_config.quant_dtype,
|
||||
per_act_token_quant=quant_config.per_act_token_quant,
|
||||
block_shape=quant_config.block_shape,
|
||||
)
|
||||
if a1q_scale is not None and a1q_scale.numel() == 1:
|
||||
a1q_scale = a1q_scale.view(1, 1)
|
||||
a1_post_scale = None
|
||||
else:
|
||||
a1q = a1
|
||||
a1q_scale = None
|
||||
a1_post_scale = (
|
||||
quant_config.a1_gscale
|
||||
if quant_config.quant_dtype == "nvfp4"
|
||||
else quant_config.a1_scale
|
||||
)
|
||||
|
||||
return self._do_dispatch(
|
||||
tokens=a1q,
|
||||
token_scales=a1q_scale,
|
||||
rank_topk_ids=topk_ids,
|
||||
rank_topk_weights=topk_weights,
|
||||
num_experts=num_experts,
|
||||
a1_scale=a1_post_scale,
|
||||
quant_config=quant_config,
|
||||
defer_input_quant=defer_input_quant,
|
||||
)
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.PrepareResultType:
|
||||
receiver = self.prepare_async(
|
||||
a1,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
num_experts,
|
||||
expert_map,
|
||||
apply_router_weight_on_input,
|
||||
quant_config,
|
||||
defer_input_quant,
|
||||
)
|
||||
return receiver()
|
||||
|
||||
def _finalize(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
fused_expert_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
do_async: bool,
|
||||
) -> Callable | None:
|
||||
a2a_idx = dbo_current_ubatch_id()
|
||||
handle = self.handles[a2a_idx]
|
||||
assert handle is not None
|
||||
|
||||
if fused_expert_output.numel() != 0:
|
||||
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceDelegate):
|
||||
weight_and_reduce_impl = TopKWeightAndReduceContiguous()
|
||||
fused_expert_output = weight_and_reduce_impl.apply(
|
||||
output=None,
|
||||
fused_expert_output=fused_expert_output,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
if fused_expert_output.dtype != torch.bfloat16:
|
||||
raise ValueError(
|
||||
f"DeepEP v2 combine requires bfloat16 input, "
|
||||
f"got {fused_expert_output.dtype}"
|
||||
)
|
||||
|
||||
combined_x, _, event = self.buffer.combine(
|
||||
x=fused_expert_output,
|
||||
handle=handle,
|
||||
topk_weights=None,
|
||||
async_with_compute_stream=False,
|
||||
)
|
||||
|
||||
output.copy_(combined_x, non_blocking=True)
|
||||
return None
|
||||
|
||||
def finalize_async(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
fused_expert_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
) -> Callable:
|
||||
self._finalize(
|
||||
output,
|
||||
fused_expert_output,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
apply_router_weight_on_input,
|
||||
weight_and_reduce_impl,
|
||||
False,
|
||||
)
|
||||
return lambda: None
|
||||
|
||||
def finalize(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
fused_expert_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
) -> None:
|
||||
self._finalize(
|
||||
output,
|
||||
fused_expert_output,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
apply_router_weight_on_input,
|
||||
weight_and_reduce_impl,
|
||||
False,
|
||||
)
|
||||
@@ -1038,6 +1038,7 @@ class RoutedExperts(PluggableLayer):
|
||||
"e_score_correction_bias",
|
||||
"w13_input_scale",
|
||||
"w2_input_scale",
|
||||
"hash_indices_table",
|
||||
}
|
||||
|
||||
# Parameters of non-expert submodules that live inside runner (RoutedExperts).
|
||||
|
||||
@@ -168,6 +168,14 @@ def fused_topk_bias(
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
):
|
||||
# The topk kernel dispatches dtype based on topk_ids (set by
|
||||
# indices_type) and assumes input_tokens/hash_indices_table match.
|
||||
if indices_type is not None:
|
||||
if input_tokens is not None and input_tokens.dtype != indices_type:
|
||||
input_tokens = input_tokens.to(dtype=indices_type)
|
||||
if hash_indices_table is not None and hash_indices_table.dtype != indices_type:
|
||||
hash_indices_table = hash_indices_table.to(dtype=indices_type)
|
||||
|
||||
if not rocm_aiter_ops.is_fused_moe_enabled():
|
||||
assert hidden_states.size(0) == gating_output.size(0), (
|
||||
"Number of tokens mismatch"
|
||||
|
||||
@@ -489,6 +489,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
self.w13_precision_config = None
|
||||
self.w2_precision_config = None
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def skip_forward_padding(self) -> bool:
|
||||
# SM100_FI_MXFP4_MXFP8_TRTLLM supports padding with mxfp8 quant
|
||||
|
||||
Reference in New Issue
Block a user