[ROCm] Support DeepSeek-V4 hash routing in AITER Triton W4A16 MoE (#60282)

Signed-off-by: Rohan Potdar <rohan.potdar@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Rohan Potdar
2026-10-15 14:10:12 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 82b9ccaa79
commit 105605b8e9
7 changed files with 122 additions and 43 deletions
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
)
from vllm.triton_utils import triton
__all__ = [
"AiterW4A16ExpertsMonolithic",
@@ -134,6 +135,8 @@ def aiter_triton_kernel_w4a16_moe_forward(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
@@ -154,7 +157,25 @@ def aiter_triton_kernel_w4a16_moe_forward(
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
if score_mode is not None:
scale = routed_scaling_factor if routed_scaling_factor is not None else 1.0
if hash_indices_table is not None:
assert score_mode == "sqrtsoftplus" and input_ids is not None
# Same block_m heuristic as aiter's routing().
n_tokens, n_experts = gating_output.shape
block_m = max(
16, min(triton.next_power_of_2(max(1, n_tokens * topk // n_experts)), 128)
)
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
gating_output,
hash_indices_table,
input_ids,
topk,
block_m,
score_mode=score_mode,
renorm=renormalize,
routed_scaling_factor=scale,
)
elif score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
@@ -162,9 +183,7 @@ def aiter_triton_kernel_w4a16_moe_forward(
score_mode=score_mode,
bias=e_score_correction_bias,
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
routed_scaling_factor=scale,
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
@@ -348,6 +367,10 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.DeepSeekV3,
]
@staticmethod
def _supports_hash_routing() -> bool:
return True
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
@@ -375,6 +398,8 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor:
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
@@ -411,4 +436,6 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
@@ -438,6 +438,8 @@ def FusedMoEFactory(
apply_router_weight_on_input=apply_router_weight_on_input,
**routed_experts_args if routed_experts_args is not None else {},
)
if hash_indices_table is not None:
object.__setattr__(routed_experts, "hash_indices_table", hash_indices_table)
if runner_cls is None:
runner_cls = MoERunner
@@ -5,7 +5,7 @@ from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum
from math import prod
from typing import final
from typing import Any, final
import torch
@@ -531,6 +531,11 @@ class FusedMoEExperts(ABC):
def is_monolithic() -> bool:
raise NotImplementedError("Implemented by subclasses.")
@staticmethod
def _supports_hash_routing() -> bool:
"""Whether a monolithic kernel implements hash routing."""
return False
@property
def expects_unquantized_inputs(self) -> bool:
"""Whether or not the PrepareFinalize should defer input quantization
@@ -579,7 +584,11 @@ class FusedMoEExperts(ABC):
return False, _make_reason(
f"parallel config {moe_config.moe_parallel_config}"
)
elif moe_config.has_hash_routing and cls.is_monolithic():
elif (
moe_config.has_hash_routing
and cls.is_monolithic()
and not cls._supports_hash_routing()
):
return False, _make_reason("hash routing")
elif moe_config.require_decomposed_backend and cls.is_monolithic():
return False, _make_reason("routes adjusted after routing")
@@ -1584,11 +1593,20 @@ class FusedMoEKernelMonolithicImpl:
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as forward(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is used for kernels
that have fused router + experts (e.g. FLASHINFER_TRTLLM).
"""
# Only experts with _supports_hash_routing() accept these.
hash_kwargs: dict[str, Any] = {}
if hash_indices_table is not None:
hash_kwargs = {
"input_ids": input_ids,
"hash_indices_table": hash_indices_table,
}
a1q, a1q_scale, router_logits = self.prepare_finalize.prepare(
hidden_states,
router_logits=router_logits,
@@ -1621,6 +1639,7 @@ class FusedMoEKernelMonolithicImpl:
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
**hash_kwargs,
)
if routing_sink is not None:
routing_sink.capture_fn(routing_replay_out)
@@ -1733,6 +1752,8 @@ class FusedMoEKernel:
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
return self.impl.apply(
@@ -1749,6 +1770,8 @@ class FusedMoEKernel:
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_sink=routing_sink,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
def apply(
@@ -79,6 +79,9 @@ class RoutedExperts(PluggableLayer):
- Executing routed experts via quant_method.apply()
"""
# Hash-routing table, owned by the gate; set by FusedMoEFactory.
hash_indices_table: torch.Tensor | None = None
def __init__(
self,
layer_name: str,
@@ -479,6 +479,8 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
input_ids=input_ids,
hash_indices_table=layer.hash_indices_table,
)
@@ -1132,4 +1134,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
input_ids=input_ids,
hash_indices_table=layer.hash_indices_table,
)