[ROCm] Support DeepSeek-V4 hash routing in AITER Triton W4A16 MoE (#60282)

Signed-off-by: Rohan Potdar <rohan.potdar@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Rohan Potdar
2026-10-15 14:10:12 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 82b9ccaa79
commit 105605b8e9
7 changed files with 122 additions and 43 deletions
+57 -29
View File
@@ -81,32 +81,11 @@ def _torch_topk_softplus_sqrt(
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
@pytest.mark.parametrize("has_hash_routing", [False, True])
@pytest.mark.parametrize(
"backend,expected_experts",
[
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
("aiter", "AiterMxfp4Experts"),
("triton_unfused", "UnfusedOAITritonExperts"),
],
)
def test_hash_routing_backend_selection(
dist_init,
default_vllm_config,
monkeypatch,
has_hash_routing,
backend,
expected_experts,
):
"""Hash tables must reach backend selection; modular AITER remains eligible."""
def _dsv4_moe_config(monkeypatch, default_vllm_config, has_hash_routing, backend):
from dataclasses import replace
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
from vllm.platforms import rocm
# Exercise both native MX and monolithic AITER selection on any ROCm device.
@@ -130,14 +109,63 @@ def test_hash_routing_backend_selection(
),
prefix="hash_routing_selection",
)
config = replace(layer.routed_experts.moe_config, moe_backend=backend)
return layer, replace(layer.routed_experts.moe_config, moe_backend=backend)
if has_hash_routing and backend == "aiter_triton_mxfp4_bf16":
with pytest.raises(ValueError, match="hash routing"):
select_deepseek_v4_mxfp4_moe_backend(config)
else:
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
assert experts_cls.__name__ == expected_experts
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
@pytest.mark.parametrize("has_hash_routing", [False, True])
@pytest.mark.parametrize(
"backend,expected_experts",
[
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
("aiter", "AiterMxfp4Experts"),
("triton_unfused", "UnfusedOAITritonExperts"),
],
)
def test_hash_routing_backend_selection(
dist_init,
default_vllm_config,
monkeypatch,
has_hash_routing,
backend,
expected_experts,
):
"""Hash tables reach backend selection and hash-capable kernels."""
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
layer, config = _dsv4_moe_config(
monkeypatch, default_vllm_config, has_hash_routing, backend
)
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
assert experts_cls.__name__ == expected_experts
# Monolithic kernels route internally, so they read the table from the layer.
assert (layer.routed_experts.hash_indices_table is not None) == has_hash_routing
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
def test_hash_routing_rejects_monolithic_without_support(
dist_init, default_vllm_config, monkeypatch
):
"""A monolithic kernel that does not implement hash routing is rejected."""
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
monkeypatch.setattr(
AiterW4A16ExpertsMonolithic,
"_supports_hash_routing",
staticmethod(lambda: False),
)
_, config = _dsv4_moe_config(
monkeypatch, default_vllm_config, True, "aiter_triton_mxfp4_bf16"
)
with pytest.raises(ValueError, match="hash routing"):
select_deepseek_v4_mxfp4_moe_backend(config)
def test_torch_topk_softplus_sqrt_breaks_ties_by_expert_id():
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
)
from vllm.triton_utils import triton
__all__ = [
"AiterW4A16ExpertsMonolithic",
@@ -134,6 +135,8 @@ def aiter_triton_kernel_w4a16_moe_forward(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
@@ -154,7 +157,25 @@ def aiter_triton_kernel_w4a16_moe_forward(
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
if score_mode is not None:
scale = routed_scaling_factor if routed_scaling_factor is not None else 1.0
if hash_indices_table is not None:
assert score_mode == "sqrtsoftplus" and input_ids is not None
# Same block_m heuristic as aiter's routing().
n_tokens, n_experts = gating_output.shape
block_m = max(
16, min(triton.next_power_of_2(max(1, n_tokens * topk // n_experts)), 128)
)
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
gating_output,
hash_indices_table,
input_ids,
topk,
block_m,
score_mode=score_mode,
renorm=renormalize,
routed_scaling_factor=scale,
)
elif score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
@@ -162,9 +183,7 @@ def aiter_triton_kernel_w4a16_moe_forward(
score_mode=score_mode,
bias=e_score_correction_bias,
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
routed_scaling_factor=scale,
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
@@ -348,6 +367,10 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.DeepSeekV3,
]
@staticmethod
def _supports_hash_routing() -> bool:
return True
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
@@ -375,6 +398,8 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor:
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
@@ -411,4 +436,6 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
@@ -438,6 +438,8 @@ def FusedMoEFactory(
apply_router_weight_on_input=apply_router_weight_on_input,
**routed_experts_args if routed_experts_args is not None else {},
)
if hash_indices_table is not None:
object.__setattr__(routed_experts, "hash_indices_table", hash_indices_table)
if runner_cls is None:
runner_cls = MoERunner
@@ -5,7 +5,7 @@ from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum
from math import prod
from typing import final
from typing import Any, final
import torch
@@ -531,6 +531,11 @@ class FusedMoEExperts(ABC):
def is_monolithic() -> bool:
raise NotImplementedError("Implemented by subclasses.")
@staticmethod
def _supports_hash_routing() -> bool:
"""Whether a monolithic kernel implements hash routing."""
return False
@property
def expects_unquantized_inputs(self) -> bool:
"""Whether or not the PrepareFinalize should defer input quantization
@@ -579,7 +584,11 @@ class FusedMoEExperts(ABC):
return False, _make_reason(
f"parallel config {moe_config.moe_parallel_config}"
)
elif moe_config.has_hash_routing and cls.is_monolithic():
elif (
moe_config.has_hash_routing
and cls.is_monolithic()
and not cls._supports_hash_routing()
):
return False, _make_reason("hash routing")
elif moe_config.require_decomposed_backend and cls.is_monolithic():
return False, _make_reason("routes adjusted after routing")
@@ -1584,11 +1593,20 @@ class FusedMoEKernelMonolithicImpl:
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as forward(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is used for kernels
that have fused router + experts (e.g. FLASHINFER_TRTLLM).
"""
# Only experts with _supports_hash_routing() accept these.
hash_kwargs: dict[str, Any] = {}
if hash_indices_table is not None:
hash_kwargs = {
"input_ids": input_ids,
"hash_indices_table": hash_indices_table,
}
a1q, a1q_scale, router_logits = self.prepare_finalize.prepare(
hidden_states,
router_logits=router_logits,
@@ -1621,6 +1639,7 @@ class FusedMoEKernelMonolithicImpl:
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
**hash_kwargs,
)
if routing_sink is not None:
routing_sink.capture_fn(routing_replay_out)
@@ -1733,6 +1752,8 @@ class FusedMoEKernel:
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
return self.impl.apply(
@@ -1749,6 +1770,8 @@ class FusedMoEKernel:
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_sink=routing_sink,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
def apply(
@@ -79,6 +79,9 @@ class RoutedExperts(PluggableLayer):
- Executing routed experts via quant_method.apply()
"""
# Hash-routing table, owned by the gate; set by FusedMoEFactory.
hash_indices_table: torch.Tensor | None = None
def __init__(
self,
layer_name: str,
@@ -479,6 +479,8 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
input_ids=input_ids,
hash_indices_table=layer.hash_indices_table,
)
@@ -1132,4 +1134,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
input_ids=input_ids,
hash_indices_table=layer.hash_indices_table,
)
-8
View File
@@ -216,14 +216,6 @@ class DeepseekV4MoE(nn.Module):
swiglu_limit=self.swiglu_limit,
router_logits_dtype=torch.float32,
)
if is_hash_moe:
# apply_monolithic() never sees hash_indices_table via the
# router path (modular-only) -- stash it on RoutedExperts
# directly. object.__setattr__ avoids nn.Module re-registering
# (and duplicating in the state_dict) this alias.
object.__setattr__(
self.experts.routed_experts, "hash_indices_table", self.gate.tid2eid
)
def forward(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None