From 105605b8e9982af14351cd8260b21acda7ab5869 Mon Sep 17 00:00:00 2001 From: Rohan Potdar Date: Thu, 15 Oct 2026 14:10:12 -0500 Subject: [PATCH] [ROCm] Support DeepSeek-V4 hash routing in AITER Triton W4A16 MoE (#60282) Signed-off-by: Rohan Potdar Co-authored-by: Claude Opus 5.5 --- tests/kernels/moe/test_topk_softplus_sqrt.py | 86 ++++++++++++------- .../experts/aiter_mxfp4_w4a16_moe.py | 35 +++++++- vllm/model_executor/layers/fused_moe/layer.py | 2 + .../layers/fused_moe/modular_kernel.py | 27 +++++- .../layers/fused_moe/routed_experts.py | 3 + .../layers/quantization/mxfp4.py | 4 + vllm/models/deepseek_v4/cpu/model.py | 8 -- 7 files changed, 122 insertions(+), 43 deletions(-) diff --git a/tests/kernels/moe/test_topk_softplus_sqrt.py b/tests/kernels/moe/test_topk_softplus_sqrt.py index 0d92fe6eac..c9abdd36b1 100644 --- a/tests/kernels/moe/test_topk_softplus_sqrt.py +++ b/tests/kernels/moe/test_topk_softplus_sqrt.py @@ -81,32 +81,11 @@ def _torch_topk_softplus_sqrt( return topk_weights.to(torch.float32), topk_ids.to(torch.int32) -@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection") -@pytest.mark.parametrize("has_hash_routing", [False, True]) -@pytest.mark.parametrize( - "backend,expected_experts", - [ - ("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"), - ("aiter", "AiterMxfp4Experts"), - ("triton_unfused", "UnfusedOAITritonExperts"), - ], -) -def test_hash_routing_backend_selection( - dist_init, - default_vllm_config, - monkeypatch, - has_hash_routing, - backend, - expected_experts, -): - """Hash tables must reach backend selection; modular AITER remains eligible.""" +def _dsv4_moe_config(monkeypatch, default_vllm_config, has_hash_routing, backend): from dataclasses import replace from vllm._aiter_ops import rocm_aiter_ops from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory - from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( - select_deepseek_v4_mxfp4_moe_backend, - ) from vllm.platforms import rocm # Exercise both native MX and monolithic AITER selection on any ROCm device. @@ -130,14 +109,63 @@ def test_hash_routing_backend_selection( ), prefix="hash_routing_selection", ) - config = replace(layer.routed_experts.moe_config, moe_backend=backend) + return layer, replace(layer.routed_experts.moe_config, moe_backend=backend) - if has_hash_routing and backend == "aiter_triton_mxfp4_bf16": - with pytest.raises(ValueError, match="hash routing"): - select_deepseek_v4_mxfp4_moe_backend(config) - else: - _, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config) - assert experts_cls.__name__ == expected_experts + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection") +@pytest.mark.parametrize("has_hash_routing", [False, True]) +@pytest.mark.parametrize( + "backend,expected_experts", + [ + ("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"), + ("aiter", "AiterMxfp4Experts"), + ("triton_unfused", "UnfusedOAITritonExperts"), + ], +) +def test_hash_routing_backend_selection( + dist_init, + default_vllm_config, + monkeypatch, + has_hash_routing, + backend, + expected_experts, +): + """Hash tables reach backend selection and hash-capable kernels.""" + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + select_deepseek_v4_mxfp4_moe_backend, + ) + + layer, config = _dsv4_moe_config( + monkeypatch, default_vllm_config, has_hash_routing, backend + ) + _, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config) + assert experts_cls.__name__ == expected_experts + # Monolithic kernels route internally, so they read the table from the layer. + assert (layer.routed_experts.hash_indices_table is not None) == has_hash_routing + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection") +def test_hash_routing_rejects_monolithic_without_support( + dist_init, default_vllm_config, monkeypatch +): + """A monolithic kernel that does not implement hash routing is rejected.""" + from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import ( + AiterW4A16ExpertsMonolithic, + ) + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + select_deepseek_v4_mxfp4_moe_backend, + ) + + monkeypatch.setattr( + AiterW4A16ExpertsMonolithic, + "_supports_hash_routing", + staticmethod(lambda: False), + ) + _, config = _dsv4_moe_config( + monkeypatch, default_vllm_config, True, "aiter_triton_mxfp4_bf16" + ) + with pytest.raises(ValueError, match="hash routing"): + select_deepseek_v4_mxfp4_moe_backend(config) def test_torch_topk_softplus_sqrt_breaks_ties_by_expert_id(): diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py index 5423a93b5f..fc9f72ad10 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py @@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kMxfp4Static, ) +from vllm.triton_utils import triton __all__ = [ "AiterW4A16ExpertsMonolithic", @@ -134,6 +135,8 @@ def aiter_triton_kernel_w4a16_moe_forward( e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, score_mode: str | None = None, + input_ids: torch.Tensor | None = None, + hash_indices_table: torch.Tensor | None = None, ): assert quant_config is not None and rocm_aiter_ops.is_enabled() from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( @@ -154,7 +157,25 @@ def aiter_triton_kernel_w4a16_moe_forward( _routing_mod.is_tdm_avail = lambda: False aiter_routing = _routing_mod.routing - if score_mode is not None: + scale = routed_scaling_factor if routed_scaling_factor is not None else 1.0 + if hash_indices_table is not None: + assert score_mode == "sqrtsoftplus" and input_ids is not None + # Same block_m heuristic as aiter's routing(). + n_tokens, n_experts = gating_output.shape + block_m = max( + 16, min(triton.next_power_of_2(max(1, n_tokens * topk // n_experts)), 128) + ) + routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash( + gating_output, + hash_indices_table, + input_ids, + topk, + block_m, + score_mode=score_mode, + renorm=renormalize, + routed_scaling_factor=scale, + ) + elif score_mode is not None: use_grouped_topk = num_expert_group is not None and num_expert_group > 1 routing_data, gather_idx, scatter_idx = aiter_routing( gating_output, @@ -162,9 +183,7 @@ def aiter_triton_kernel_w4a16_moe_forward( score_mode=score_mode, bias=e_score_correction_bias, renorm=renormalize, - routed_scaling_factor=( - routed_scaling_factor if routed_scaling_factor is not None else 1.0 - ), + routed_scaling_factor=scale, use_grouped_topk=use_grouped_topk, num_expert_group=num_expert_group, topk_group=topk_group, @@ -348,6 +367,10 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): RoutingMethodType.DeepSeekV3, ] + @staticmethod + def _supports_hash_routing() -> bool: + return True + @staticmethod def _supports_router_logits_dtype( router_logits_dtype: torch.dtype | None, @@ -375,6 +398,8 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): routed_scaling_factor: float | None = None, topk_group: int | None = None, routing_replay_out: torch.Tensor | None = None, + input_ids: torch.Tensor | None = None, + hash_indices_table: torch.Tensor | None = None, ) -> torch.Tensor: routing_method = self.moe_config.routing_method if routing_method == RoutingMethodType.DeepseekV4: @@ -411,4 +436,6 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): e_score_correction_bias=e_score_correction_bias, routed_scaling_factor=routed_scaling_factor, score_mode=score_mode, + input_ids=input_ids, + hash_indices_table=hash_indices_table, ) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 839eba5d78..a51a1e9d0b 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -438,6 +438,8 @@ def FusedMoEFactory( apply_router_weight_on_input=apply_router_weight_on_input, **routed_experts_args if routed_experts_args is not None else {}, ) + if hash_indices_table is not None: + object.__setattr__(routed_experts, "hash_indices_table", hash_indices_table) if runner_cls is None: runner_cls = MoERunner diff --git a/vllm/model_executor/layers/fused_moe/modular_kernel.py b/vllm/model_executor/layers/fused_moe/modular_kernel.py index 785369a344..99928afb13 100644 --- a/vllm/model_executor/layers/fused_moe/modular_kernel.py +++ b/vllm/model_executor/layers/fused_moe/modular_kernel.py @@ -5,7 +5,7 @@ from collections.abc import Callable from dataclasses import dataclass from enum import Enum from math import prod -from typing import final +from typing import Any, final import torch @@ -531,6 +531,11 @@ class FusedMoEExperts(ABC): def is_monolithic() -> bool: raise NotImplementedError("Implemented by subclasses.") + @staticmethod + def _supports_hash_routing() -> bool: + """Whether a monolithic kernel implements hash routing.""" + return False + @property def expects_unquantized_inputs(self) -> bool: """Whether or not the PrepareFinalize should defer input quantization @@ -579,7 +584,11 @@ class FusedMoEExperts(ABC): return False, _make_reason( f"parallel config {moe_config.moe_parallel_config}" ) - elif moe_config.has_hash_routing and cls.is_monolithic(): + elif ( + moe_config.has_hash_routing + and cls.is_monolithic() + and not cls._supports_hash_routing() + ): return False, _make_reason("hash routing") elif moe_config.require_decomposed_backend and cls.is_monolithic(): return False, _make_reason("routes adjusted after routing") @@ -1584,11 +1593,20 @@ class FusedMoEKernelMonolithicImpl: topk_group: int | None = None, *, routing_sink: RoutedExpertsSink | None, + input_ids: torch.Tensor | None = None, + hash_indices_table: torch.Tensor | None = None, ) -> torch.Tensor | UnfinalizedMoEOutput: """Same as forward(), except uses router_logits as opposed to the topk_ids and topk_weights. This is used for kernels that have fused router + experts (e.g. FLASHINFER_TRTLLM). """ + # Only experts with _supports_hash_routing() accept these. + hash_kwargs: dict[str, Any] = {} + if hash_indices_table is not None: + hash_kwargs = { + "input_ids": input_ids, + "hash_indices_table": hash_indices_table, + } a1q, a1q_scale, router_logits = self.prepare_finalize.prepare( hidden_states, router_logits=router_logits, @@ -1621,6 +1639,7 @@ class FusedMoEKernelMonolithicImpl: routed_scaling_factor=routed_scaling_factor, topk_group=topk_group, routing_replay_out=routing_replay_out, + **hash_kwargs, ) if routing_sink is not None: routing_sink.capture_fn(routing_replay_out) @@ -1733,6 +1752,8 @@ class FusedMoEKernel: topk_group: int | None = None, *, routing_sink: RoutedExpertsSink | None, + input_ids: torch.Tensor | None = None, + hash_indices_table: torch.Tensor | None = None, ) -> torch.Tensor | UnfinalizedMoEOutput: assert isinstance(self.impl, FusedMoEKernelMonolithicImpl) return self.impl.apply( @@ -1749,6 +1770,8 @@ class FusedMoEKernel: routed_scaling_factor=routed_scaling_factor, topk_group=topk_group, routing_sink=routing_sink, + input_ids=input_ids, + hash_indices_table=hash_indices_table, ) def apply( diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 6fb49314fc..89a401eba3 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -79,6 +79,9 @@ class RoutedExperts(PluggableLayer): - Executing routed experts via quant_method.apply() """ + # Hash-routing table, owned by the gate; set by FusedMoEFactory. + hash_indices_table: torch.Tensor | None = None + def __init__( self, layer_name: str, diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 8f42f23970..c93977d86c 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -479,6 +479,8 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, routing_sink=layer.routing_sink, + input_ids=input_ids, + hash_indices_table=layer.hash_indices_table, ) @@ -1132,4 +1134,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, routing_sink=layer.routing_sink, + input_ids=input_ids, + hash_indices_table=layer.hash_indices_table, ) diff --git a/vllm/models/deepseek_v4/cpu/model.py b/vllm/models/deepseek_v4/cpu/model.py index 02ea09bcc4..71b9a9810d 100644 --- a/vllm/models/deepseek_v4/cpu/model.py +++ b/vllm/models/deepseek_v4/cpu/model.py @@ -216,14 +216,6 @@ class DeepseekV4MoE(nn.Module): swiglu_limit=self.swiglu_limit, router_logits_dtype=torch.float32, ) - if is_hash_moe: - # apply_monolithic() never sees hash_indices_table via the - # router path (modular-only) -- stash it on RoutedExperts - # directly. object.__setattr__ avoids nn.Module re-registering - # (and duplicating in the state_dict) this alias. - object.__setattr__( - self.experts.routed_experts, "hash_indices_table", self.gate.tid2eid - ) def forward( self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None