[ROCm] Support DeepSeek-V4 hash routing in AITER Triton W4A16 MoE (#60282)
Signed-off-by: Rohan Potdar <rohan.potdar@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
82b9ccaa79
commit
105605b8e9
@@ -81,32 +81,11 @@ def _torch_topk_softplus_sqrt(
|
||||
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
@pytest.mark.parametrize("has_hash_routing", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"backend,expected_experts",
|
||||
[
|
||||
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
|
||||
("aiter", "AiterMxfp4Experts"),
|
||||
("triton_unfused", "UnfusedOAITritonExperts"),
|
||||
],
|
||||
)
|
||||
def test_hash_routing_backend_selection(
|
||||
dist_init,
|
||||
default_vllm_config,
|
||||
monkeypatch,
|
||||
has_hash_routing,
|
||||
backend,
|
||||
expected_experts,
|
||||
):
|
||||
"""Hash tables must reach backend selection; modular AITER remains eligible."""
|
||||
def _dsv4_moe_config(monkeypatch, default_vllm_config, has_hash_routing, backend):
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.platforms import rocm
|
||||
|
||||
# Exercise both native MX and monolithic AITER selection on any ROCm device.
|
||||
@@ -130,14 +109,63 @@ def test_hash_routing_backend_selection(
|
||||
),
|
||||
prefix="hash_routing_selection",
|
||||
)
|
||||
config = replace(layer.routed_experts.moe_config, moe_backend=backend)
|
||||
return layer, replace(layer.routed_experts.moe_config, moe_backend=backend)
|
||||
|
||||
if has_hash_routing and backend == "aiter_triton_mxfp4_bf16":
|
||||
with pytest.raises(ValueError, match="hash routing"):
|
||||
select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
else:
|
||||
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
assert experts_cls.__name__ == expected_experts
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
@pytest.mark.parametrize("has_hash_routing", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"backend,expected_experts",
|
||||
[
|
||||
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
|
||||
("aiter", "AiterMxfp4Experts"),
|
||||
("triton_unfused", "UnfusedOAITritonExperts"),
|
||||
],
|
||||
)
|
||||
def test_hash_routing_backend_selection(
|
||||
dist_init,
|
||||
default_vllm_config,
|
||||
monkeypatch,
|
||||
has_hash_routing,
|
||||
backend,
|
||||
expected_experts,
|
||||
):
|
||||
"""Hash tables reach backend selection and hash-capable kernels."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
|
||||
layer, config = _dsv4_moe_config(
|
||||
monkeypatch, default_vllm_config, has_hash_routing, backend
|
||||
)
|
||||
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
assert experts_cls.__name__ == expected_experts
|
||||
# Monolithic kernels route internally, so they read the table from the layer.
|
||||
assert (layer.routed_experts.hash_indices_table is not None) == has_hash_routing
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
def test_hash_routing_rejects_monolithic_without_support(
|
||||
dist_init, default_vllm_config, monkeypatch
|
||||
):
|
||||
"""A monolithic kernel that does not implement hash routing is rejected."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
"_supports_hash_routing",
|
||||
staticmethod(lambda: False),
|
||||
)
|
||||
_, config = _dsv4_moe_config(
|
||||
monkeypatch, default_vllm_config, True, "aiter_triton_mxfp4_bf16"
|
||||
)
|
||||
with pytest.raises(ValueError, match="hash routing"):
|
||||
select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
|
||||
|
||||
def test_torch_topk_softplus_sqrt_breaks_ties_by_expert_id():
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
__all__ = [
|
||||
"AiterW4A16ExpertsMonolithic",
|
||||
@@ -134,6 +135,8 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
score_mode: str | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
):
|
||||
assert quant_config is not None and rocm_aiter_ops.is_enabled()
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
@@ -154,7 +157,25 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
||||
_routing_mod.is_tdm_avail = lambda: False
|
||||
aiter_routing = _routing_mod.routing
|
||||
|
||||
if score_mode is not None:
|
||||
scale = routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
if hash_indices_table is not None:
|
||||
assert score_mode == "sqrtsoftplus" and input_ids is not None
|
||||
# Same block_m heuristic as aiter's routing().
|
||||
n_tokens, n_experts = gating_output.shape
|
||||
block_m = max(
|
||||
16, min(triton.next_power_of_2(max(1, n_tokens * topk // n_experts)), 128)
|
||||
)
|
||||
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
|
||||
gating_output,
|
||||
hash_indices_table,
|
||||
input_ids,
|
||||
topk,
|
||||
block_m,
|
||||
score_mode=score_mode,
|
||||
renorm=renormalize,
|
||||
routed_scaling_factor=scale,
|
||||
)
|
||||
elif score_mode is not None:
|
||||
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
|
||||
routing_data, gather_idx, scatter_idx = aiter_routing(
|
||||
gating_output,
|
||||
@@ -162,9 +183,7 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
||||
score_mode=score_mode,
|
||||
bias=e_score_correction_bias,
|
||||
renorm=renormalize,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
routed_scaling_factor=scale,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
@@ -348,6 +367,10 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_hash_routing() -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
@@ -375,6 +398,8 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
routing_method = self.moe_config.routing_method
|
||||
if routing_method == RoutingMethodType.DeepseekV4:
|
||||
@@ -411,4 +436,6 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
score_mode=score_mode,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
@@ -438,6 +438,8 @@ def FusedMoEFactory(
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
**routed_experts_args if routed_experts_args is not None else {},
|
||||
)
|
||||
if hash_indices_table is not None:
|
||||
object.__setattr__(routed_experts, "hash_indices_table", hash_indices_table)
|
||||
|
||||
if runner_cls is None:
|
||||
runner_cls = MoERunner
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from math import prod
|
||||
from typing import final
|
||||
from typing import Any, final
|
||||
|
||||
import torch
|
||||
|
||||
@@ -531,6 +531,11 @@ class FusedMoEExperts(ABC):
|
||||
def is_monolithic() -> bool:
|
||||
raise NotImplementedError("Implemented by subclasses.")
|
||||
|
||||
@staticmethod
|
||||
def _supports_hash_routing() -> bool:
|
||||
"""Whether a monolithic kernel implements hash routing."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
"""Whether or not the PrepareFinalize should defer input quantization
|
||||
@@ -579,7 +584,11 @@ class FusedMoEExperts(ABC):
|
||||
return False, _make_reason(
|
||||
f"parallel config {moe_config.moe_parallel_config}"
|
||||
)
|
||||
elif moe_config.has_hash_routing and cls.is_monolithic():
|
||||
elif (
|
||||
moe_config.has_hash_routing
|
||||
and cls.is_monolithic()
|
||||
and not cls._supports_hash_routing()
|
||||
):
|
||||
return False, _make_reason("hash routing")
|
||||
elif moe_config.require_decomposed_backend and cls.is_monolithic():
|
||||
return False, _make_reason("routes adjusted after routing")
|
||||
@@ -1584,11 +1593,20 @@ class FusedMoEKernelMonolithicImpl:
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Same as forward(), except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is used for kernels
|
||||
that have fused router + experts (e.g. FLASHINFER_TRTLLM).
|
||||
"""
|
||||
# Only experts with _supports_hash_routing() accept these.
|
||||
hash_kwargs: dict[str, Any] = {}
|
||||
if hash_indices_table is not None:
|
||||
hash_kwargs = {
|
||||
"input_ids": input_ids,
|
||||
"hash_indices_table": hash_indices_table,
|
||||
}
|
||||
a1q, a1q_scale, router_logits = self.prepare_finalize.prepare(
|
||||
hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -1621,6 +1639,7 @@ class FusedMoEKernelMonolithicImpl:
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_replay_out=routing_replay_out,
|
||||
**hash_kwargs,
|
||||
)
|
||||
if routing_sink is not None:
|
||||
routing_sink.capture_fn(routing_replay_out)
|
||||
@@ -1733,6 +1752,8 @@ class FusedMoEKernel:
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
|
||||
return self.impl.apply(
|
||||
@@ -1749,6 +1770,8 @@ class FusedMoEKernel:
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_sink=routing_sink,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -79,6 +79,9 @@ class RoutedExperts(PluggableLayer):
|
||||
- Executing routed experts via quant_method.apply()
|
||||
"""
|
||||
|
||||
# Hash-routing table, owned by the gate; set by FusedMoEFactory.
|
||||
hash_indices_table: torch.Tensor | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_name: str,
|
||||
|
||||
@@ -479,6 +479,8 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=layer.hash_indices_table,
|
||||
)
|
||||
|
||||
|
||||
@@ -1132,4 +1134,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=layer.hash_indices_table,
|
||||
)
|
||||
|
||||
@@ -216,14 +216,6 @@ class DeepseekV4MoE(nn.Module):
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
router_logits_dtype=torch.float32,
|
||||
)
|
||||
if is_hash_moe:
|
||||
# apply_monolithic() never sees hash_indices_table via the
|
||||
# router path (modular-only) -- stash it on RoutedExperts
|
||||
# directly. object.__setattr__ avoids nn.Module re-registering
|
||||
# (and duplicating in the state_dict) this alias.
|
||||
object.__setattr__(
|
||||
self.experts.routed_experts, "hash_indices_table", self.gate.tid2eid
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
|
||||
|
||||
Reference in New Issue
Block a user