[ROCm] Support DeepSeek-V4 hash routing in AITER Triton W4A16 MoE (#60282)
Signed-off-by: Rohan Potdar <rohan.potdar@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
82b9ccaa79
commit
105605b8e9
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
__all__ = [
|
||||
"AiterW4A16ExpertsMonolithic",
|
||||
@@ -134,6 +135,8 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
score_mode: str | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
):
|
||||
assert quant_config is not None and rocm_aiter_ops.is_enabled()
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
@@ -154,7 +157,25 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
||||
_routing_mod.is_tdm_avail = lambda: False
|
||||
aiter_routing = _routing_mod.routing
|
||||
|
||||
if score_mode is not None:
|
||||
scale = routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
if hash_indices_table is not None:
|
||||
assert score_mode == "sqrtsoftplus" and input_ids is not None
|
||||
# Same block_m heuristic as aiter's routing().
|
||||
n_tokens, n_experts = gating_output.shape
|
||||
block_m = max(
|
||||
16, min(triton.next_power_of_2(max(1, n_tokens * topk // n_experts)), 128)
|
||||
)
|
||||
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
|
||||
gating_output,
|
||||
hash_indices_table,
|
||||
input_ids,
|
||||
topk,
|
||||
block_m,
|
||||
score_mode=score_mode,
|
||||
renorm=renormalize,
|
||||
routed_scaling_factor=scale,
|
||||
)
|
||||
elif score_mode is not None:
|
||||
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
|
||||
routing_data, gather_idx, scatter_idx = aiter_routing(
|
||||
gating_output,
|
||||
@@ -162,9 +183,7 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
||||
score_mode=score_mode,
|
||||
bias=e_score_correction_bias,
|
||||
renorm=renormalize,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
routed_scaling_factor=scale,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
@@ -348,6 +367,10 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_hash_routing() -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
@@ -375,6 +398,8 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
routing_method = self.moe_config.routing_method
|
||||
if routing_method == RoutingMethodType.DeepseekV4:
|
||||
@@ -411,4 +436,6 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
score_mode=score_mode,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
@@ -438,6 +438,8 @@ def FusedMoEFactory(
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
**routed_experts_args if routed_experts_args is not None else {},
|
||||
)
|
||||
if hash_indices_table is not None:
|
||||
object.__setattr__(routed_experts, "hash_indices_table", hash_indices_table)
|
||||
|
||||
if runner_cls is None:
|
||||
runner_cls = MoERunner
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from math import prod
|
||||
from typing import final
|
||||
from typing import Any, final
|
||||
|
||||
import torch
|
||||
|
||||
@@ -531,6 +531,11 @@ class FusedMoEExperts(ABC):
|
||||
def is_monolithic() -> bool:
|
||||
raise NotImplementedError("Implemented by subclasses.")
|
||||
|
||||
@staticmethod
|
||||
def _supports_hash_routing() -> bool:
|
||||
"""Whether a monolithic kernel implements hash routing."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
"""Whether or not the PrepareFinalize should defer input quantization
|
||||
@@ -579,7 +584,11 @@ class FusedMoEExperts(ABC):
|
||||
return False, _make_reason(
|
||||
f"parallel config {moe_config.moe_parallel_config}"
|
||||
)
|
||||
elif moe_config.has_hash_routing and cls.is_monolithic():
|
||||
elif (
|
||||
moe_config.has_hash_routing
|
||||
and cls.is_monolithic()
|
||||
and not cls._supports_hash_routing()
|
||||
):
|
||||
return False, _make_reason("hash routing")
|
||||
elif moe_config.require_decomposed_backend and cls.is_monolithic():
|
||||
return False, _make_reason("routes adjusted after routing")
|
||||
@@ -1584,11 +1593,20 @@ class FusedMoEKernelMonolithicImpl:
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Same as forward(), except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is used for kernels
|
||||
that have fused router + experts (e.g. FLASHINFER_TRTLLM).
|
||||
"""
|
||||
# Only experts with _supports_hash_routing() accept these.
|
||||
hash_kwargs: dict[str, Any] = {}
|
||||
if hash_indices_table is not None:
|
||||
hash_kwargs = {
|
||||
"input_ids": input_ids,
|
||||
"hash_indices_table": hash_indices_table,
|
||||
}
|
||||
a1q, a1q_scale, router_logits = self.prepare_finalize.prepare(
|
||||
hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -1621,6 +1639,7 @@ class FusedMoEKernelMonolithicImpl:
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_replay_out=routing_replay_out,
|
||||
**hash_kwargs,
|
||||
)
|
||||
if routing_sink is not None:
|
||||
routing_sink.capture_fn(routing_replay_out)
|
||||
@@ -1733,6 +1752,8 @@ class FusedMoEKernel:
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
|
||||
return self.impl.apply(
|
||||
@@ -1749,6 +1770,8 @@ class FusedMoEKernel:
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_sink=routing_sink,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -79,6 +79,9 @@ class RoutedExperts(PluggableLayer):
|
||||
- Executing routed experts via quant_method.apply()
|
||||
"""
|
||||
|
||||
# Hash-routing table, owned by the gate; set by FusedMoEFactory.
|
||||
hash_indices_table: torch.Tensor | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_name: str,
|
||||
|
||||
@@ -479,6 +479,8 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=layer.hash_indices_table,
|
||||
)
|
||||
|
||||
|
||||
@@ -1132,4 +1134,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=layer.hash_indices_table,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user