[ROCm][AITER] Hotfix for memory access fault errors in AITER triton MOE routing (#50859)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-03 21:07:46 -05:00
committed by GitHub
co-authored by Andreas Karatzas
parent 74295e3bd4
commit 385d4c084e
2 changed files with 269 additions and 1 deletions
@@ -5,6 +5,7 @@ import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import rocm_aiter_ops
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -26,6 +27,52 @@ __all__ = [
]
def _get_padding_mask() -> torch.Tensor | None:
"""
Retrieves a boolean mask with non-padding (0) and padding (1) tokens.
`slot_mapping < 0` comes from:
slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1)
in gpu_model_runner.py.
"""
forward_context = get_forward_context()
# model runner v2.
if forward_context.is_padding is not None:
return forward_context.is_padding
# model runner v1.
slot_mapping = forward_context.slot_mapping
if isinstance(slot_mapping, list):
slot_mapping_dict = slot_mapping[0]
else:
slot_mapping_dict = slot_mapping
if isinstance(slot_mapping_dict, dict):
slot_mapping_sample = next(iter(slot_mapping_dict.values()), None)
if isinstance(slot_mapping_sample, torch.Tensor):
return slot_mapping_sample < 0
else:
return None
def patch_gating_output(
gating_output: torch.Tensor, global_num_experts: int
) -> torch.Tensor:
if global_num_experts != 128:
is_padding = _get_padding_mask()
if is_padding is not None:
assert is_padding.shape[0] == gating_output.shape[0]
gating_output = gating_output.masked_fill(is_padding[:, None], 0.0)
return gating_output
def aiter_triton_kernel_w4a8_moe_forward(
hidden_states: torch.Tensor,
w1, # Tensor or triton_kernels.Tensor
@@ -59,6 +106,11 @@ def aiter_triton_kernel_w4a8_moe_forward(
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
# See context in #50859.
# TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged,
# AITER released, and AITER pin in vLLM increased from 0.1.19.
gating_output = patch_gating_output(gating_output, global_num_experts)
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output, topk, sm_first=not renormalize
)
@@ -438,6 +490,11 @@ def aiter_triton_kernel_w4a16_moe_forward(
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
# See context in #50859.
# TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged,
# AITER released, and AITER pin in vLLM increased from 0.1.19.
gating_output = patch_gating_output(gating_output, global_num_experts)
if score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(