[ROCm][AITER] Hotfix for memory access fault errors in AITER triton MOE routing (#50859)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
74295e3bd4
commit
385d4c084e
@@ -5,6 +5,7 @@ import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -26,6 +27,52 @@ __all__ = [
|
||||
]
|
||||
|
||||
|
||||
def _get_padding_mask() -> torch.Tensor | None:
|
||||
"""
|
||||
Retrieves a boolean mask with non-padding (0) and padding (1) tokens.
|
||||
|
||||
`slot_mapping < 0` comes from:
|
||||
|
||||
slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1)
|
||||
|
||||
in gpu_model_runner.py.
|
||||
"""
|
||||
forward_context = get_forward_context()
|
||||
|
||||
# model runner v2.
|
||||
if forward_context.is_padding is not None:
|
||||
return forward_context.is_padding
|
||||
|
||||
# model runner v1.
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
|
||||
if isinstance(slot_mapping, list):
|
||||
slot_mapping_dict = slot_mapping[0]
|
||||
else:
|
||||
slot_mapping_dict = slot_mapping
|
||||
|
||||
if isinstance(slot_mapping_dict, dict):
|
||||
slot_mapping_sample = next(iter(slot_mapping_dict.values()), None)
|
||||
|
||||
if isinstance(slot_mapping_sample, torch.Tensor):
|
||||
return slot_mapping_sample < 0
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def patch_gating_output(
|
||||
gating_output: torch.Tensor, global_num_experts: int
|
||||
) -> torch.Tensor:
|
||||
if global_num_experts != 128:
|
||||
is_padding = _get_padding_mask()
|
||||
|
||||
if is_padding is not None:
|
||||
assert is_padding.shape[0] == gating_output.shape[0]
|
||||
gating_output = gating_output.masked_fill(is_padding[:, None], 0.0)
|
||||
|
||||
return gating_output
|
||||
|
||||
|
||||
def aiter_triton_kernel_w4a8_moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
w1, # Tensor or triton_kernels.Tensor
|
||||
@@ -59,6 +106,11 @@ def aiter_triton_kernel_w4a8_moe_forward(
|
||||
_routing_mod.is_tdm_avail = lambda: False
|
||||
aiter_routing = _routing_mod.routing
|
||||
|
||||
# See context in #50859.
|
||||
# TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged,
|
||||
# AITER released, and AITER pin in vLLM increased from 0.1.19.
|
||||
gating_output = patch_gating_output(gating_output, global_num_experts)
|
||||
|
||||
routing_data, gather_idx, scatter_idx = aiter_routing(
|
||||
gating_output, topk, sm_first=not renormalize
|
||||
)
|
||||
@@ -438,6 +490,11 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
||||
_routing_mod.is_tdm_avail = lambda: False
|
||||
aiter_routing = _routing_mod.routing
|
||||
|
||||
# See context in #50859.
|
||||
# TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged,
|
||||
# AITER released, and AITER pin in vLLM increased from 0.1.19.
|
||||
gating_output = patch_gating_output(gating_output, global_num_experts)
|
||||
|
||||
if score_mode is not None:
|
||||
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
|
||||
routing_data, gather_idx, scatter_idx = aiter_routing(
|
||||
|
||||
Reference in New Issue
Block a user