[ROCm][AITER] Hotfix for memory access fault errors in AITER triton MOE routing (#50859)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-03 21:07:46 -05:00
committed by GitHub
co-authored by Andreas Karatzas
parent 74295e3bd4
commit 385d4c084e
2 changed files with 269 additions and 1 deletions
+212 -1
View File
@@ -19,18 +19,38 @@ from dataclasses import dataclass
import huggingface_hub
import lm_eval
import pytest
import torch
from packaging import version
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.config import get_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe.config import (
mxfp4_w4a8_moe_quant_config,
mxfp4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
aiter_triton_kernel_w4a8_moe_forward,
aiter_triton_kernel_w4a16_moe_forward,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.torch_utils import set_random_seed
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
from vllm.platforms.rocm import on_gfx950, on_gfx1250
else:
def on_gfx950() -> bool:
return False
def on_gfx1250() -> bool:
return False
MODEL_ACCURACIES = {
# Full quantization: attention linears and MoE linears
@@ -128,3 +148,194 @@ def test_gpt_oss_attention_quantization(
f"Accuracy {measured_accuracy:.4f} is below threshold "
f"{expected_accuracy - rtol:.4f} (expected >= {expected_accuracy} - {rtol})"
)
@pytest.mark.skipif(
not (on_gfx950() or on_gfx1250()),
reason="AITER MXFP4 MoE requires gfx950 or gfx1250",
)
@pytest.mark.skipif(not is_aiter_found_and_supported(), reason="aiter is not installed")
@pytest.mark.skipif(torch.accelerator.device_count() == 0, reason="no gpu available")
@pytest.mark.parametrize(
"mxfp4_backend",
[Mxfp4MoeBackend.AITER_MXFP4_FP8, Mxfp4MoeBackend.AITER_MXFP4_BF16],
ids=["w4a8", "w4a16"],
)
@pytest.mark.parametrize("num_experts", [32, 64, 128])
def test_aiter_mxfp4_moe_ignores_padded_rows(
mxfp4_backend: Mxfp4MoeBackend,
num_experts: int,
monkeypatch: pytest.MonkeyPatch,
dist_init,
) -> None:
"""
Garbage in cudagraph padding rows must not reach the unpadded outputs.
A cudagraph replay of a size-`TOKENS_PADDED` graph driven by
`TOKENS_UNPADDED` real tokens leaves the trailing padding row containing
-inf/inf/nan.
"""
TOKENS_PADDED = 8
TOKENS_UNPADDED = 7
# `rocm_aiter_ops` snapshots the environment at import time, so setting the
# env var alone is not enough.
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
set_random_seed(0)
device = torch.device("cuda")
hidden_size = 2048
intermediate_size = 512
num_experts = 32
topk = 4
is_w4a8 = mxfp4_backend is Mxfp4MoeBackend.AITER_MXFP4_FP8
# TODO: clean this in oracle/mxfp4.
# `convert_gpt_oss_weight_to_mxfp4_moe_kernel_format` maps
# AITER_MXFP4_BF16 onto the CK layout (`shuffle_weight_a16w4`), which the
# triton monolithic cannot consume -- it wants the `_swizzle_mxfp4`
# PrecisionConfig that the TRITON branch of the same converter produces.
weight_backend = (
Mxfp4MoeBackend.AITER_MXFP4_FP8 if is_w4a8 else Mxfp4MoeBackend.TRITON
)
layer = torch.nn.Module()
layer.w13_weight = torch.randint(
0,
256,
(num_experts, 2 * intermediate_size, hidden_size // 2),
dtype=torch.uint8,
device=device,
)
layer.w2_weight = torch.randint(
0,
256,
(num_experts, hidden_size, intermediate_size // 2),
dtype=torch.uint8,
device=device,
)
# Keep the exponents near 127 (2**0) so the dequantized weights stay in a sane
# range and 255 (e8m0 NaN) is never hit.
layer.w13_weight_scale = torch.randint(
124,
131,
(num_experts, 2 * intermediate_size, hidden_size // 32),
dtype=torch.uint8,
device=device,
)
layer.w2_weight_scale = torch.randint(
124,
131,
(num_experts, hidden_size, intermediate_size // 32),
dtype=torch.uint8,
device=device,
)
layer.w13_bias = torch.randn(
(num_experts, 2 * intermediate_size), dtype=torch.float32, device=device
)
layer.w2_bias = torch.randn(
(num_experts, hidden_size), dtype=torch.float32, device=device
)
# Read off `layer`, not off the arguments, by the AITER_MXFP4_FP8 branch.
layer.w13_input_scale = torch.full(
(num_experts,), 0.1, dtype=torch.float32, device=device
)
layer.w2_input_scale = torch.full(
(num_experts,), 0.1, dtype=torch.float32, device=device
)
(
w13_weight,
w2_weight,
w13_scale,
w2_scale,
w13_bias,
w2_bias,
) = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
weight_backend,
layer,
layer.w13_weight,
layer.w2_weight,
layer.w13_weight_scale,
layer.w2_weight_scale,
w13_bias=layer.w13_bias,
w2_bias=layer.w2_bias,
)
quant_config_factory = (
mxfp4_w4a8_moe_quant_config if is_w4a8 else mxfp4_w4a16_moe_quant_config
)
quant_config = quant_config_factory(
w1_scale=w13_scale,
w2_scale=w2_scale,
w1_bias=w13_bias,
w2_bias=w2_bias,
)
hidden_states = torch.randn(
(TOKENS_PADDED, hidden_size), dtype=torch.bfloat16, device=device
)
gating_output = torch.randn(
(TOKENS_PADDED, num_experts), dtype=torch.bfloat16, device=device
)
hidden_states[TOKENS_UNPADDED:, 0] = float("inf")
hidden_states[TOKENS_UNPADDED:, 1] = float("-inf")
hidden_states[TOKENS_UNPADDED:, 2] = float("nan")
gating_output[TOKENS_UNPADDED:, :] = float("-inf")
gating_output[TOKENS_UNPADDED:, 0] = float("inf")
gating_output[TOKENS_UNPADDED:, 1] = float("nan")
gating_output[TOKENS_UNPADDED:, 2] = -float("nan")
is_padding = torch.zeros(TOKENS_PADDED, dtype=torch.bool, device=device)
is_padding[TOKENS_UNPADDED:] = True
# Poison memory: after `del blocks`, further `torch.empty` in routing with wrongful
# expert ids may read into these memory sections resulting in
# potential memory access fault.
blocks = []
for numel in (16, 32, 64, 128, 256, 512, 1024, 4096, 1 << 14, 1 << 16, 1 << 20):
blocks.append(torch.full((numel,), 100e7, dtype=torch.int32, device=device))
# return blocks
del blocks
torch.accelerator.synchronize()
with set_forward_context(None, get_current_vllm_config(), is_padding=is_padding):
if is_w4a8:
# AITER_MXFP4_FP8 routes through the triton monolithic expert, which
# does its own top-k inside `aiter.ops.triton.moe.moe_routing`.
output = aiter_triton_kernel_w4a8_moe_forward(
hidden_states=hidden_states,
w1=w13_weight,
w2=w2_weight,
gating_output=gating_output,
topk=topk,
renormalize=True,
quant_config=quant_config,
global_num_experts=num_experts,
unpadded_N_w1=2 * intermediate_size,
unpadded_K_w1=hidden_size,
unpadded_N_w2=hidden_size,
unpadded_K_w2=intermediate_size,
)
else:
output = aiter_triton_kernel_w4a16_moe_forward(
hidden_states=hidden_states,
w1=w13_weight,
w2=w2_weight,
gating_output=gating_output,
topk=topk,
renormalize=True,
quant_config=quant_config,
global_num_experts=num_experts,
unpadded_N_w1=2 * intermediate_size,
unpadded_K_w1=hidden_size,
unpadded_N_w2=hidden_size,
unpadded_K_w2=intermediate_size,
)
assert torch.isfinite(output[:TOKENS_UNPADDED]).all()