[ROCm][AITER] Hotfix for memory access fault errors in AITER triton MOE routing (#50859)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
74295e3bd4
commit
385d4c084e
@@ -19,18 +19,38 @@ from dataclasses import dataclass
|
|||||||
import huggingface_hub
|
import huggingface_hub
|
||||||
import lm_eval
|
import lm_eval
|
||||||
import pytest
|
import pytest
|
||||||
|
import torch
|
||||||
from packaging import version
|
from packaging import version
|
||||||
|
|
||||||
|
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||||
|
from vllm.config import get_current_vllm_config
|
||||||
|
from vllm.forward_context import set_forward_context
|
||||||
|
from vllm.model_executor.layers.fused_moe.config import (
|
||||||
|
mxfp4_w4a8_moe_quant_config,
|
||||||
|
mxfp4_w4a16_moe_quant_config,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
|
||||||
|
aiter_triton_kernel_w4a8_moe_forward,
|
||||||
|
aiter_triton_kernel_w4a16_moe_forward,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||||
|
Mxfp4MoeBackend,
|
||||||
|
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||||
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.transformers_utils.repo_utils import hf_api
|
from vllm.transformers_utils.repo_utils import hf_api
|
||||||
|
from vllm.utils.torch_utils import set_random_seed
|
||||||
|
|
||||||
if current_platform.is_rocm():
|
if current_platform.is_rocm():
|
||||||
from vllm.platforms.rocm import on_gfx950
|
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||||
else:
|
else:
|
||||||
|
|
||||||
def on_gfx950() -> bool:
|
def on_gfx950() -> bool:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def on_gfx1250() -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
MODEL_ACCURACIES = {
|
MODEL_ACCURACIES = {
|
||||||
# Full quantization: attention linears and MoE linears
|
# Full quantization: attention linears and MoE linears
|
||||||
@@ -128,3 +148,194 @@ def test_gpt_oss_attention_quantization(
|
|||||||
f"Accuracy {measured_accuracy:.4f} is below threshold "
|
f"Accuracy {measured_accuracy:.4f} is below threshold "
|
||||||
f"{expected_accuracy - rtol:.4f} (expected >= {expected_accuracy} - {rtol})"
|
f"{expected_accuracy - rtol:.4f} (expected >= {expected_accuracy} - {rtol})"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
not (on_gfx950() or on_gfx1250()),
|
||||||
|
reason="AITER MXFP4 MoE requires gfx950 or gfx1250",
|
||||||
|
)
|
||||||
|
@pytest.mark.skipif(not is_aiter_found_and_supported(), reason="aiter is not installed")
|
||||||
|
@pytest.mark.skipif(torch.accelerator.device_count() == 0, reason="no gpu available")
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"mxfp4_backend",
|
||||||
|
[Mxfp4MoeBackend.AITER_MXFP4_FP8, Mxfp4MoeBackend.AITER_MXFP4_BF16],
|
||||||
|
ids=["w4a8", "w4a16"],
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("num_experts", [32, 64, 128])
|
||||||
|
def test_aiter_mxfp4_moe_ignores_padded_rows(
|
||||||
|
mxfp4_backend: Mxfp4MoeBackend,
|
||||||
|
num_experts: int,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
dist_init,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Garbage in cudagraph padding rows must not reach the unpadded outputs.
|
||||||
|
|
||||||
|
A cudagraph replay of a size-`TOKENS_PADDED` graph driven by
|
||||||
|
`TOKENS_UNPADDED` real tokens leaves the trailing padding row containing
|
||||||
|
-inf/inf/nan.
|
||||||
|
"""
|
||||||
|
TOKENS_PADDED = 8
|
||||||
|
TOKENS_UNPADDED = 7
|
||||||
|
|
||||||
|
# `rocm_aiter_ops` snapshots the environment at import time, so setting the
|
||||||
|
# env var alone is not enough.
|
||||||
|
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||||
|
rocm_aiter_ops.refresh_env_variables()
|
||||||
|
|
||||||
|
set_random_seed(0)
|
||||||
|
device = torch.device("cuda")
|
||||||
|
hidden_size = 2048
|
||||||
|
intermediate_size = 512
|
||||||
|
|
||||||
|
num_experts = 32
|
||||||
|
topk = 4
|
||||||
|
is_w4a8 = mxfp4_backend is Mxfp4MoeBackend.AITER_MXFP4_FP8
|
||||||
|
|
||||||
|
# TODO: clean this in oracle/mxfp4.
|
||||||
|
# `convert_gpt_oss_weight_to_mxfp4_moe_kernel_format` maps
|
||||||
|
# AITER_MXFP4_BF16 onto the CK layout (`shuffle_weight_a16w4`), which the
|
||||||
|
# triton monolithic cannot consume -- it wants the `_swizzle_mxfp4`
|
||||||
|
# PrecisionConfig that the TRITON branch of the same converter produces.
|
||||||
|
weight_backend = (
|
||||||
|
Mxfp4MoeBackend.AITER_MXFP4_FP8 if is_w4a8 else Mxfp4MoeBackend.TRITON
|
||||||
|
)
|
||||||
|
|
||||||
|
layer = torch.nn.Module()
|
||||||
|
layer.w13_weight = torch.randint(
|
||||||
|
0,
|
||||||
|
256,
|
||||||
|
(num_experts, 2 * intermediate_size, hidden_size // 2),
|
||||||
|
dtype=torch.uint8,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
layer.w2_weight = torch.randint(
|
||||||
|
0,
|
||||||
|
256,
|
||||||
|
(num_experts, hidden_size, intermediate_size // 2),
|
||||||
|
dtype=torch.uint8,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Keep the exponents near 127 (2**0) so the dequantized weights stay in a sane
|
||||||
|
# range and 255 (e8m0 NaN) is never hit.
|
||||||
|
layer.w13_weight_scale = torch.randint(
|
||||||
|
124,
|
||||||
|
131,
|
||||||
|
(num_experts, 2 * intermediate_size, hidden_size // 32),
|
||||||
|
dtype=torch.uint8,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
layer.w2_weight_scale = torch.randint(
|
||||||
|
124,
|
||||||
|
131,
|
||||||
|
(num_experts, hidden_size, intermediate_size // 32),
|
||||||
|
dtype=torch.uint8,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
layer.w13_bias = torch.randn(
|
||||||
|
(num_experts, 2 * intermediate_size), dtype=torch.float32, device=device
|
||||||
|
)
|
||||||
|
layer.w2_bias = torch.randn(
|
||||||
|
(num_experts, hidden_size), dtype=torch.float32, device=device
|
||||||
|
)
|
||||||
|
# Read off `layer`, not off the arguments, by the AITER_MXFP4_FP8 branch.
|
||||||
|
layer.w13_input_scale = torch.full(
|
||||||
|
(num_experts,), 0.1, dtype=torch.float32, device=device
|
||||||
|
)
|
||||||
|
layer.w2_input_scale = torch.full(
|
||||||
|
(num_experts,), 0.1, dtype=torch.float32, device=device
|
||||||
|
)
|
||||||
|
|
||||||
|
(
|
||||||
|
w13_weight,
|
||||||
|
w2_weight,
|
||||||
|
w13_scale,
|
||||||
|
w2_scale,
|
||||||
|
w13_bias,
|
||||||
|
w2_bias,
|
||||||
|
) = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||||
|
weight_backend,
|
||||||
|
layer,
|
||||||
|
layer.w13_weight,
|
||||||
|
layer.w2_weight,
|
||||||
|
layer.w13_weight_scale,
|
||||||
|
layer.w2_weight_scale,
|
||||||
|
w13_bias=layer.w13_bias,
|
||||||
|
w2_bias=layer.w2_bias,
|
||||||
|
)
|
||||||
|
|
||||||
|
quant_config_factory = (
|
||||||
|
mxfp4_w4a8_moe_quant_config if is_w4a8 else mxfp4_w4a16_moe_quant_config
|
||||||
|
)
|
||||||
|
quant_config = quant_config_factory(
|
||||||
|
w1_scale=w13_scale,
|
||||||
|
w2_scale=w2_scale,
|
||||||
|
w1_bias=w13_bias,
|
||||||
|
w2_bias=w2_bias,
|
||||||
|
)
|
||||||
|
|
||||||
|
hidden_states = torch.randn(
|
||||||
|
(TOKENS_PADDED, hidden_size), dtype=torch.bfloat16, device=device
|
||||||
|
)
|
||||||
|
gating_output = torch.randn(
|
||||||
|
(TOKENS_PADDED, num_experts), dtype=torch.bfloat16, device=device
|
||||||
|
)
|
||||||
|
|
||||||
|
hidden_states[TOKENS_UNPADDED:, 0] = float("inf")
|
||||||
|
hidden_states[TOKENS_UNPADDED:, 1] = float("-inf")
|
||||||
|
hidden_states[TOKENS_UNPADDED:, 2] = float("nan")
|
||||||
|
|
||||||
|
gating_output[TOKENS_UNPADDED:, :] = float("-inf")
|
||||||
|
gating_output[TOKENS_UNPADDED:, 0] = float("inf")
|
||||||
|
gating_output[TOKENS_UNPADDED:, 1] = float("nan")
|
||||||
|
gating_output[TOKENS_UNPADDED:, 2] = -float("nan")
|
||||||
|
|
||||||
|
is_padding = torch.zeros(TOKENS_PADDED, dtype=torch.bool, device=device)
|
||||||
|
is_padding[TOKENS_UNPADDED:] = True
|
||||||
|
|
||||||
|
# Poison memory: after `del blocks`, further `torch.empty` in routing with wrongful
|
||||||
|
# expert ids may read into these memory sections resulting in
|
||||||
|
# potential memory access fault.
|
||||||
|
blocks = []
|
||||||
|
for numel in (16, 32, 64, 128, 256, 512, 1024, 4096, 1 << 14, 1 << 16, 1 << 20):
|
||||||
|
blocks.append(torch.full((numel,), 100e7, dtype=torch.int32, device=device))
|
||||||
|
# return blocks
|
||||||
|
del blocks
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
|
||||||
|
with set_forward_context(None, get_current_vllm_config(), is_padding=is_padding):
|
||||||
|
if is_w4a8:
|
||||||
|
# AITER_MXFP4_FP8 routes through the triton monolithic expert, which
|
||||||
|
# does its own top-k inside `aiter.ops.triton.moe.moe_routing`.
|
||||||
|
output = aiter_triton_kernel_w4a8_moe_forward(
|
||||||
|
hidden_states=hidden_states,
|
||||||
|
w1=w13_weight,
|
||||||
|
w2=w2_weight,
|
||||||
|
gating_output=gating_output,
|
||||||
|
topk=topk,
|
||||||
|
renormalize=True,
|
||||||
|
quant_config=quant_config,
|
||||||
|
global_num_experts=num_experts,
|
||||||
|
unpadded_N_w1=2 * intermediate_size,
|
||||||
|
unpadded_K_w1=hidden_size,
|
||||||
|
unpadded_N_w2=hidden_size,
|
||||||
|
unpadded_K_w2=intermediate_size,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
output = aiter_triton_kernel_w4a16_moe_forward(
|
||||||
|
hidden_states=hidden_states,
|
||||||
|
w1=w13_weight,
|
||||||
|
w2=w2_weight,
|
||||||
|
gating_output=gating_output,
|
||||||
|
topk=topk,
|
||||||
|
renormalize=True,
|
||||||
|
quant_config=quant_config,
|
||||||
|
global_num_experts=num_experts,
|
||||||
|
unpadded_N_w1=2 * intermediate_size,
|
||||||
|
unpadded_K_w1=hidden_size,
|
||||||
|
unpadded_N_w2=hidden_size,
|
||||||
|
unpadded_K_w2=intermediate_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert torch.isfinite(output[:TOKENS_UNPADDED]).all()
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import torch
|
|||||||
|
|
||||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||||
from vllm._aiter_ops import rocm_aiter_ops
|
from vllm._aiter_ops import rocm_aiter_ops
|
||||||
|
from vllm.forward_context import get_forward_context
|
||||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||||
from vllm.model_executor.layers.fused_moe.config import (
|
from vllm.model_executor.layers.fused_moe.config import (
|
||||||
FusedMoEConfig,
|
FusedMoEConfig,
|
||||||
@@ -26,6 +27,52 @@ __all__ = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _get_padding_mask() -> torch.Tensor | None:
|
||||||
|
"""
|
||||||
|
Retrieves a boolean mask with non-padding (0) and padding (1) tokens.
|
||||||
|
|
||||||
|
`slot_mapping < 0` comes from:
|
||||||
|
|
||||||
|
slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1)
|
||||||
|
|
||||||
|
in gpu_model_runner.py.
|
||||||
|
"""
|
||||||
|
forward_context = get_forward_context()
|
||||||
|
|
||||||
|
# model runner v2.
|
||||||
|
if forward_context.is_padding is not None:
|
||||||
|
return forward_context.is_padding
|
||||||
|
|
||||||
|
# model runner v1.
|
||||||
|
slot_mapping = forward_context.slot_mapping
|
||||||
|
|
||||||
|
if isinstance(slot_mapping, list):
|
||||||
|
slot_mapping_dict = slot_mapping[0]
|
||||||
|
else:
|
||||||
|
slot_mapping_dict = slot_mapping
|
||||||
|
|
||||||
|
if isinstance(slot_mapping_dict, dict):
|
||||||
|
slot_mapping_sample = next(iter(slot_mapping_dict.values()), None)
|
||||||
|
|
||||||
|
if isinstance(slot_mapping_sample, torch.Tensor):
|
||||||
|
return slot_mapping_sample < 0
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def patch_gating_output(
|
||||||
|
gating_output: torch.Tensor, global_num_experts: int
|
||||||
|
) -> torch.Tensor:
|
||||||
|
if global_num_experts != 128:
|
||||||
|
is_padding = _get_padding_mask()
|
||||||
|
|
||||||
|
if is_padding is not None:
|
||||||
|
assert is_padding.shape[0] == gating_output.shape[0]
|
||||||
|
gating_output = gating_output.masked_fill(is_padding[:, None], 0.0)
|
||||||
|
|
||||||
|
return gating_output
|
||||||
|
|
||||||
|
|
||||||
def aiter_triton_kernel_w4a8_moe_forward(
|
def aiter_triton_kernel_w4a8_moe_forward(
|
||||||
hidden_states: torch.Tensor,
|
hidden_states: torch.Tensor,
|
||||||
w1, # Tensor or triton_kernels.Tensor
|
w1, # Tensor or triton_kernels.Tensor
|
||||||
@@ -59,6 +106,11 @@ def aiter_triton_kernel_w4a8_moe_forward(
|
|||||||
_routing_mod.is_tdm_avail = lambda: False
|
_routing_mod.is_tdm_avail = lambda: False
|
||||||
aiter_routing = _routing_mod.routing
|
aiter_routing = _routing_mod.routing
|
||||||
|
|
||||||
|
# See context in #50859.
|
||||||
|
# TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged,
|
||||||
|
# AITER released, and AITER pin in vLLM increased from 0.1.19.
|
||||||
|
gating_output = patch_gating_output(gating_output, global_num_experts)
|
||||||
|
|
||||||
routing_data, gather_idx, scatter_idx = aiter_routing(
|
routing_data, gather_idx, scatter_idx = aiter_routing(
|
||||||
gating_output, topk, sm_first=not renormalize
|
gating_output, topk, sm_first=not renormalize
|
||||||
)
|
)
|
||||||
@@ -438,6 +490,11 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
|||||||
_routing_mod.is_tdm_avail = lambda: False
|
_routing_mod.is_tdm_avail = lambda: False
|
||||||
aiter_routing = _routing_mod.routing
|
aiter_routing = _routing_mod.routing
|
||||||
|
|
||||||
|
# See context in #50859.
|
||||||
|
# TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged,
|
||||||
|
# AITER released, and AITER pin in vLLM increased from 0.1.19.
|
||||||
|
gating_output = patch_gating_output(gating_output, global_num_experts)
|
||||||
|
|
||||||
if score_mode is not None:
|
if score_mode is not None:
|
||||||
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
|
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
|
||||||
routing_data, gather_idx, scatter_idx = aiter_routing(
|
routing_data, gather_idx, scatter_idx = aiter_routing(
|
||||||
|
|||||||
Reference in New Issue
Block a user