diff --git a/tests/models/quantization/test_gpt_oss.py b/tests/models/quantization/test_gpt_oss.py index f7a7b98c3e..f1325f7147 100644 --- a/tests/models/quantization/test_gpt_oss.py +++ b/tests/models/quantization/test_gpt_oss.py @@ -19,18 +19,38 @@ from dataclasses import dataclass import huggingface_hub import lm_eval import pytest +import torch from packaging import version +from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops +from vllm.config import get_current_vllm_config +from vllm.forward_context import set_forward_context +from vllm.model_executor.layers.fused_moe.config import ( + mxfp4_w4a8_moe_quant_config, + mxfp4_w4a16_moe_quant_config, +) +from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import ( + aiter_triton_kernel_w4a8_moe_forward, + aiter_triton_kernel_w4a16_moe_forward, +) +from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + Mxfp4MoeBackend, + convert_gpt_oss_weight_to_mxfp4_moe_kernel_format, +) from vllm.platforms import current_platform from vllm.transformers_utils.repo_utils import hf_api +from vllm.utils.torch_utils import set_random_seed if current_platform.is_rocm(): - from vllm.platforms.rocm import on_gfx950 + from vllm.platforms.rocm import on_gfx950, on_gfx1250 else: def on_gfx950() -> bool: return False + def on_gfx1250() -> bool: + return False + MODEL_ACCURACIES = { # Full quantization: attention linears and MoE linears @@ -128,3 +148,194 @@ def test_gpt_oss_attention_quantization( f"Accuracy {measured_accuracy:.4f} is below threshold " f"{expected_accuracy - rtol:.4f} (expected >= {expected_accuracy} - {rtol})" ) + + +@pytest.mark.skipif( + not (on_gfx950() or on_gfx1250()), + reason="AITER MXFP4 MoE requires gfx950 or gfx1250", +) +@pytest.mark.skipif(not is_aiter_found_and_supported(), reason="aiter is not installed") +@pytest.mark.skipif(torch.accelerator.device_count() == 0, reason="no gpu available") +@pytest.mark.parametrize( + "mxfp4_backend", + [Mxfp4MoeBackend.AITER_MXFP4_FP8, Mxfp4MoeBackend.AITER_MXFP4_BF16], + ids=["w4a8", "w4a16"], +) +@pytest.mark.parametrize("num_experts", [32, 64, 128]) +def test_aiter_mxfp4_moe_ignores_padded_rows( + mxfp4_backend: Mxfp4MoeBackend, + num_experts: int, + monkeypatch: pytest.MonkeyPatch, + dist_init, +) -> None: + """ + Garbage in cudagraph padding rows must not reach the unpadded outputs. + + A cudagraph replay of a size-`TOKENS_PADDED` graph driven by + `TOKENS_UNPADDED` real tokens leaves the trailing padding row containing + -inf/inf/nan. + """ + TOKENS_PADDED = 8 + TOKENS_UNPADDED = 7 + + # `rocm_aiter_ops` snapshots the environment at import time, so setting the + # env var alone is not enough. + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + rocm_aiter_ops.refresh_env_variables() + + set_random_seed(0) + device = torch.device("cuda") + hidden_size = 2048 + intermediate_size = 512 + + num_experts = 32 + topk = 4 + is_w4a8 = mxfp4_backend is Mxfp4MoeBackend.AITER_MXFP4_FP8 + + # TODO: clean this in oracle/mxfp4. + # `convert_gpt_oss_weight_to_mxfp4_moe_kernel_format` maps + # AITER_MXFP4_BF16 onto the CK layout (`shuffle_weight_a16w4`), which the + # triton monolithic cannot consume -- it wants the `_swizzle_mxfp4` + # PrecisionConfig that the TRITON branch of the same converter produces. + weight_backend = ( + Mxfp4MoeBackend.AITER_MXFP4_FP8 if is_w4a8 else Mxfp4MoeBackend.TRITON + ) + + layer = torch.nn.Module() + layer.w13_weight = torch.randint( + 0, + 256, + (num_experts, 2 * intermediate_size, hidden_size // 2), + dtype=torch.uint8, + device=device, + ) + layer.w2_weight = torch.randint( + 0, + 256, + (num_experts, hidden_size, intermediate_size // 2), + dtype=torch.uint8, + device=device, + ) + + # Keep the exponents near 127 (2**0) so the dequantized weights stay in a sane + # range and 255 (e8m0 NaN) is never hit. + layer.w13_weight_scale = torch.randint( + 124, + 131, + (num_experts, 2 * intermediate_size, hidden_size // 32), + dtype=torch.uint8, + device=device, + ) + layer.w2_weight_scale = torch.randint( + 124, + 131, + (num_experts, hidden_size, intermediate_size // 32), + dtype=torch.uint8, + device=device, + ) + layer.w13_bias = torch.randn( + (num_experts, 2 * intermediate_size), dtype=torch.float32, device=device + ) + layer.w2_bias = torch.randn( + (num_experts, hidden_size), dtype=torch.float32, device=device + ) + # Read off `layer`, not off the arguments, by the AITER_MXFP4_FP8 branch. + layer.w13_input_scale = torch.full( + (num_experts,), 0.1, dtype=torch.float32, device=device + ) + layer.w2_input_scale = torch.full( + (num_experts,), 0.1, dtype=torch.float32, device=device + ) + + ( + w13_weight, + w2_weight, + w13_scale, + w2_scale, + w13_bias, + w2_bias, + ) = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format( + weight_backend, + layer, + layer.w13_weight, + layer.w2_weight, + layer.w13_weight_scale, + layer.w2_weight_scale, + w13_bias=layer.w13_bias, + w2_bias=layer.w2_bias, + ) + + quant_config_factory = ( + mxfp4_w4a8_moe_quant_config if is_w4a8 else mxfp4_w4a16_moe_quant_config + ) + quant_config = quant_config_factory( + w1_scale=w13_scale, + w2_scale=w2_scale, + w1_bias=w13_bias, + w2_bias=w2_bias, + ) + + hidden_states = torch.randn( + (TOKENS_PADDED, hidden_size), dtype=torch.bfloat16, device=device + ) + gating_output = torch.randn( + (TOKENS_PADDED, num_experts), dtype=torch.bfloat16, device=device + ) + + hidden_states[TOKENS_UNPADDED:, 0] = float("inf") + hidden_states[TOKENS_UNPADDED:, 1] = float("-inf") + hidden_states[TOKENS_UNPADDED:, 2] = float("nan") + + gating_output[TOKENS_UNPADDED:, :] = float("-inf") + gating_output[TOKENS_UNPADDED:, 0] = float("inf") + gating_output[TOKENS_UNPADDED:, 1] = float("nan") + gating_output[TOKENS_UNPADDED:, 2] = -float("nan") + + is_padding = torch.zeros(TOKENS_PADDED, dtype=torch.bool, device=device) + is_padding[TOKENS_UNPADDED:] = True + + # Poison memory: after `del blocks`, further `torch.empty` in routing with wrongful + # expert ids may read into these memory sections resulting in + # potential memory access fault. + blocks = [] + for numel in (16, 32, 64, 128, 256, 512, 1024, 4096, 1 << 14, 1 << 16, 1 << 20): + blocks.append(torch.full((numel,), 100e7, dtype=torch.int32, device=device)) + # return blocks + del blocks + torch.accelerator.synchronize() + + with set_forward_context(None, get_current_vllm_config(), is_padding=is_padding): + if is_w4a8: + # AITER_MXFP4_FP8 routes through the triton monolithic expert, which + # does its own top-k inside `aiter.ops.triton.moe.moe_routing`. + output = aiter_triton_kernel_w4a8_moe_forward( + hidden_states=hidden_states, + w1=w13_weight, + w2=w2_weight, + gating_output=gating_output, + topk=topk, + renormalize=True, + quant_config=quant_config, + global_num_experts=num_experts, + unpadded_N_w1=2 * intermediate_size, + unpadded_K_w1=hidden_size, + unpadded_N_w2=hidden_size, + unpadded_K_w2=intermediate_size, + ) + else: + output = aiter_triton_kernel_w4a16_moe_forward( + hidden_states=hidden_states, + w1=w13_weight, + w2=w2_weight, + gating_output=gating_output, + topk=topk, + renormalize=True, + quant_config=quant_config, + global_num_experts=num_experts, + unpadded_N_w1=2 * intermediate_size, + unpadded_K_w1=hidden_size, + unpadded_N_w2=hidden_size, + unpadded_K_w2=intermediate_size, + ) + + assert torch.isfinite(output[:TOKENS_UNPADDED]).all() diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py index 53880cb916..abc0498645 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py @@ -5,6 +5,7 @@ import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm._aiter_ops import rocm_aiter_ops +from vllm.forward_context import get_forward_context from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, @@ -26,6 +27,52 @@ __all__ = [ ] +def _get_padding_mask() -> torch.Tensor | None: + """ + Retrieves a boolean mask with non-padding (0) and padding (1) tokens. + + `slot_mapping < 0` comes from: + + slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1) + + in gpu_model_runner.py. + """ + forward_context = get_forward_context() + + # model runner v2. + if forward_context.is_padding is not None: + return forward_context.is_padding + + # model runner v1. + slot_mapping = forward_context.slot_mapping + + if isinstance(slot_mapping, list): + slot_mapping_dict = slot_mapping[0] + else: + slot_mapping_dict = slot_mapping + + if isinstance(slot_mapping_dict, dict): + slot_mapping_sample = next(iter(slot_mapping_dict.values()), None) + + if isinstance(slot_mapping_sample, torch.Tensor): + return slot_mapping_sample < 0 + else: + return None + + +def patch_gating_output( + gating_output: torch.Tensor, global_num_experts: int +) -> torch.Tensor: + if global_num_experts != 128: + is_padding = _get_padding_mask() + + if is_padding is not None: + assert is_padding.shape[0] == gating_output.shape[0] + gating_output = gating_output.masked_fill(is_padding[:, None], 0.0) + + return gating_output + + def aiter_triton_kernel_w4a8_moe_forward( hidden_states: torch.Tensor, w1, # Tensor or triton_kernels.Tensor @@ -59,6 +106,11 @@ def aiter_triton_kernel_w4a8_moe_forward( _routing_mod.is_tdm_avail = lambda: False aiter_routing = _routing_mod.routing + # See context in #50859. + # TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged, + # AITER released, and AITER pin in vLLM increased from 0.1.19. + gating_output = patch_gating_output(gating_output, global_num_experts) + routing_data, gather_idx, scatter_idx = aiter_routing( gating_output, topk, sm_first=not renormalize ) @@ -438,6 +490,11 @@ def aiter_triton_kernel_w4a16_moe_forward( _routing_mod.is_tdm_avail = lambda: False aiter_routing = _routing_mod.routing + # See context in #50859. + # TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged, + # AITER released, and AITER pin in vLLM increased from 0.1.19. + gating_output = patch_gating_output(gating_output, global_num_experts) + if score_mode is not None: use_grouped_topk = num_expert_group is not None and num_expert_group > 1 routing_data, gather_idx, scatter_idx = aiter_routing(