From 0de0362ea1c69b93f9ed36126a1b5c94f0ce2f22 Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Fri, 7 Aug 2026 10:19:26 +0200 Subject: [PATCH] [ROCm][CI] Loosen block-FP8 fused MoE test tolerance for large-K shapes (#48847) Signed-off-by: Stefan Koncarevic Signed-off-by: Andreas Karatzas Co-authored-by: Andreas Karatzas Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/kernels/moe/test_block_fp8.py | 56 +++++++++++++++---- .../fused_moe/experts/aiter_mxfp4_w4a8_moe.py | 11 +++- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/tests/kernels/moe/test_block_fp8.py b/tests/kernels/moe/test_block_fp8.py index 9a47c6e1d2..131e303fe3 100644 --- a/tests/kernels/moe/test_block_fp8.py +++ b/tests/kernels/moe/test_block_fp8.py @@ -11,10 +11,7 @@ from tests.kernels.moe.utils import ( make_test_weights, modular_triton_fused_moe, ) -from tests.kernels.quant_utils import ( - native_per_token_group_quant_fp8, - native_w8a8_block_matmul, -) +from tests.kernels.quant_utils import native_w8a8_block_matmul from vllm.config import VllmConfig, set_current_vllm_config from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.fused_moe import ( @@ -34,6 +31,9 @@ from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import ( from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import ( TritonOrDeepGemmExperts, ) +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + per_token_group_quant_fp8, +) from vllm.platforms import current_platform from vllm.utils.deep_gemm import ( get_mk_alignment_for_contiguous_layout, @@ -105,8 +105,16 @@ TOP_KS = [1, 2, 6] SEEDS = [0] -def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape): - """Fused moe with block-wise quantization using native torch.""" +def torch_w8a8_block_fp8_moe( + a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape, silu_fp32=False +): + """Fused MoE with block-wise fp8 quantization using native torch. + + silu_fp32=True computes the intermediate SiLU in fp32 and quantizes it + directly, matching the modular Helion silu_and_mul_per_block_quant kernel; + False rounds the SiLU output to bf16 first, matching fused_experts. Each + kernel is checked against the reference variant matching its precision. + """ B, D = a.shape topk = topk_ids.size(1) a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) @@ -116,7 +124,10 @@ def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block topk_ids = topk_ids.view(-1) _, block_k = block_shape[0], block_shape[1] - a_q, a_s = native_per_token_group_quant_fp8(a, block_k) + # Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the + # kernels use) so the reference is bit-identical here; removes ~0.06-0.10% fp8 + # boundary-flip divergence that otherwise accumulates over K. Matmul stays fp32. + a_q, a_s = per_token_group_quant_fp8(a, block_k, dtype=current_platform.fp8_dtype()) a_q = a_q.to(torch.float32) for i in range(w1.shape[0]): mask = topk_ids == i @@ -124,8 +135,12 @@ def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block inter_out = native_w8a8_block_matmul( a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype ) - act_out = SiluAndMul().forward_native(inter_out) - act_out_q, act_out_s = native_per_token_group_quant_fp8(act_out, block_k) + act_out = SiluAndMul().forward_native( + inter_out.float() if silu_fp32 else inter_out + ) + act_out_q, act_out_s = per_token_group_quant_fp8( + act_out, block_k, dtype=current_platform.fp8_dtype() + ) out[mask] = native_w8a8_block_matmul( act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype ) @@ -206,8 +221,29 @@ def test_w8a8_block_fp8_fused_moe( # 0.039 only needed for M >= 8192 tol = 0.035 if M < 8192 else 0.039 + + # The modular path fuses SiLU+quant in fp32 (silu_and_mul_per_block_quant), + # while fused_experts/the reference round SiLU to bf16 first. On large K/N this + # ~1-ULP gap pushes m_out past the base tol, so validate m_out against an + # fp32-SiLU reference — keeping the tight base tolerance, no widened override. + if current_platform.is_rocm() and K >= 4096 and N >= 1024: + with set_current_vllm_config(vllm_config): + ref_out_m = torch_w8a8_block_fp8_moe( + a, + w1, + w2, + quant_config.w1_scale, + quant_config.w2_scale, + topk_weights, + topk_ids, + block_size, + silu_fp32=True, + ) + else: + ref_out_m = ref_out + torch.testing.assert_close(out, ref_out, atol=tol, rtol=tol) - torch.testing.assert_close(m_out, ref_out, atol=tol, rtol=tol) + torch.testing.assert_close(m_out, ref_out_m, atol=tol, rtol=tol) @pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG) diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py index abc0498645..8d68ba38f7 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py @@ -5,7 +5,7 @@ import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm._aiter_ops import rocm_aiter_ops -from vllm.forward_context import get_forward_context +from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, @@ -27,6 +27,9 @@ __all__ = [ ] +# TODO(akaratza): Remove +# _get_padding_mask and patch_gating_output after +# https://github.com/ROCm/aiter/pull/4530 is released and consumed by vLLM. def _get_padding_mask() -> torch.Tensor | None: """ Retrieves a boolean mask with non-padding (0) and padding (1) tokens. @@ -36,7 +39,13 @@ def _get_padding_mask() -> torch.Tensor | None: slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1) in gpu_model_runner.py. + + Direct kernel callers do not have model-runner padding metadata, so no + mask is needed when there is no forward context. """ + if not is_forward_context_available(): + return None + forward_context = get_forward_context() # model runner v2.