[ROCm][CI] Loosen block-FP8 fused MoE test tolerance for large-K shapes (#48847)

Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
stefankoncarevic
2026-08-07 01:19:26 -07:00
committed by GitHub
co-authored by Andreas Karatzas mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 58fcaa0baa
commit 0de0362ea1
2 changed files with 56 additions and 11 deletions
+46 -10
View File
@@ -11,10 +11,7 @@ from tests.kernels.moe.utils import (
make_test_weights,
modular_triton_fused_moe,
)
from tests.kernels.quant_utils import (
native_per_token_group_quant_fp8,
native_w8a8_block_matmul,
)
from tests.kernels.quant_utils import native_w8a8_block_matmul
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import (
@@ -34,6 +31,9 @@ from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
get_mk_alignment_for_contiguous_layout,
@@ -105,8 +105,16 @@ TOP_KS = [1, 2, 6]
SEEDS = [0]
def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape):
"""Fused moe with block-wise quantization using native torch."""
def torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape, silu_fp32=False
):
"""Fused MoE with block-wise fp8 quantization using native torch.
silu_fp32=True computes the intermediate SiLU in fp32 and quantizes it
directly, matching the modular Helion silu_and_mul_per_block_quant kernel;
False rounds the SiLU output to bf16 first, matching fused_experts. Each
kernel is checked against the reference variant matching its precision.
"""
B, D = a.shape
topk = topk_ids.size(1)
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
@@ -116,7 +124,10 @@ def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block
topk_ids = topk_ids.view(-1)
_, block_k = block_shape[0], block_shape[1]
a_q, a_s = native_per_token_group_quant_fp8(a, block_k)
# Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the
# kernels use) so the reference is bit-identical here; removes ~0.06-0.10% fp8
# boundary-flip divergence that otherwise accumulates over K. Matmul stays fp32.
a_q, a_s = per_token_group_quant_fp8(a, block_k, dtype=current_platform.fp8_dtype())
a_q = a_q.to(torch.float32)
for i in range(w1.shape[0]):
mask = topk_ids == i
@@ -124,8 +135,12 @@ def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block
inter_out = native_w8a8_block_matmul(
a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype
)
act_out = SiluAndMul().forward_native(inter_out)
act_out_q, act_out_s = native_per_token_group_quant_fp8(act_out, block_k)
act_out = SiluAndMul().forward_native(
inter_out.float() if silu_fp32 else inter_out
)
act_out_q, act_out_s = per_token_group_quant_fp8(
act_out, block_k, dtype=current_platform.fp8_dtype()
)
out[mask] = native_w8a8_block_matmul(
act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype
)
@@ -206,8 +221,29 @@ def test_w8a8_block_fp8_fused_moe(
# 0.039 only needed for M >= 8192
tol = 0.035 if M < 8192 else 0.039
# The modular path fuses SiLU+quant in fp32 (silu_and_mul_per_block_quant),
# while fused_experts/the reference round SiLU to bf16 first. On large K/N this
# ~1-ULP gap pushes m_out past the base tol, so validate m_out against an
# fp32-SiLU reference — keeping the tight base tolerance, no widened override.
if current_platform.is_rocm() and K >= 4096 and N >= 1024:
with set_current_vllm_config(vllm_config):
ref_out_m = torch_w8a8_block_fp8_moe(
a,
w1,
w2,
quant_config.w1_scale,
quant_config.w2_scale,
topk_weights,
topk_ids,
block_size,
silu_fp32=True,
)
else:
ref_out_m = ref_out
torch.testing.assert_close(out, ref_out, atol=tol, rtol=tol)
torch.testing.assert_close(m_out, ref_out, atol=tol, rtol=tol)
torch.testing.assert_close(m_out, ref_out_m, atol=tol, rtol=tol)
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG)
@@ -5,7 +5,7 @@ import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import rocm_aiter_ops
from vllm.forward_context import get_forward_context
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -27,6 +27,9 @@ __all__ = [
]
# TODO(akaratza): Remove
# _get_padding_mask and patch_gating_output after
# https://github.com/ROCm/aiter/pull/4530 is released and consumed by vLLM.
def _get_padding_mask() -> torch.Tensor | None:
"""
Retrieves a boolean mask with non-padding (0) and padding (1) tokens.
@@ -36,7 +39,13 @@ def _get_padding_mask() -> torch.Tensor | None:
slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1)
in gpu_model_runner.py.
Direct kernel callers do not have model-runner padding metadata, so no
mask is needed when there is no forward context.
"""
if not is_forward_context_available():
return None
forward_context = get_forward_context()
# model runner v2.