[ROCm][CI] Loosen block-FP8 fused MoE test tolerance for large-K shapes (#48847)
Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Andreas Karatzas
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
58fcaa0baa
commit
0de0362ea1
@@ -11,10 +11,7 @@ from tests.kernels.moe.utils import (
|
||||
make_test_weights,
|
||||
modular_triton_fused_moe,
|
||||
)
|
||||
from tests.kernels.quant_utils import (
|
||||
native_per_token_group_quant_fp8,
|
||||
native_w8a8_block_matmul,
|
||||
)
|
||||
from tests.kernels.quant_utils import native_w8a8_block_matmul
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -34,6 +31,9 @@ from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
get_mk_alignment_for_contiguous_layout,
|
||||
@@ -105,8 +105,16 @@ TOP_KS = [1, 2, 6]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape):
|
||||
"""Fused moe with block-wise quantization using native torch."""
|
||||
def torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape, silu_fp32=False
|
||||
):
|
||||
"""Fused MoE with block-wise fp8 quantization using native torch.
|
||||
|
||||
silu_fp32=True computes the intermediate SiLU in fp32 and quantizes it
|
||||
directly, matching the modular Helion silu_and_mul_per_block_quant kernel;
|
||||
False rounds the SiLU output to bf16 first, matching fused_experts. Each
|
||||
kernel is checked against the reference variant matching its precision.
|
||||
"""
|
||||
B, D = a.shape
|
||||
topk = topk_ids.size(1)
|
||||
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
@@ -116,7 +124,10 @@ def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block
|
||||
topk_ids = topk_ids.view(-1)
|
||||
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
a_q, a_s = native_per_token_group_quant_fp8(a, block_k)
|
||||
# Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the
|
||||
# kernels use) so the reference is bit-identical here; removes ~0.06-0.10% fp8
|
||||
# boundary-flip divergence that otherwise accumulates over K. Matmul stays fp32.
|
||||
a_q, a_s = per_token_group_quant_fp8(a, block_k, dtype=current_platform.fp8_dtype())
|
||||
a_q = a_q.to(torch.float32)
|
||||
for i in range(w1.shape[0]):
|
||||
mask = topk_ids == i
|
||||
@@ -124,8 +135,12 @@ def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block
|
||||
inter_out = native_w8a8_block_matmul(
|
||||
a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype
|
||||
)
|
||||
act_out = SiluAndMul().forward_native(inter_out)
|
||||
act_out_q, act_out_s = native_per_token_group_quant_fp8(act_out, block_k)
|
||||
act_out = SiluAndMul().forward_native(
|
||||
inter_out.float() if silu_fp32 else inter_out
|
||||
)
|
||||
act_out_q, act_out_s = per_token_group_quant_fp8(
|
||||
act_out, block_k, dtype=current_platform.fp8_dtype()
|
||||
)
|
||||
out[mask] = native_w8a8_block_matmul(
|
||||
act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype
|
||||
)
|
||||
@@ -206,8 +221,29 @@ def test_w8a8_block_fp8_fused_moe(
|
||||
|
||||
# 0.039 only needed for M >= 8192
|
||||
tol = 0.035 if M < 8192 else 0.039
|
||||
|
||||
# The modular path fuses SiLU+quant in fp32 (silu_and_mul_per_block_quant),
|
||||
# while fused_experts/the reference round SiLU to bf16 first. On large K/N this
|
||||
# ~1-ULP gap pushes m_out past the base tol, so validate m_out against an
|
||||
# fp32-SiLU reference — keeping the tight base tolerance, no widened override.
|
||||
if current_platform.is_rocm() and K >= 4096 and N >= 1024:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out_m = torch_w8a8_block_fp8_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
quant_config.w1_scale,
|
||||
quant_config.w2_scale,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
block_size,
|
||||
silu_fp32=True,
|
||||
)
|
||||
else:
|
||||
ref_out_m = ref_out
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=tol, rtol=tol)
|
||||
torch.testing.assert_close(m_out, ref_out, atol=tol, rtol=tol)
|
||||
torch.testing.assert_close(m_out, ref_out_m, atol=tol, rtol=tol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG)
|
||||
|
||||
@@ -5,7 +5,7 @@ import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -27,6 +27,9 @@ __all__ = [
|
||||
]
|
||||
|
||||
|
||||
# TODO(akaratza): Remove
|
||||
# _get_padding_mask and patch_gating_output after
|
||||
# https://github.com/ROCm/aiter/pull/4530 is released and consumed by vLLM.
|
||||
def _get_padding_mask() -> torch.Tensor | None:
|
||||
"""
|
||||
Retrieves a boolean mask with non-padding (0) and padding (1) tokens.
|
||||
@@ -36,7 +39,13 @@ def _get_padding_mask() -> torch.Tensor | None:
|
||||
slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1)
|
||||
|
||||
in gpu_model_runner.py.
|
||||
|
||||
Direct kernel callers do not have model-runner padding metadata, so no
|
||||
mask is needed when there is no forward context.
|
||||
"""
|
||||
if not is_forward_context_available():
|
||||
return None
|
||||
|
||||
forward_context = get_forward_context()
|
||||
|
||||
# model runner v2.
|
||||
|
||||
Reference in New Issue
Block a user