[ROCm][CI] Enable AITER FP8/unquantized cases in modular-kernel sweep + fix MoRI per-tensor FP8 dispatch (#57074)
Signed-off-by: Divakar Verma <divakar.verma@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
a615f53364
commit
95f4925c3a
@@ -1704,8 +1704,9 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --ignore=kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 pytest -v -s kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":amd: (MI300) Quantization Kernels" # TBD
|
||||
timeout_in_minutes: 55
|
||||
|
||||
@@ -15,6 +15,7 @@ from tests.kernels.quantization.nvfp4_utils import (
|
||||
dequantize_nvfp4_to_dtype,
|
||||
)
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
@@ -33,7 +34,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
@@ -165,31 +168,34 @@ class Config:
|
||||
|
||||
return vllm_config, env_dict
|
||||
|
||||
def fp8_quant_key_pair(self) -> tuple[QuantKey, QuantKey]:
|
||||
"""Derive the (weight_quant_key, activation_quant_key) pair an FP8
|
||||
quant config of this shape corresponds to (either OCP or FNUZ FP8,
|
||||
see ``current_platform.fp8_dtype()``)."""
|
||||
if self.quant_block_shape is not None:
|
||||
return kFp8Static128BlockSym, kFp8Dynamic128Sym
|
||||
if self.is_per_out_ch_quant:
|
||||
return (
|
||||
kFp8StaticChannelSym,
|
||||
kFp8DynamicTokenSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym,
|
||||
)
|
||||
return (
|
||||
kFp8StaticTensorSym,
|
||||
kFp8DynamicTensorSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym,
|
||||
)
|
||||
|
||||
def fe_supports_quant_scheme(self) -> bool:
|
||||
"""Check if the fused experts class supports this quant config.
|
||||
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
|
||||
if self.quant_config is None or self.quant_dtype is None:
|
||||
return True
|
||||
if self.quant_dtype != torch.float8_e4m3fn:
|
||||
if not is_fp8(self.quant_dtype):
|
||||
return True
|
||||
# Derive QuantKeys from test config
|
||||
if self.quant_block_shape is not None:
|
||||
w_key = kFp8Static128BlockSym
|
||||
a_key = kFp8Dynamic128Sym
|
||||
elif self.is_per_out_ch_quant:
|
||||
w_key = kFp8StaticChannelSym
|
||||
a_key = (
|
||||
kFp8DynamicTokenSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym
|
||||
)
|
||||
else:
|
||||
w_key = kFp8StaticTensorSym
|
||||
a_key = (
|
||||
kFp8DynamicTensorSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym
|
||||
)
|
||||
w_key, a_key = self.fp8_quant_key_pair()
|
||||
fe_cls = self.fused_experts_type
|
||||
if hasattr(fe_cls, "_supports_quant_scheme"):
|
||||
try:
|
||||
@@ -199,10 +205,7 @@ class Config:
|
||||
return True
|
||||
|
||||
def is_fp8_block_quantized(self):
|
||||
return (
|
||||
self.quant_dtype == torch.float8_e4m3fn
|
||||
and self.quant_block_shape is not None
|
||||
)
|
||||
return is_fp8(self.quant_dtype) and self.quant_block_shape is not None
|
||||
|
||||
def is_batched_prepare_finalize(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
@@ -334,6 +337,11 @@ class Config:
|
||||
return False, "Needs Aiter, but Aiter not available."
|
||||
if self.needs_mori() and not has_mori(): # noqa: SIM103
|
||||
return False, "Needs MoRI, but MoRI not available."
|
||||
if self.needs_mori() and not rocm_aiter_ops.is_fused_moe_enabled():
|
||||
return False, (
|
||||
"Mori requires AITER's fused-moe backend to be enabled "
|
||||
"(VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1)."
|
||||
)
|
||||
|
||||
try:
|
||||
if not self.fused_experts_type._supports_current_device():
|
||||
@@ -370,7 +378,7 @@ class WeightTensors:
|
||||
def is_quantized(self) -> bool:
|
||||
# or w1_scale is not None?
|
||||
return (
|
||||
self.w1.dtype == torch.float8_e4m3fn
|
||||
is_fp8(self.w1.dtype)
|
||||
or self.w1.dtype == torch.uint8
|
||||
or self.w1.dtype == torch.int8
|
||||
)
|
||||
|
||||
@@ -86,14 +86,14 @@ MK_FUSED_EXPERT_TYPES: list[mk.FusedMoEExpertsModular] = []
|
||||
standard_format = mk.FusedMoEActivationFormat.Standard
|
||||
batched_format = mk.FusedMoEActivationFormat.BatchedExperts
|
||||
common_float_types: list[torch.dtype | str] = [
|
||||
torch.float8_e4m3fn,
|
||||
current_platform.fp8_dtype(),
|
||||
torch.bfloat16,
|
||||
torch.float16,
|
||||
torch.float32,
|
||||
]
|
||||
common_float_and_int_types = common_float_types + [torch.int8]
|
||||
nvfp4_types = ["nvfp4"]
|
||||
fp8_types = [torch.float8_e4m3fn]
|
||||
fp8_types = [current_platform.fp8_dtype()]
|
||||
|
||||
|
||||
def register_prepare_and_finalize(
|
||||
@@ -324,7 +324,9 @@ if has_aiter():
|
||||
register_experts(
|
||||
AiterExperts,
|
||||
standard_format,
|
||||
fp8_types,
|
||||
# AiterExperts also supports the fully-unquantized (None, None)
|
||||
# scheme (see SUPPORTED_W_A in rocm_aiter_moe.py), not just fp8.
|
||||
common_float_types,
|
||||
blocked_quantization_support=True,
|
||||
needs_aiter=True,
|
||||
)
|
||||
@@ -397,35 +399,35 @@ MK_QUANT_CONFIGS: list[TestMoEQuantConfig | None] = [
|
||||
None,
|
||||
# per-channel / per-column weights and per-tensor activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=torch.float8_e4m3fn,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=True,
|
||||
per_act_token_quant=False,
|
||||
block_shape=None,
|
||||
),
|
||||
# per-channel / per-column weights and per-token activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=torch.float8_e4m3fn,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=True,
|
||||
per_act_token_quant=True,
|
||||
block_shape=None,
|
||||
),
|
||||
# per-tensor weights and per-tensor activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=torch.float8_e4m3fn,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=False,
|
||||
per_act_token_quant=False,
|
||||
block_shape=None,
|
||||
),
|
||||
# per-tensor weights and per-token activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=torch.float8_e4m3fn,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=False,
|
||||
per_act_token_quant=True,
|
||||
block_shape=None,
|
||||
),
|
||||
# block-quantized weights and 128 block per-token activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=torch.float8_e4m3fn,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=False,
|
||||
per_act_token_quant=False,
|
||||
block_shape=[128, 128],
|
||||
|
||||
@@ -37,6 +37,7 @@ from .modular_kernel_tools.parallel_utils import (
|
||||
ProcessGroupInfo,
|
||||
parallel_launch_with_config,
|
||||
)
|
||||
from .utils import check_accuracy
|
||||
|
||||
has_any_multi_gpu_package = (
|
||||
has_deep_ep() or has_deep_gemm() or has_flashinfer_cutlass_fused_moe()
|
||||
@@ -47,12 +48,6 @@ meets_multi_gpu_requirements = pytest.mark.skipif(
|
||||
reason="Requires deep_ep or deep_gemm or flashinfer packages",
|
||||
)
|
||||
|
||||
if current_platform.is_fp8_fnuz():
|
||||
pytest.skip(
|
||||
"Tests in this file require float8_e4m3fn and platform does not support",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def format_result(verbose, msg, ex=None):
|
||||
if ex is not None:
|
||||
@@ -70,6 +65,28 @@ def format_result(verbose, msg, ex=None):
|
||||
print(".", end="")
|
||||
|
||||
|
||||
def assert_aiter_quant_scheme_case(config: Config) -> None:
|
||||
"""Make the AITER (weight_quant_key, activation_quant_key) pair this
|
||||
config exercises explicit, instead of AiterExperts being reached only
|
||||
indirectly through the general quant-config sweep.
|
||||
See https://github.com/vllm-project/vllm/issues/54966."""
|
||||
fe_cls = config.fused_experts_type
|
||||
if getattr(fe_cls, "__name__", "") != "AiterExperts":
|
||||
return
|
||||
|
||||
if config.quant_config is None:
|
||||
w_key, a_key = None, None
|
||||
else:
|
||||
w_key, a_key = config.fp8_quant_key_pair()
|
||||
|
||||
assert fe_cls._supports_quant_scheme(w_key, a_key), (
|
||||
f"AITER case (weight_key={w_key}, activation_key={a_key}) reached "
|
||||
"the modular-kernel harness, but AiterExperts._supports_quant_scheme "
|
||||
"does not declare it supported."
|
||||
)
|
||||
print(f"[AITER case] weight_key={w_key}, activation_key={a_key}")
|
||||
|
||||
|
||||
def rank_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -125,6 +142,25 @@ def rank_worker(
|
||||
count -= 1
|
||||
continue
|
||||
|
||||
# Skip unsupported: AITER x DeepEP-HT/Mori dispatch crashes
|
||||
# with an illegal memory access at world_size>1 on gfx942.
|
||||
# https://github.com/vllm-project/vllm/issues/57029
|
||||
if (
|
||||
config.world_size > 1
|
||||
and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts"
|
||||
and getattr(config.prepare_finalize_type, "__name__", "")
|
||||
in ("DeepEPHTPrepareAndFinalize", "MoriPrepareAndFinalize")
|
||||
):
|
||||
print(
|
||||
f"Skipping[{pgi.rank}]: m={m}, topk={topk}"
|
||||
" (AITER x DeepEP-HT/Mori illegal memory access,"
|
||||
" https://github.com/vllm-project/vllm/issues/57029)"
|
||||
)
|
||||
count -= 1
|
||||
continue
|
||||
|
||||
assert_aiter_quant_scheme_case(config)
|
||||
|
||||
# modular kernel out
|
||||
mk_out = run_modular_kernel(pgi, vllm_config, config, weights, rank_tensors)
|
||||
|
||||
@@ -152,32 +188,7 @@ def rank_worker(
|
||||
and config.quant_config is not None
|
||||
)
|
||||
if is_aiter_fp8:
|
||||
diff = (ref_out - mk_out).abs()
|
||||
n_total = diff.numel()
|
||||
max_diff = diff.max().item()
|
||||
n_exceed = int((diff > atol).sum().item())
|
||||
pct_exceed = n_exceed / n_total * 100
|
||||
# FP8 hw matmul vs f32 reference: up to ~4% of
|
||||
# elements may exceed base tolerance, but max
|
||||
# error should stay within 3x base tolerance.
|
||||
max_pct_allowed = 5.0
|
||||
relaxed_atol = atol * 4
|
||||
print(
|
||||
f"[AITER FP8 precision] "
|
||||
f"max_diff={max_diff:.6f}, "
|
||||
f"exceed_atol={n_exceed}/{n_total} "
|
||||
f"({pct_exceed:.4f}%), "
|
||||
f"max_pct_allowed={max_pct_allowed}%, "
|
||||
f"relaxed_limit={relaxed_atol}"
|
||||
)
|
||||
assert pct_exceed <= max_pct_allowed, (
|
||||
f"AITER FP8: {pct_exceed:.2f}% elements exceed "
|
||||
f"atol={atol} (max allowed {max_pct_allowed}%)"
|
||||
)
|
||||
assert max_diff <= relaxed_atol, (
|
||||
f"AITER FP8: max_diff={max_diff:.6f} exceeds "
|
||||
f"relaxed limit {relaxed_atol}"
|
||||
)
|
||||
check_accuracy(ref_out, mk_out, atol=atol, rtol=rtol, percent=0.9)
|
||||
else:
|
||||
torch.testing.assert_close(ref_out, mk_out, atol=atol, rtol=rtol)
|
||||
format_result(verbose, config.describe())
|
||||
|
||||
@@ -284,13 +284,20 @@ def maybe_make_prepare_finalize(
|
||||
# Note: We may want to use FP8 dispatch just to reduce
|
||||
# data movement.
|
||||
use_fp8_dispatch = (
|
||||
quant_config.is_per_act_token or quant_config.is_block_quantized
|
||||
quant_config.is_per_act_token
|
||||
or quant_config.is_block_quantized
|
||||
or quant_config.is_per_tensor
|
||||
)
|
||||
if use_fp8_dispatch:
|
||||
# For PTPC (per token per channel) quant, scale dim is 1
|
||||
# For 1x128 quant, scale dim is hidden_dim // 128
|
||||
# For PTPC (per token per channel) or per-tensor quant,
|
||||
# scale dim is 1. For 1x128 quant, scale dim is
|
||||
# hidden_dim // 128
|
||||
quant_dtype = quant_config.quant_dtype
|
||||
scale_dim = 1 if quant_config.is_per_act_token else moe.hidden_dim // 128
|
||||
scale_dim = (
|
||||
1
|
||||
if (quant_config.is_per_act_token or quant_config.is_per_tensor)
|
||||
else moe.hidden_dim // 128
|
||||
)
|
||||
else:
|
||||
# Unquantized dispatch (e.g. AITER with defer_input_quant):
|
||||
# dispatch raw BF16/FP16 data, no scales needed.
|
||||
|
||||
@@ -82,6 +82,15 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
elif quant_config.is_per_act_token:
|
||||
quant_func = get_hip_quant(QuantType.per_Token)
|
||||
a1, scale = quant_func(a1, quant_dtype=current_platform.fp8_dtype())
|
||||
elif quant_config.is_per_tensor:
|
||||
quant_func = get_hip_quant(QuantType.per_Tensor)
|
||||
a1, scale = quant_func(
|
||||
a1,
|
||||
scale=quant_config.a1_scale,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
)
|
||||
# mori expects one scale slot per token; broadcast.
|
||||
scale = scale.expand(a1.shape[0], 1).contiguous()
|
||||
|
||||
(
|
||||
dispatch_a1,
|
||||
|
||||
Reference in New Issue
Block a user