[ROCm][CI] Enable AITER FP8/unquantized cases in modular-kernel sweep + fix MoRI per-tensor FP8 dispatch (#57074)

Signed-off-by: Divakar Verma <divakar.verma@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Divakar Verma
2026-09-16 19:53:06 -05:00
committed by GitHub
co-authored by Andreas Karatzas
parent a615f53364
commit 95f4925c3a
6 changed files with 107 additions and 69 deletions
@@ -15,6 +15,7 @@ from tests.kernels.quantization.nvfp4_utils import (
dequantize_nvfp4_to_dtype,
)
from tests.kernels.utils import torch_experts
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import (
get_dp_group,
@@ -33,7 +34,9 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
@@ -165,31 +168,34 @@ class Config:
return vllm_config, env_dict
def fp8_quant_key_pair(self) -> tuple[QuantKey, QuantKey]:
"""Derive the (weight_quant_key, activation_quant_key) pair an FP8
quant config of this shape corresponds to (either OCP or FNUZ FP8,
see ``current_platform.fp8_dtype()``)."""
if self.quant_block_shape is not None:
return kFp8Static128BlockSym, kFp8Dynamic128Sym
if self.is_per_out_ch_quant:
return (
kFp8StaticChannelSym,
kFp8DynamicTokenSym
if self.is_per_act_token_quant
else kFp8StaticTensorSym,
)
return (
kFp8StaticTensorSym,
kFp8DynamicTensorSym
if self.is_per_act_token_quant
else kFp8StaticTensorSym,
)
def fe_supports_quant_scheme(self) -> bool:
"""Check if the fused experts class supports this quant config.
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
if self.quant_config is None or self.quant_dtype is None:
return True
if self.quant_dtype != torch.float8_e4m3fn:
if not is_fp8(self.quant_dtype):
return True
# Derive QuantKeys from test config
if self.quant_block_shape is not None:
w_key = kFp8Static128BlockSym
a_key = kFp8Dynamic128Sym
elif self.is_per_out_ch_quant:
w_key = kFp8StaticChannelSym
a_key = (
kFp8DynamicTokenSym
if self.is_per_act_token_quant
else kFp8StaticTensorSym
)
else:
w_key = kFp8StaticTensorSym
a_key = (
kFp8DynamicTensorSym
if self.is_per_act_token_quant
else kFp8StaticTensorSym
)
w_key, a_key = self.fp8_quant_key_pair()
fe_cls = self.fused_experts_type
if hasattr(fe_cls, "_supports_quant_scheme"):
try:
@@ -199,10 +205,7 @@ class Config:
return True
def is_fp8_block_quantized(self):
return (
self.quant_dtype == torch.float8_e4m3fn
and self.quant_block_shape is not None
)
return is_fp8(self.quant_dtype) and self.quant_block_shape is not None
def is_batched_prepare_finalize(self):
info = prepare_finalize_info(self.prepare_finalize_type)
@@ -334,6 +337,11 @@ class Config:
return False, "Needs Aiter, but Aiter not available."
if self.needs_mori() and not has_mori(): # noqa: SIM103
return False, "Needs MoRI, but MoRI not available."
if self.needs_mori() and not rocm_aiter_ops.is_fused_moe_enabled():
return False, (
"Mori requires AITER's fused-moe backend to be enabled "
"(VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1)."
)
try:
if not self.fused_experts_type._supports_current_device():
@@ -370,7 +378,7 @@ class WeightTensors:
def is_quantized(self) -> bool:
# or w1_scale is not None?
return (
self.w1.dtype == torch.float8_e4m3fn
is_fp8(self.w1.dtype)
or self.w1.dtype == torch.uint8
or self.w1.dtype == torch.int8
)
@@ -86,14 +86,14 @@ MK_FUSED_EXPERT_TYPES: list[mk.FusedMoEExpertsModular] = []
standard_format = mk.FusedMoEActivationFormat.Standard
batched_format = mk.FusedMoEActivationFormat.BatchedExperts
common_float_types: list[torch.dtype | str] = [
torch.float8_e4m3fn,
current_platform.fp8_dtype(),
torch.bfloat16,
torch.float16,
torch.float32,
]
common_float_and_int_types = common_float_types + [torch.int8]
nvfp4_types = ["nvfp4"]
fp8_types = [torch.float8_e4m3fn]
fp8_types = [current_platform.fp8_dtype()]
def register_prepare_and_finalize(
@@ -324,7 +324,9 @@ if has_aiter():
register_experts(
AiterExperts,
standard_format,
fp8_types,
# AiterExperts also supports the fully-unquantized (None, None)
# scheme (see SUPPORTED_W_A in rocm_aiter_moe.py), not just fp8.
common_float_types,
blocked_quantization_support=True,
needs_aiter=True,
)
@@ -397,35 +399,35 @@ MK_QUANT_CONFIGS: list[TestMoEQuantConfig | None] = [
None,
# per-channel / per-column weights and per-tensor activations
TestMoEQuantConfig(
quant_dtype=torch.float8_e4m3fn,
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=True,
per_act_token_quant=False,
block_shape=None,
),
# per-channel / per-column weights and per-token activations
TestMoEQuantConfig(
quant_dtype=torch.float8_e4m3fn,
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=True,
per_act_token_quant=True,
block_shape=None,
),
# per-tensor weights and per-tensor activations
TestMoEQuantConfig(
quant_dtype=torch.float8_e4m3fn,
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=False,
per_act_token_quant=False,
block_shape=None,
),
# per-tensor weights and per-token activations
TestMoEQuantConfig(
quant_dtype=torch.float8_e4m3fn,
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=False,
per_act_token_quant=True,
block_shape=None,
),
# block-quantized weights and 128 block per-token activations
TestMoEQuantConfig(
quant_dtype=torch.float8_e4m3fn,
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=False,
per_act_token_quant=False,
block_shape=[128, 128],