[ROCm][CI] Enable AITER FP8/unquantized cases in modular-kernel sweep + fix MoRI per-tensor FP8 dispatch (#57074)
Signed-off-by: Divakar Verma <divakar.verma@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
a615f53364
commit
95f4925c3a
@@ -15,6 +15,7 @@ from tests.kernels.quantization.nvfp4_utils import (
|
||||
dequantize_nvfp4_to_dtype,
|
||||
)
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
@@ -33,7 +34,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
@@ -165,31 +168,34 @@ class Config:
|
||||
|
||||
return vllm_config, env_dict
|
||||
|
||||
def fp8_quant_key_pair(self) -> tuple[QuantKey, QuantKey]:
|
||||
"""Derive the (weight_quant_key, activation_quant_key) pair an FP8
|
||||
quant config of this shape corresponds to (either OCP or FNUZ FP8,
|
||||
see ``current_platform.fp8_dtype()``)."""
|
||||
if self.quant_block_shape is not None:
|
||||
return kFp8Static128BlockSym, kFp8Dynamic128Sym
|
||||
if self.is_per_out_ch_quant:
|
||||
return (
|
||||
kFp8StaticChannelSym,
|
||||
kFp8DynamicTokenSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym,
|
||||
)
|
||||
return (
|
||||
kFp8StaticTensorSym,
|
||||
kFp8DynamicTensorSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym,
|
||||
)
|
||||
|
||||
def fe_supports_quant_scheme(self) -> bool:
|
||||
"""Check if the fused experts class supports this quant config.
|
||||
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
|
||||
if self.quant_config is None or self.quant_dtype is None:
|
||||
return True
|
||||
if self.quant_dtype != torch.float8_e4m3fn:
|
||||
if not is_fp8(self.quant_dtype):
|
||||
return True
|
||||
# Derive QuantKeys from test config
|
||||
if self.quant_block_shape is not None:
|
||||
w_key = kFp8Static128BlockSym
|
||||
a_key = kFp8Dynamic128Sym
|
||||
elif self.is_per_out_ch_quant:
|
||||
w_key = kFp8StaticChannelSym
|
||||
a_key = (
|
||||
kFp8DynamicTokenSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym
|
||||
)
|
||||
else:
|
||||
w_key = kFp8StaticTensorSym
|
||||
a_key = (
|
||||
kFp8DynamicTensorSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym
|
||||
)
|
||||
w_key, a_key = self.fp8_quant_key_pair()
|
||||
fe_cls = self.fused_experts_type
|
||||
if hasattr(fe_cls, "_supports_quant_scheme"):
|
||||
try:
|
||||
@@ -199,10 +205,7 @@ class Config:
|
||||
return True
|
||||
|
||||
def is_fp8_block_quantized(self):
|
||||
return (
|
||||
self.quant_dtype == torch.float8_e4m3fn
|
||||
and self.quant_block_shape is not None
|
||||
)
|
||||
return is_fp8(self.quant_dtype) and self.quant_block_shape is not None
|
||||
|
||||
def is_batched_prepare_finalize(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
@@ -334,6 +337,11 @@ class Config:
|
||||
return False, "Needs Aiter, but Aiter not available."
|
||||
if self.needs_mori() and not has_mori(): # noqa: SIM103
|
||||
return False, "Needs MoRI, but MoRI not available."
|
||||
if self.needs_mori() and not rocm_aiter_ops.is_fused_moe_enabled():
|
||||
return False, (
|
||||
"Mori requires AITER's fused-moe backend to be enabled "
|
||||
"(VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1)."
|
||||
)
|
||||
|
||||
try:
|
||||
if not self.fused_experts_type._supports_current_device():
|
||||
@@ -370,7 +378,7 @@ class WeightTensors:
|
||||
def is_quantized(self) -> bool:
|
||||
# or w1_scale is not None?
|
||||
return (
|
||||
self.w1.dtype == torch.float8_e4m3fn
|
||||
is_fp8(self.w1.dtype)
|
||||
or self.w1.dtype == torch.uint8
|
||||
or self.w1.dtype == torch.int8
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user