diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index eadfa36e04..4b72256a3f 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1704,8 +1704,9 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --ignore=kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 pytest -v -s kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI300) Quantization Kernels" # TBD timeout_in_minutes: 55 diff --git a/tests/kernels/moe/modular_kernel_tools/common.py b/tests/kernels/moe/modular_kernel_tools/common.py index 8adfa6588c..3ec522c8ec 100644 --- a/tests/kernels/moe/modular_kernel_tools/common.py +++ b/tests/kernels/moe/modular_kernel_tools/common.py @@ -15,6 +15,7 @@ from tests.kernels.quantization.nvfp4_utils import ( dequantize_nvfp4_to_dtype, ) from tests.kernels.utils import torch_experts +from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig from vllm.distributed import ( get_dp_group, @@ -33,7 +34,9 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, RoutingMethodType, ) +from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8 from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, kFp8Dynamic128Sym, kFp8DynamicTensorSym, kFp8DynamicTokenSym, @@ -165,31 +168,34 @@ class Config: return vllm_config, env_dict + def fp8_quant_key_pair(self) -> tuple[QuantKey, QuantKey]: + """Derive the (weight_quant_key, activation_quant_key) pair an FP8 + quant config of this shape corresponds to (either OCP or FNUZ FP8, + see ``current_platform.fp8_dtype()``).""" + if self.quant_block_shape is not None: + return kFp8Static128BlockSym, kFp8Dynamic128Sym + if self.is_per_out_ch_quant: + return ( + kFp8StaticChannelSym, + kFp8DynamicTokenSym + if self.is_per_act_token_quant + else kFp8StaticTensorSym, + ) + return ( + kFp8StaticTensorSym, + kFp8DynamicTensorSym + if self.is_per_act_token_quant + else kFp8StaticTensorSym, + ) + def fe_supports_quant_scheme(self) -> bool: """Check if the fused experts class supports this quant config. See https://github.com/ROCm/aiter/issues/2419 for AITER gaps.""" if self.quant_config is None or self.quant_dtype is None: return True - if self.quant_dtype != torch.float8_e4m3fn: + if not is_fp8(self.quant_dtype): return True - # Derive QuantKeys from test config - if self.quant_block_shape is not None: - w_key = kFp8Static128BlockSym - a_key = kFp8Dynamic128Sym - elif self.is_per_out_ch_quant: - w_key = kFp8StaticChannelSym - a_key = ( - kFp8DynamicTokenSym - if self.is_per_act_token_quant - else kFp8StaticTensorSym - ) - else: - w_key = kFp8StaticTensorSym - a_key = ( - kFp8DynamicTensorSym - if self.is_per_act_token_quant - else kFp8StaticTensorSym - ) + w_key, a_key = self.fp8_quant_key_pair() fe_cls = self.fused_experts_type if hasattr(fe_cls, "_supports_quant_scheme"): try: @@ -199,10 +205,7 @@ class Config: return True def is_fp8_block_quantized(self): - return ( - self.quant_dtype == torch.float8_e4m3fn - and self.quant_block_shape is not None - ) + return is_fp8(self.quant_dtype) and self.quant_block_shape is not None def is_batched_prepare_finalize(self): info = prepare_finalize_info(self.prepare_finalize_type) @@ -334,6 +337,11 @@ class Config: return False, "Needs Aiter, but Aiter not available." if self.needs_mori() and not has_mori(): # noqa: SIM103 return False, "Needs MoRI, but MoRI not available." + if self.needs_mori() and not rocm_aiter_ops.is_fused_moe_enabled(): + return False, ( + "Mori requires AITER's fused-moe backend to be enabled " + "(VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1)." + ) try: if not self.fused_experts_type._supports_current_device(): @@ -370,7 +378,7 @@ class WeightTensors: def is_quantized(self) -> bool: # or w1_scale is not None? return ( - self.w1.dtype == torch.float8_e4m3fn + is_fp8(self.w1.dtype) or self.w1.dtype == torch.uint8 or self.w1.dtype == torch.int8 ) diff --git a/tests/kernels/moe/modular_kernel_tools/mk_objects.py b/tests/kernels/moe/modular_kernel_tools/mk_objects.py index 7f1924bac5..e0f37f5436 100644 --- a/tests/kernels/moe/modular_kernel_tools/mk_objects.py +++ b/tests/kernels/moe/modular_kernel_tools/mk_objects.py @@ -86,14 +86,14 @@ MK_FUSED_EXPERT_TYPES: list[mk.FusedMoEExpertsModular] = [] standard_format = mk.FusedMoEActivationFormat.Standard batched_format = mk.FusedMoEActivationFormat.BatchedExperts common_float_types: list[torch.dtype | str] = [ - torch.float8_e4m3fn, + current_platform.fp8_dtype(), torch.bfloat16, torch.float16, torch.float32, ] common_float_and_int_types = common_float_types + [torch.int8] nvfp4_types = ["nvfp4"] -fp8_types = [torch.float8_e4m3fn] +fp8_types = [current_platform.fp8_dtype()] def register_prepare_and_finalize( @@ -324,7 +324,9 @@ if has_aiter(): register_experts( AiterExperts, standard_format, - fp8_types, + # AiterExperts also supports the fully-unquantized (None, None) + # scheme (see SUPPORTED_W_A in rocm_aiter_moe.py), not just fp8. + common_float_types, blocked_quantization_support=True, needs_aiter=True, ) @@ -397,35 +399,35 @@ MK_QUANT_CONFIGS: list[TestMoEQuantConfig | None] = [ None, # per-channel / per-column weights and per-tensor activations TestMoEQuantConfig( - quant_dtype=torch.float8_e4m3fn, + quant_dtype=current_platform.fp8_dtype(), per_out_ch_quant=True, per_act_token_quant=False, block_shape=None, ), # per-channel / per-column weights and per-token activations TestMoEQuantConfig( - quant_dtype=torch.float8_e4m3fn, + quant_dtype=current_platform.fp8_dtype(), per_out_ch_quant=True, per_act_token_quant=True, block_shape=None, ), # per-tensor weights and per-tensor activations TestMoEQuantConfig( - quant_dtype=torch.float8_e4m3fn, + quant_dtype=current_platform.fp8_dtype(), per_out_ch_quant=False, per_act_token_quant=False, block_shape=None, ), # per-tensor weights and per-token activations TestMoEQuantConfig( - quant_dtype=torch.float8_e4m3fn, + quant_dtype=current_platform.fp8_dtype(), per_out_ch_quant=False, per_act_token_quant=True, block_shape=None, ), # block-quantized weights and 128 block per-token activations TestMoEQuantConfig( - quant_dtype=torch.float8_e4m3fn, + quant_dtype=current_platform.fp8_dtype(), per_out_ch_quant=False, per_act_token_quant=False, block_shape=[128, 128], diff --git a/tests/kernels/moe/test_modular_kernel_combinations.py b/tests/kernels/moe/test_modular_kernel_combinations.py index 0c0e1d61f9..fc1d8ac07b 100644 --- a/tests/kernels/moe/test_modular_kernel_combinations.py +++ b/tests/kernels/moe/test_modular_kernel_combinations.py @@ -37,6 +37,7 @@ from .modular_kernel_tools.parallel_utils import ( ProcessGroupInfo, parallel_launch_with_config, ) +from .utils import check_accuracy has_any_multi_gpu_package = ( has_deep_ep() or has_deep_gemm() or has_flashinfer_cutlass_fused_moe() @@ -47,12 +48,6 @@ meets_multi_gpu_requirements = pytest.mark.skipif( reason="Requires deep_ep or deep_gemm or flashinfer packages", ) -if current_platform.is_fp8_fnuz(): - pytest.skip( - "Tests in this file require float8_e4m3fn and platform does not support", - allow_module_level=True, - ) - def format_result(verbose, msg, ex=None): if ex is not None: @@ -70,6 +65,28 @@ def format_result(verbose, msg, ex=None): print(".", end="") +def assert_aiter_quant_scheme_case(config: Config) -> None: + """Make the AITER (weight_quant_key, activation_quant_key) pair this + config exercises explicit, instead of AiterExperts being reached only + indirectly through the general quant-config sweep. + See https://github.com/vllm-project/vllm/issues/54966.""" + fe_cls = config.fused_experts_type + if getattr(fe_cls, "__name__", "") != "AiterExperts": + return + + if config.quant_config is None: + w_key, a_key = None, None + else: + w_key, a_key = config.fp8_quant_key_pair() + + assert fe_cls._supports_quant_scheme(w_key, a_key), ( + f"AITER case (weight_key={w_key}, activation_key={a_key}) reached " + "the modular-kernel harness, but AiterExperts._supports_quant_scheme " + "does not declare it supported." + ) + print(f"[AITER case] weight_key={w_key}, activation_key={a_key}") + + def rank_worker( pgi: ProcessGroupInfo, vllm_config: VllmConfig, @@ -125,6 +142,25 @@ def rank_worker( count -= 1 continue + # Skip unsupported: AITER x DeepEP-HT/Mori dispatch crashes + # with an illegal memory access at world_size>1 on gfx942. + # https://github.com/vllm-project/vllm/issues/57029 + if ( + config.world_size > 1 + and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts" + and getattr(config.prepare_finalize_type, "__name__", "") + in ("DeepEPHTPrepareAndFinalize", "MoriPrepareAndFinalize") + ): + print( + f"Skipping[{pgi.rank}]: m={m}, topk={topk}" + " (AITER x DeepEP-HT/Mori illegal memory access," + " https://github.com/vllm-project/vllm/issues/57029)" + ) + count -= 1 + continue + + assert_aiter_quant_scheme_case(config) + # modular kernel out mk_out = run_modular_kernel(pgi, vllm_config, config, weights, rank_tensors) @@ -152,32 +188,7 @@ def rank_worker( and config.quant_config is not None ) if is_aiter_fp8: - diff = (ref_out - mk_out).abs() - n_total = diff.numel() - max_diff = diff.max().item() - n_exceed = int((diff > atol).sum().item()) - pct_exceed = n_exceed / n_total * 100 - # FP8 hw matmul vs f32 reference: up to ~4% of - # elements may exceed base tolerance, but max - # error should stay within 3x base tolerance. - max_pct_allowed = 5.0 - relaxed_atol = atol * 4 - print( - f"[AITER FP8 precision] " - f"max_diff={max_diff:.6f}, " - f"exceed_atol={n_exceed}/{n_total} " - f"({pct_exceed:.4f}%), " - f"max_pct_allowed={max_pct_allowed}%, " - f"relaxed_limit={relaxed_atol}" - ) - assert pct_exceed <= max_pct_allowed, ( - f"AITER FP8: {pct_exceed:.2f}% elements exceed " - f"atol={atol} (max allowed {max_pct_allowed}%)" - ) - assert max_diff <= relaxed_atol, ( - f"AITER FP8: max_diff={max_diff:.6f} exceeds " - f"relaxed limit {relaxed_atol}" - ) + check_accuracy(ref_out, mk_out, atol=atol, rtol=rtol, percent=0.9) else: torch.testing.assert_close(ref_out, mk_out, atol=atol, rtol=rtol) format_result(verbose, config.describe()) diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index 75ea91bfca..6984e34e56 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -284,13 +284,20 @@ def maybe_make_prepare_finalize( # Note: We may want to use FP8 dispatch just to reduce # data movement. use_fp8_dispatch = ( - quant_config.is_per_act_token or quant_config.is_block_quantized + quant_config.is_per_act_token + or quant_config.is_block_quantized + or quant_config.is_per_tensor ) if use_fp8_dispatch: - # For PTPC (per token per channel) quant, scale dim is 1 - # For 1x128 quant, scale dim is hidden_dim // 128 + # For PTPC (per token per channel) or per-tensor quant, + # scale dim is 1. For 1x128 quant, scale dim is + # hidden_dim // 128 quant_dtype = quant_config.quant_dtype - scale_dim = 1 if quant_config.is_per_act_token else moe.hidden_dim // 128 + scale_dim = ( + 1 + if (quant_config.is_per_act_token or quant_config.is_per_tensor) + else moe.hidden_dim // 128 + ) else: # Unquantized dispatch (e.g. AITER with defer_input_quant): # dispatch raw BF16/FP16 data, no scales needed. diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py index 35e15325b4..75b97c69aa 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py @@ -82,6 +82,15 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): elif quant_config.is_per_act_token: quant_func = get_hip_quant(QuantType.per_Token) a1, scale = quant_func(a1, quant_dtype=current_platform.fp8_dtype()) + elif quant_config.is_per_tensor: + quant_func = get_hip_quant(QuantType.per_Tensor) + a1, scale = quant_func( + a1, + scale=quant_config.a1_scale, + quant_dtype=current_platform.fp8_dtype(), + ) + # mori expects one scale slot per token; broadcast. + scale = scale.expand(a1.shape[0], 1).contiguous() ( dispatch_a1,