From 0b131b16c93308df90d013c5ac24c6fa25bd863b Mon Sep 17 00:00:00 2001 From: xaguilar-amd Date: Wed, 17 Jun 2026 16:05:34 +0200 Subject: [PATCH] [ROCm][AITER][Quark] Tag per-channel FP8 weights as PER_CHANNEL so AITER pre-shuffled GEMM is selected (#44626) Signed-off-by: Xavier Aguilar --- .../quantization/quark/schemes/quark_w8a8_fp8.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py index 6d94e26f96..280159700e 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py @@ -16,8 +16,8 @@ from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, kFp8DynamicTokenSym, + kFp8StaticChannelSym, kFp8StaticTensorSym, - kFp8StaticTokenSym, ) from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( normalize_e4m3fn_to_e4m3fnuz, @@ -49,13 +49,17 @@ class QuarkW8A8Fp8(QuarkScheme): per_token_activation = ( not self.is_static_input_scheme and self.input_qscheme == "per_channel" ) - per_token_weight = self.weight_qscheme == "per_channel" + per_channel_weight = self.weight_qscheme == "per_channel" self.activation_quant_key = ( kFp8DynamicTokenSym if per_token_activation else kFp8StaticTensorSym ) + # A per-output-channel weight scale is one fp32 value per weight row + # (length N). Tag it as ``GroupShape.PER_CHANNEL`` to match the + # canonical compressed-tensors CHANNEL strategy, so kernel selection + # (e.g. AITER's pre-shuffled FP8 GEMM) treats it uniformly. self.weight_quant_key = ( - kFp8StaticTokenSym if per_token_weight else kFp8StaticTensorSym + kFp8StaticChannelSym if per_channel_weight else kFp8StaticTensorSym ) self.out_dtype = torch.get_default_dtype() self.input_dtype = get_current_vllm_config().model_config.dtype