[ROCm][CI] Add missing quantization methods and fix online quant test failures (#39801)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -144,13 +144,21 @@ class RowWiseTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel):
|
||||
# For CUDA platform please validate if the torch._scaled_mm supports
|
||||
# rowwise scaled GEMM before using it
|
||||
|
||||
# torch._scaled_mm rowwise requires scale_a = (m, 1), scale_b = (1, n).
|
||||
# CompressedTensors stores weight_scale as (n, 1), so `.t()` yields (1, n).
|
||||
# ModelOpt FP8_PER_CHANNEL_PER_TOKEN stores it as 1-D (n,); reshape to
|
||||
# (1, n) so both paths satisfy the rowwise contract.
|
||||
scale_b = Bs.view(1, -1) if Bs.dim() == 1 else Bs.t()
|
||||
if As.dim() == 1:
|
||||
As = As.view(-1, 1)
|
||||
|
||||
# Fused GEMM_DQ Rowwise GEMM
|
||||
output = torch._scaled_mm(
|
||||
A,
|
||||
B,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=As,
|
||||
scale_b=Bs.t(),
|
||||
scale_b=scale_b,
|
||||
bias=bias,
|
||||
)
|
||||
|
||||
|
||||
@@ -1025,9 +1025,15 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
get_current_vllm_config().model_config.hf_config, "model_type", None
|
||||
)
|
||||
|
||||
# TODO(aiter): extend once rocm_aiter_fused_experts gains dispatch
|
||||
# for the other OCP MX schemes. Today its CK MoE kernel only has an
|
||||
# entry for `w_mxfp4` (w4a16); mixed schemes like `w_mxfp4_a_mxfp6_*`
|
||||
# fall through to QuantMethod.NO and raise "Unsupported kernel config
|
||||
# for moe heuristic dispatch".
|
||||
_AITER_NATIVE_OCP_MX_SCHEMES = ("w_mxfp4",)
|
||||
self.emulate = (
|
||||
not current_platform.supports_mx()
|
||||
or not self.ocp_mx_scheme.startswith("w_mxfp4")
|
||||
or self.ocp_mx_scheme not in _AITER_NATIVE_OCP_MX_SCHEMES
|
||||
) and (
|
||||
self.mxfp4_backend is Mxfp4MoeBackend.NONE or not self.use_rocm_aiter_moe
|
||||
)
|
||||
|
||||
@@ -376,11 +376,15 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
dq_w = self.dequant_func(layer.weight, layer.weight_scale, x.dtype)
|
||||
qdq_x = self.quant_dequant_func(x)
|
||||
return F.linear(qdq_x, dq_w, bias)
|
||||
else:
|
||||
return torch.ops.vllm.gemm_with_dynamic_quant(
|
||||
x,
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
self.rocm_use_aiter_fp4_asm_gemm,
|
||||
self.out_dtype,
|
||||
)
|
||||
y = torch.ops.vllm.gemm_with_dynamic_quant(
|
||||
x,
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
self.rocm_use_aiter_fp4_asm_gemm,
|
||||
self.out_dtype,
|
||||
)
|
||||
# gemm_with_dynamic_quant has no bias argument; add it here so the
|
||||
# native path matches F.linear (e.g. qkv_proj with qkv_bias=True).
|
||||
if bias is not None:
|
||||
y = y + bias
|
||||
return y
|
||||
|
||||
@@ -65,7 +65,7 @@ class BaseModelLoader(ABC):
|
||||
|
||||
# Log peak GPU memory after loading weights. This is needed
|
||||
# to have test coverage on peak memory for online quantization.
|
||||
if current_platform.is_cuda():
|
||||
if current_platform.is_cuda_alike():
|
||||
peak_memory = torch.accelerator.max_memory_allocated()
|
||||
logger.debug_once(
|
||||
"Peak GPU memory after loading weights: %s GiB",
|
||||
|
||||
Reference in New Issue
Block a user