[ROCm][Quark][7/N] Use MXFP4 linear kernel abstraction for emulation backend (#48949)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-07-31 10:07:15 -07:00
committed by GitHub
parent d87d2ca747
commit e67a2e0a56
17 changed files with 824 additions and 90 deletions
@@ -9,6 +9,9 @@ from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsScheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
@@ -35,7 +38,9 @@ class CompressedTensorsW4A4Mxfp4(CompressedTensorsScheme):
def __init__(self):
self.group_size = 32
self.kernel = init_mxfp4_linear_kernel()
self.kernel = init_mxfp4_linear_kernel(
activation_quant_key=kMxfp4Dynamic,
)
@classmethod
def get_min_capability(cls) -> int:
@@ -3,25 +3,28 @@
from collections.abc import Callable
from fractions import Fraction
from functools import partial
from typing import Any
import torch
import torch.nn.functional as F
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
dequant_mxfp4,
quant_dequant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
dequant_mxfp6,
quant_dequant_mxfp6,
from vllm.model_executor.kernels.linear import (
MxFp4LinearKernel,
MxFp6LinearKernel,
init_mxfp4_linear_kernel,
init_mxfp6_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
OCP_MX_BLOCK_SIZE,
OCP_MX_Scheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp6E2M3Dynamic,
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
@@ -35,8 +38,22 @@ from .quark_scheme import QuarkScheme
logger = init_logger(__name__)
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Static,
"mxfp6_e3m2": kMxfp6E3M2Static,
"mxfp6_e2m3": kMxfp6E2M3Static,
}
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Dynamic,
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
}
class QuarkOCP_MX(QuarkScheme):
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
def __init__(
self,
weight_quant_spec: dict[str, Any],
@@ -49,35 +66,26 @@ class QuarkOCP_MX(QuarkScheme):
self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp")
self.input_dtype: str | None = None
if input_quant_spec is not None:
input_quant = input_quant_spec["dtype"]
if input_quant == "fp8_e4m3":
self.input_dtype = "fp8"
else:
self.input_dtype = input_quant.replace("fp", "mxfp")
self.input_dtype = input_quant_spec["dtype"].replace("fp", "mxfp")
self.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(
self.input_dtype, self.weight_dtype
if self.input_dtype not in [None, *_ACTIVATION_QUANT_KEY_MAP]:
raise ValueError(
f"Unsupported input_dtype={self.input_dtype} for QuarkOCP_MX. "
f"Supported activation dtypes are {_ACTIVATION_QUANT_KEY_MAP.keys()}, "
"or None for weight-only quantization."
)
self.weight_quant_key = _WEIGHT_QUANT_KEY_MAP[self.weight_dtype]
self.activation_quant_key = (
_ACTIVATION_QUANT_KEY_MAP[self.input_dtype]
if self.input_dtype is not None
else None
)
if self.weight_dtype == "mxfp4":
self.packed_factor: int | Fraction = 2
self.dequant_func = dequant_mxfp4
else:
self.packed_factor = Fraction(numerator=8, denominator=6)
self.dequant_func = partial(
dequant_mxfp6, quant_dtype=self.weight_dtype.replace("mx", "")
)
if self.input_dtype is None:
self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = (
lambda x: x
) # no input Q/DQ for weight-only
elif self.input_dtype == "mxfp4":
self.quant_dequant_func = quant_dequant_mxfp4
else:
self.quant_dequant_func = partial(
quant_dequant_mxfp6, quant_dtype=self.input_dtype.replace("mx", "")
)
if input_quant_spec is None:
self.static_input_scales = False
@@ -90,16 +98,6 @@ class QuarkOCP_MX(QuarkScheme):
"implemented. Please open an issue."
)
# TODO: integrate (or test) mixed-precision kernel.
self.emulate = not current_platform.supports_mx() or (
self.input_dtype != "mxfp4" or self.weight_dtype != "mxfp4"
)
# TODO: Move emulation code path as a kernel, and always
# use init_mxfp4_linear_kernel.
if not self.emulate:
self.ocp_mx_linear = init_mxfp4_linear_kernel()
if not current_platform.supports_mx():
logger.warning_once(
"The current platform does not support native MXFP4/MXFP6 "
@@ -151,17 +149,10 @@ class QuarkOCP_MX(QuarkScheme):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
if self.emulate:
if self.dynamic_mxfp4_quant:
self.process_dynamic_mxfp4_weights_after_loading(layer)
else:
layer.weight_scale = torch.nn.Parameter(
layer.weight_scale.data, requires_grad=False
)
else:
if self.dynamic_mxfp4_quant:
self.process_dynamic_mxfp4_weights_after_loading(layer)
self.ocp_mx_linear.process_weights_after_loading(layer)
if self.dynamic_mxfp4_quant:
self.process_dynamic_mxfp4_weights_after_loading(layer)
self.ocp_mx_linear.process_weights_after_loading(layer)
def create_weights(
self,
@@ -218,14 +209,20 @@ class QuarkOCP_MX(QuarkScheme):
)
layer.register_parameter("weight_scale", weight_scale)
if self.weight_quant_key == kMxfp4Static:
self.ocp_mx_linear = init_mxfp4_linear_kernel(
activation_quant_key=self.activation_quant_key,
)
elif self.weight_quant_key in [kMxfp6E2M3Static, kMxfp6E3M2Static]:
self.ocp_mx_linear = init_mxfp6_linear_kernel(
weight_quant_key=self.weight_quant_key,
activation_quant_key=self.activation_quant_key,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if self.emulate:
dq_w = self.dequant_func(layer.weight, layer.weight_scale, x.dtype)
qdq_x = self.quant_dequant_func(x)
return F.linear(qdq_x, dq_w, bias)
return self.ocp_mx_linear.apply_weights(layer, x, bias)
@@ -90,6 +90,7 @@ class ScaleDesc:
GroupShape.PER_CHANNEL: "per_channel",
}
group_shape = d.get(self.group_shape, str(self.group_shape))
return (
f"{fx.graph.dtype_abbrs[self.dtype]},"
f"{'static' if self.static else 'dynamic'},{group_shape}"
@@ -106,15 +107,24 @@ class QuantKey:
symmetric: symmetric if True, asymmetric if False
"""
dtype: torch.dtype
# TODO: QuantKey.dtype is assumed to be `torch.dtype` in matcher_utils.py,
# but #37990 introduced e.g. `kInt4Static` that uses a `ScalarType` dtype,
# same for kMxfp6 that does not have a native torch representation.
# Logical dtype and storage (torch) dtype should be separated (see #48949).
dtype: torch.dtype | ScalarType
scale: ScaleDesc
scale2: ScaleDesc | None = None
symmetric: bool = True
def __str__(self):
scale2_str = f"scale2({self.scale2})," if self.scale2 else ""
dtype_description = (
fx.graph.dtype_abbrs[self.dtype]
if isinstance(self.dtype, torch.dtype)
else self.dtype
)
return (
f"QuantKey({fx.graph.dtype_abbrs[self.dtype]},"
f"QuantKey({dtype_description},"
f"scale({self.scale}),{scale2_str}"
f"{'a' if not self.symmetric else ''}symmetric)"
)
@@ -172,6 +182,26 @@ kMxfp8Dynamic = QuantKey(FP8_DTYPE, scale=kMxfp8DynamicGroupScale, symmetric=Tru
kMxfp4StaticGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32))
kMxfp4Static = QuantKey(FP4_DTYPE, scale=kMxfp4StaticGroupScale, symmetric=True)
kMxfp6E3M2StaticGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32))
kMxfp6E3M2Static = QuantKey(
scalar_types.float6_e3m2f, scale=kMxfp6E3M2StaticGroupScale, symmetric=True
)
kMxfp6E3M2DynamicGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, False, GroupShape(1, 32))
kMxfp6E3M2Dynamic = QuantKey(
scalar_types.float6_e3m2f, scale=kMxfp6E3M2DynamicGroupScale, symmetric=True
)
kMxfp6E2M3StaticGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32))
kMxfp6E2M3Static = QuantKey(
scalar_types.float6_e2m3f, scale=kMxfp6E2M3StaticGroupScale, symmetric=True
)
kMxfp6E2M3DynamicGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, False, GroupShape(1, 32))
kMxfp6E2M3Dynamic = QuantKey(
scalar_types.float6_e2m3f, scale=kMxfp6E2M3DynamicGroupScale, symmetric=True
)
# TODO: convert this to use SCALAR_TYPE. This is not right.
kInt4StaticGroupScale = ScaleDesc(torch.float16, True, GroupShape(1, -1))
kInt4Static = QuantKey(INT4_DTYPE, scale=kInt4StaticGroupScale, symmetric=True)