[ROCm][Quark][7/N] Use MXFP4 linear kernel abstraction for emulation backend (#48949)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-07-31 10:07:15 -07:00
committed by GitHub
parent d87d2ca747
commit e67a2e0a56
17 changed files with 824 additions and 90 deletions
@@ -3,25 +3,28 @@
from collections.abc import Callable
from fractions import Fraction
from functools import partial
from typing import Any
import torch
import torch.nn.functional as F
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
dequant_mxfp4,
quant_dequant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
dequant_mxfp6,
quant_dequant_mxfp6,
from vllm.model_executor.kernels.linear import (
MxFp4LinearKernel,
MxFp6LinearKernel,
init_mxfp4_linear_kernel,
init_mxfp6_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
OCP_MX_BLOCK_SIZE,
OCP_MX_Scheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp6E2M3Dynamic,
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
@@ -35,8 +38,22 @@ from .quark_scheme import QuarkScheme
logger = init_logger(__name__)
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Static,
"mxfp6_e3m2": kMxfp6E3M2Static,
"mxfp6_e2m3": kMxfp6E2M3Static,
}
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Dynamic,
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
}
class QuarkOCP_MX(QuarkScheme):
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
def __init__(
self,
weight_quant_spec: dict[str, Any],
@@ -49,35 +66,26 @@ class QuarkOCP_MX(QuarkScheme):
self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp")
self.input_dtype: str | None = None
if input_quant_spec is not None:
input_quant = input_quant_spec["dtype"]
if input_quant == "fp8_e4m3":
self.input_dtype = "fp8"
else:
self.input_dtype = input_quant.replace("fp", "mxfp")
self.input_dtype = input_quant_spec["dtype"].replace("fp", "mxfp")
self.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(
self.input_dtype, self.weight_dtype
if self.input_dtype not in [None, *_ACTIVATION_QUANT_KEY_MAP]:
raise ValueError(
f"Unsupported input_dtype={self.input_dtype} for QuarkOCP_MX. "
f"Supported activation dtypes are {_ACTIVATION_QUANT_KEY_MAP.keys()}, "
"or None for weight-only quantization."
)
self.weight_quant_key = _WEIGHT_QUANT_KEY_MAP[self.weight_dtype]
self.activation_quant_key = (
_ACTIVATION_QUANT_KEY_MAP[self.input_dtype]
if self.input_dtype is not None
else None
)
if self.weight_dtype == "mxfp4":
self.packed_factor: int | Fraction = 2
self.dequant_func = dequant_mxfp4
else:
self.packed_factor = Fraction(numerator=8, denominator=6)
self.dequant_func = partial(
dequant_mxfp6, quant_dtype=self.weight_dtype.replace("mx", "")
)
if self.input_dtype is None:
self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = (
lambda x: x
) # no input Q/DQ for weight-only
elif self.input_dtype == "mxfp4":
self.quant_dequant_func = quant_dequant_mxfp4
else:
self.quant_dequant_func = partial(
quant_dequant_mxfp6, quant_dtype=self.input_dtype.replace("mx", "")
)
if input_quant_spec is None:
self.static_input_scales = False
@@ -90,16 +98,6 @@ class QuarkOCP_MX(QuarkScheme):
"implemented. Please open an issue."
)
# TODO: integrate (or test) mixed-precision kernel.
self.emulate = not current_platform.supports_mx() or (
self.input_dtype != "mxfp4" or self.weight_dtype != "mxfp4"
)
# TODO: Move emulation code path as a kernel, and always
# use init_mxfp4_linear_kernel.
if not self.emulate:
self.ocp_mx_linear = init_mxfp4_linear_kernel()
if not current_platform.supports_mx():
logger.warning_once(
"The current platform does not support native MXFP4/MXFP6 "
@@ -151,17 +149,10 @@ class QuarkOCP_MX(QuarkScheme):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
if self.emulate:
if self.dynamic_mxfp4_quant:
self.process_dynamic_mxfp4_weights_after_loading(layer)
else:
layer.weight_scale = torch.nn.Parameter(
layer.weight_scale.data, requires_grad=False
)
else:
if self.dynamic_mxfp4_quant:
self.process_dynamic_mxfp4_weights_after_loading(layer)
self.ocp_mx_linear.process_weights_after_loading(layer)
if self.dynamic_mxfp4_quant:
self.process_dynamic_mxfp4_weights_after_loading(layer)
self.ocp_mx_linear.process_weights_after_loading(layer)
def create_weights(
self,
@@ -218,14 +209,20 @@ class QuarkOCP_MX(QuarkScheme):
)
layer.register_parameter("weight_scale", weight_scale)
if self.weight_quant_key == kMxfp4Static:
self.ocp_mx_linear = init_mxfp4_linear_kernel(
activation_quant_key=self.activation_quant_key,
)
elif self.weight_quant_key in [kMxfp6E2M3Static, kMxfp6E3M2Static]:
self.ocp_mx_linear = init_mxfp6_linear_kernel(
weight_quant_key=self.weight_quant_key,
activation_quant_key=self.activation_quant_key,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if self.emulate:
dq_w = self.dequant_func(layer.weight, layer.weight_scale, x.dtype)
qdq_x = self.quant_dequant_func(x)
return F.linear(qdq_x, dq_w, bias)
return self.ocp_mx_linear.apply_weights(layer, x, bias)