[ROCm][Quark][7/N] Use MXFP4 linear kernel abstraction for emulation backend (#48949)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
+6
-1
@@ -9,6 +9,9 @@ from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
@@ -35,7 +38,9 @@ class CompressedTensorsW4A4Mxfp4(CompressedTensorsScheme):
|
||||
|
||||
def __init__(self):
|
||||
self.group_size = 32
|
||||
self.kernel = init_mxfp4_linear_kernel()
|
||||
self.kernel = init_mxfp4_linear_kernel(
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
|
||||
@@ -3,25 +3,28 @@
|
||||
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
dequant_mxfp4,
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
|
||||
dequant_mxfp6,
|
||||
quant_dequant_mxfp6,
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MxFp4LinearKernel,
|
||||
MxFp6LinearKernel,
|
||||
init_mxfp4_linear_kernel,
|
||||
init_mxfp6_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
OCP_MX_BLOCK_SIZE,
|
||||
OCP_MX_Scheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
@@ -35,8 +38,22 @@ from .quark_scheme import QuarkScheme
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Static,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Static,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Static,
|
||||
}
|
||||
|
||||
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Dynamic,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
|
||||
}
|
||||
|
||||
|
||||
class QuarkOCP_MX(QuarkScheme):
|
||||
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_spec: dict[str, Any],
|
||||
@@ -49,35 +66,26 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp")
|
||||
self.input_dtype: str | None = None
|
||||
if input_quant_spec is not None:
|
||||
input_quant = input_quant_spec["dtype"]
|
||||
if input_quant == "fp8_e4m3":
|
||||
self.input_dtype = "fp8"
|
||||
else:
|
||||
self.input_dtype = input_quant.replace("fp", "mxfp")
|
||||
self.input_dtype = input_quant_spec["dtype"].replace("fp", "mxfp")
|
||||
|
||||
self.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(
|
||||
self.input_dtype, self.weight_dtype
|
||||
if self.input_dtype not in [None, *_ACTIVATION_QUANT_KEY_MAP]:
|
||||
raise ValueError(
|
||||
f"Unsupported input_dtype={self.input_dtype} for QuarkOCP_MX. "
|
||||
f"Supported activation dtypes are {_ACTIVATION_QUANT_KEY_MAP.keys()}, "
|
||||
"or None for weight-only quantization."
|
||||
)
|
||||
|
||||
self.weight_quant_key = _WEIGHT_QUANT_KEY_MAP[self.weight_dtype]
|
||||
self.activation_quant_key = (
|
||||
_ACTIVATION_QUANT_KEY_MAP[self.input_dtype]
|
||||
if self.input_dtype is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if self.weight_dtype == "mxfp4":
|
||||
self.packed_factor: int | Fraction = 2
|
||||
self.dequant_func = dequant_mxfp4
|
||||
else:
|
||||
self.packed_factor = Fraction(numerator=8, denominator=6)
|
||||
self.dequant_func = partial(
|
||||
dequant_mxfp6, quant_dtype=self.weight_dtype.replace("mx", "")
|
||||
)
|
||||
|
||||
if self.input_dtype is None:
|
||||
self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = (
|
||||
lambda x: x
|
||||
) # no input Q/DQ for weight-only
|
||||
elif self.input_dtype == "mxfp4":
|
||||
self.quant_dequant_func = quant_dequant_mxfp4
|
||||
else:
|
||||
self.quant_dequant_func = partial(
|
||||
quant_dequant_mxfp6, quant_dtype=self.input_dtype.replace("mx", "")
|
||||
)
|
||||
|
||||
if input_quant_spec is None:
|
||||
self.static_input_scales = False
|
||||
@@ -90,16 +98,6 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
"implemented. Please open an issue."
|
||||
)
|
||||
|
||||
# TODO: integrate (or test) mixed-precision kernel.
|
||||
self.emulate = not current_platform.supports_mx() or (
|
||||
self.input_dtype != "mxfp4" or self.weight_dtype != "mxfp4"
|
||||
)
|
||||
|
||||
# TODO: Move emulation code path as a kernel, and always
|
||||
# use init_mxfp4_linear_kernel.
|
||||
if not self.emulate:
|
||||
self.ocp_mx_linear = init_mxfp4_linear_kernel()
|
||||
|
||||
if not current_platform.supports_mx():
|
||||
logger.warning_once(
|
||||
"The current platform does not support native MXFP4/MXFP6 "
|
||||
@@ -151,17 +149,10 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
|
||||
|
||||
if self.emulate:
|
||||
if self.dynamic_mxfp4_quant:
|
||||
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
||||
else:
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
layer.weight_scale.data, requires_grad=False
|
||||
)
|
||||
else:
|
||||
if self.dynamic_mxfp4_quant:
|
||||
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
||||
self.ocp_mx_linear.process_weights_after_loading(layer)
|
||||
if self.dynamic_mxfp4_quant:
|
||||
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
||||
|
||||
self.ocp_mx_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -218,14 +209,20 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
if self.weight_quant_key == kMxfp4Static:
|
||||
self.ocp_mx_linear = init_mxfp4_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
)
|
||||
elif self.weight_quant_key in [kMxfp6E2M3Static, kMxfp6E3M2Static]:
|
||||
self.ocp_mx_linear = init_mxfp6_linear_kernel(
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.emulate:
|
||||
dq_w = self.dequant_func(layer.weight, layer.weight_scale, x.dtype)
|
||||
qdq_x = self.quant_dequant_func(x)
|
||||
return F.linear(qdq_x, dq_w, bias)
|
||||
return self.ocp_mx_linear.apply_weights(layer, x, bias)
|
||||
|
||||
@@ -90,6 +90,7 @@ class ScaleDesc:
|
||||
GroupShape.PER_CHANNEL: "per_channel",
|
||||
}
|
||||
group_shape = d.get(self.group_shape, str(self.group_shape))
|
||||
|
||||
return (
|
||||
f"{fx.graph.dtype_abbrs[self.dtype]},"
|
||||
f"{'static' if self.static else 'dynamic'},{group_shape}"
|
||||
@@ -106,15 +107,24 @@ class QuantKey:
|
||||
symmetric: symmetric if True, asymmetric if False
|
||||
"""
|
||||
|
||||
dtype: torch.dtype
|
||||
# TODO: QuantKey.dtype is assumed to be `torch.dtype` in matcher_utils.py,
|
||||
# but #37990 introduced e.g. `kInt4Static` that uses a `ScalarType` dtype,
|
||||
# same for kMxfp6 that does not have a native torch representation.
|
||||
# Logical dtype and storage (torch) dtype should be separated (see #48949).
|
||||
dtype: torch.dtype | ScalarType
|
||||
scale: ScaleDesc
|
||||
scale2: ScaleDesc | None = None
|
||||
symmetric: bool = True
|
||||
|
||||
def __str__(self):
|
||||
scale2_str = f"scale2({self.scale2})," if self.scale2 else ""
|
||||
dtype_description = (
|
||||
fx.graph.dtype_abbrs[self.dtype]
|
||||
if isinstance(self.dtype, torch.dtype)
|
||||
else self.dtype
|
||||
)
|
||||
return (
|
||||
f"QuantKey({fx.graph.dtype_abbrs[self.dtype]},"
|
||||
f"QuantKey({dtype_description},"
|
||||
f"scale({self.scale}),{scale2_str}"
|
||||
f"{'a' if not self.symmetric else ''}symmetric)"
|
||||
)
|
||||
@@ -172,6 +182,26 @@ kMxfp8Dynamic = QuantKey(FP8_DTYPE, scale=kMxfp8DynamicGroupScale, symmetric=Tru
|
||||
kMxfp4StaticGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32))
|
||||
kMxfp4Static = QuantKey(FP4_DTYPE, scale=kMxfp4StaticGroupScale, symmetric=True)
|
||||
|
||||
kMxfp6E3M2StaticGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32))
|
||||
kMxfp6E3M2Static = QuantKey(
|
||||
scalar_types.float6_e3m2f, scale=kMxfp6E3M2StaticGroupScale, symmetric=True
|
||||
)
|
||||
|
||||
kMxfp6E3M2DynamicGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, False, GroupShape(1, 32))
|
||||
kMxfp6E3M2Dynamic = QuantKey(
|
||||
scalar_types.float6_e3m2f, scale=kMxfp6E3M2DynamicGroupScale, symmetric=True
|
||||
)
|
||||
|
||||
kMxfp6E2M3StaticGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32))
|
||||
kMxfp6E2M3Static = QuantKey(
|
||||
scalar_types.float6_e2m3f, scale=kMxfp6E2M3StaticGroupScale, symmetric=True
|
||||
)
|
||||
|
||||
kMxfp6E2M3DynamicGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, False, GroupShape(1, 32))
|
||||
kMxfp6E2M3Dynamic = QuantKey(
|
||||
scalar_types.float6_e2m3f, scale=kMxfp6E2M3DynamicGroupScale, symmetric=True
|
||||
)
|
||||
|
||||
# TODO: convert this to use SCALAR_TYPE. This is not right.
|
||||
kInt4StaticGroupScale = ScaleDesc(torch.float16, True, GroupShape(1, -1))
|
||||
kInt4Static = QuantKey(INT4_DTYPE, scale=kInt4StaticGroupScale, symmetric=True)
|
||||
|
||||
Reference in New Issue
Block a user