[ROCm][Quark][7/N] Use MXFP4 linear kernel abstraction for emulation backend (#48949)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
@@ -3,25 +3,28 @@
|
||||
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
dequant_mxfp4,
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
|
||||
dequant_mxfp6,
|
||||
quant_dequant_mxfp6,
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MxFp4LinearKernel,
|
||||
MxFp6LinearKernel,
|
||||
init_mxfp4_linear_kernel,
|
||||
init_mxfp6_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
OCP_MX_BLOCK_SIZE,
|
||||
OCP_MX_Scheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
@@ -35,8 +38,22 @@ from .quark_scheme import QuarkScheme
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Static,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Static,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Static,
|
||||
}
|
||||
|
||||
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Dynamic,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
|
||||
}
|
||||
|
||||
|
||||
class QuarkOCP_MX(QuarkScheme):
|
||||
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_spec: dict[str, Any],
|
||||
@@ -49,35 +66,26 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp")
|
||||
self.input_dtype: str | None = None
|
||||
if input_quant_spec is not None:
|
||||
input_quant = input_quant_spec["dtype"]
|
||||
if input_quant == "fp8_e4m3":
|
||||
self.input_dtype = "fp8"
|
||||
else:
|
||||
self.input_dtype = input_quant.replace("fp", "mxfp")
|
||||
self.input_dtype = input_quant_spec["dtype"].replace("fp", "mxfp")
|
||||
|
||||
self.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(
|
||||
self.input_dtype, self.weight_dtype
|
||||
if self.input_dtype not in [None, *_ACTIVATION_QUANT_KEY_MAP]:
|
||||
raise ValueError(
|
||||
f"Unsupported input_dtype={self.input_dtype} for QuarkOCP_MX. "
|
||||
f"Supported activation dtypes are {_ACTIVATION_QUANT_KEY_MAP.keys()}, "
|
||||
"or None for weight-only quantization."
|
||||
)
|
||||
|
||||
self.weight_quant_key = _WEIGHT_QUANT_KEY_MAP[self.weight_dtype]
|
||||
self.activation_quant_key = (
|
||||
_ACTIVATION_QUANT_KEY_MAP[self.input_dtype]
|
||||
if self.input_dtype is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if self.weight_dtype == "mxfp4":
|
||||
self.packed_factor: int | Fraction = 2
|
||||
self.dequant_func = dequant_mxfp4
|
||||
else:
|
||||
self.packed_factor = Fraction(numerator=8, denominator=6)
|
||||
self.dequant_func = partial(
|
||||
dequant_mxfp6, quant_dtype=self.weight_dtype.replace("mx", "")
|
||||
)
|
||||
|
||||
if self.input_dtype is None:
|
||||
self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = (
|
||||
lambda x: x
|
||||
) # no input Q/DQ for weight-only
|
||||
elif self.input_dtype == "mxfp4":
|
||||
self.quant_dequant_func = quant_dequant_mxfp4
|
||||
else:
|
||||
self.quant_dequant_func = partial(
|
||||
quant_dequant_mxfp6, quant_dtype=self.input_dtype.replace("mx", "")
|
||||
)
|
||||
|
||||
if input_quant_spec is None:
|
||||
self.static_input_scales = False
|
||||
@@ -90,16 +98,6 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
"implemented. Please open an issue."
|
||||
)
|
||||
|
||||
# TODO: integrate (or test) mixed-precision kernel.
|
||||
self.emulate = not current_platform.supports_mx() or (
|
||||
self.input_dtype != "mxfp4" or self.weight_dtype != "mxfp4"
|
||||
)
|
||||
|
||||
# TODO: Move emulation code path as a kernel, and always
|
||||
# use init_mxfp4_linear_kernel.
|
||||
if not self.emulate:
|
||||
self.ocp_mx_linear = init_mxfp4_linear_kernel()
|
||||
|
||||
if not current_platform.supports_mx():
|
||||
logger.warning_once(
|
||||
"The current platform does not support native MXFP4/MXFP6 "
|
||||
@@ -151,17 +149,10 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
|
||||
|
||||
if self.emulate:
|
||||
if self.dynamic_mxfp4_quant:
|
||||
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
||||
else:
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
layer.weight_scale.data, requires_grad=False
|
||||
)
|
||||
else:
|
||||
if self.dynamic_mxfp4_quant:
|
||||
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
||||
self.ocp_mx_linear.process_weights_after_loading(layer)
|
||||
if self.dynamic_mxfp4_quant:
|
||||
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
||||
|
||||
self.ocp_mx_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -218,14 +209,20 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
if self.weight_quant_key == kMxfp4Static:
|
||||
self.ocp_mx_linear = init_mxfp4_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
)
|
||||
elif self.weight_quant_key in [kMxfp6E2M3Static, kMxfp6E3M2Static]:
|
||||
self.ocp_mx_linear = init_mxfp6_linear_kernel(
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.emulate:
|
||||
dq_w = self.dequant_func(layer.weight, layer.weight_scale, x.dtype)
|
||||
qdq_x = self.quant_dequant_func(x)
|
||||
return F.linear(qdq_x, dq_w, bias)
|
||||
return self.ocp_mx_linear.apply_weights(layer, x, bias)
|
||||
|
||||
Reference in New Issue
Block a user