[Quantization][Refactor][1/N] Adopt QuantKey in QuarkConfig and methods, relying on weight_quant_key, act_quant_key for quant method dispatch (#52958)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
d0e695a91b
commit
63988f3c2d
@@ -14,6 +14,11 @@ from vllm.model_executor.kernels.linear.nvfp4.emulation import (
|
||||
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
|
||||
QuarkScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
@@ -36,9 +41,14 @@ class QuarkNVFP4(QuarkScheme):
|
||||
- input_scale_2: bfloat16/float32, scalar (global input scale)
|
||||
"""
|
||||
|
||||
supported_activation_quant_keys = [kNvfp4Dynamic]
|
||||
supported_weight_quant_keys = [kNvfp4Static]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(kNvfp4Static, activation_quant_key)
|
||||
self.kernel = init_nvfp4_linear_kernel()
|
||||
self.group_size = 16
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -15,15 +14,14 @@ from vllm.model_executor.kernels.linear import (
|
||||
init_mxfp6_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
OCP_MX_BLOCK_SIZE,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
@@ -38,45 +36,32 @@ from .quark_scheme import QuarkScheme
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Static,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Static,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Static,
|
||||
}
|
||||
|
||||
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Dynamic,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
|
||||
}
|
||||
|
||||
|
||||
class QuarkOCP_MX(QuarkScheme):
|
||||
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
|
||||
supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None]
|
||||
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_spec: dict[str, Any],
|
||||
input_quant_spec: dict[str, Any] | None,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
dynamic_mxfp4_quant: bool = False,
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.dynamic_mxfp4_quant = dynamic_mxfp4_quant
|
||||
self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp")
|
||||
self.input_dtype: str | None = None
|
||||
if input_quant_spec is not None:
|
||||
self.input_dtype = input_quant_spec["dtype"].replace("fp", "mxfp")
|
||||
|
||||
if self.input_dtype not in [None, *_ACTIVATION_QUANT_KEY_MAP]:
|
||||
raise ValueError(
|
||||
f"Unsupported input_dtype={self.input_dtype} for QuarkOCP_MX. "
|
||||
f"Supported activation dtypes are {_ACTIVATION_QUANT_KEY_MAP.keys()}, "
|
||||
"or None for weight-only quantization."
|
||||
self.weight_dtype = next(
|
||||
dtype
|
||||
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
|
||||
if quant_key == weight_quant_key
|
||||
)
|
||||
self.input_dtype = (
|
||||
next(
|
||||
dtype
|
||||
for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
|
||||
if quant_key == activation_quant_key
|
||||
)
|
||||
|
||||
self.weight_quant_key = _WEIGHT_QUANT_KEY_MAP[self.weight_dtype]
|
||||
self.activation_quant_key = (
|
||||
_ACTIVATION_QUANT_KEY_MAP[self.input_dtype]
|
||||
if self.input_dtype is not None
|
||||
if activation_quant_key is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -85,17 +70,6 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
else:
|
||||
self.packed_factor = Fraction(numerator=8, denominator=6)
|
||||
|
||||
if input_quant_spec is None:
|
||||
self.static_input_scales = False
|
||||
else:
|
||||
self.static_input_scales = not input_quant_spec.get("is_dynamic")
|
||||
|
||||
if self.static_input_scales:
|
||||
raise NotImplementedError(
|
||||
"QuarkOCP_MX with static input scales is currently not "
|
||||
"implemented. Please open an issue."
|
||||
)
|
||||
|
||||
if not current_platform.supports_mx():
|
||||
logger.warning_once(
|
||||
"The current platform does not support native MXFP4/MXFP6 "
|
||||
|
||||
@@ -5,6 +5,8 @@ from abc import ABC, abstractmethod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
__all__ = ["QuarkScheme"]
|
||||
|
||||
|
||||
@@ -14,6 +16,23 @@ class QuarkScheme(ABC):
|
||||
of different quantization schemes supported by Quark.
|
||||
"""
|
||||
|
||||
supported_activation_quant_keys: list[QuantKey | None] = []
|
||||
supported_weight_quant_keys: list[QuantKey] = []
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
if activation_quant_key not in self.supported_activation_quant_keys:
|
||||
raise ValueError(
|
||||
f"Unsupported activation quant key: {activation_quant_key}"
|
||||
)
|
||||
if weight_quant_key not in self.supported_weight_quant_keys:
|
||||
raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
|
||||
self.activation_quant_key = activation_quant_key
|
||||
self.weight_quant_key = weight_quant_key
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -11,7 +10,10 @@ import torch.nn.functional as F
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
get_fp8_min_max,
|
||||
kFp8StaticTensorSym,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
@@ -38,39 +40,28 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
|
||||
Uses the AITER Triton kernel and falls back to emulation if AITER not available.
|
||||
"""
|
||||
|
||||
supported_activation_quant_keys = [kFp8StaticTensorSym]
|
||||
supported_weight_quant_keys = [kMxfp4Static]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_spec: dict[str, Any],
|
||||
input_quant_spec: dict[str, Any],
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(kMxfp4Static, activation_quant_key)
|
||||
self.out_dtype = None
|
||||
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.packed_factor: Fraction = Fraction(2, 1) # 2 FP4 values per byte
|
||||
self.weight_block_size = OCP_MX_BLOCK_SIZE
|
||||
|
||||
self.is_static_input_scheme = not input_quant_spec.get("is_dynamic")
|
||||
self.input_qscheme = input_quant_spec.get("qscheme") # "per_tensor"
|
||||
|
||||
self.fp8_min, self.fp8_max = get_fp8_min_max()
|
||||
self.fp8_dtype = current_platform.fp8_dtype()
|
||||
|
||||
if not self.is_static_input_scheme:
|
||||
raise NotImplementedError(
|
||||
"Dynamic FP8 activation quantization is not yet supported "
|
||||
"for W4A8. The current implementation expects static per-tensor "
|
||||
"FP8 scales stored in the checkpoint."
|
||||
)
|
||||
|
||||
kernel_supported_gpu = False
|
||||
if current_platform.is_rocm():
|
||||
kernel_supported_gpu = current_platform.supports_mx()
|
||||
|
||||
self.use_aiter_kernel = (
|
||||
is_aiter_found_and_supported()
|
||||
and self.is_static_input_scheme
|
||||
and kernel_supported_gpu
|
||||
)
|
||||
self.use_aiter_kernel = is_aiter_found_and_supported() and kernel_supported_gpu
|
||||
|
||||
if not self.use_aiter_kernel:
|
||||
logger.warning_once(
|
||||
@@ -128,17 +119,16 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
# INPUT SCALE (FP8 per-tensor static scale)
|
||||
if self.is_static_input_scheme:
|
||||
input_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(
|
||||
len(output_partition_sizes),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
# Initialize to avoid NaN
|
||||
input_scale[:] = torch.finfo(torch.float32).min
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
input_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(
|
||||
len(output_partition_sizes),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
# Initialize to avoid NaN
|
||||
input_scale[:] = torch.finfo(torch.float32).min
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Ensuring weights & scales are non-trainable
|
||||
@@ -147,16 +137,15 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
|
||||
layer.weight_scale.data, requires_grad=False
|
||||
)
|
||||
|
||||
if self.is_static_input_scheme:
|
||||
input_scale = layer.input_scale.data
|
||||
# For fused modules (QKV), take the max scale
|
||||
if input_scale.numel() != 1:
|
||||
input_scale = input_scale.max()
|
||||
input_scale = layer.input_scale.data
|
||||
# For fused modules (QKV), take the max scale
|
||||
if input_scale.numel() != 1:
|
||||
input_scale = input_scale.max()
|
||||
|
||||
layer.input_scale = torch.nn.Parameter(
|
||||
torch.tensor(input_scale, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.input_scale = torch.nn.Parameter(
|
||||
torch.tensor(input_scale, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
@@ -18,8 +17,11 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockE8M0Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
@@ -42,36 +44,26 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkW8A8Fp8(QuarkScheme):
|
||||
supported_activation_quant_keys: list[QuantKey | None] = [
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticTensorSym,
|
||||
]
|
||||
supported_weight_quant_keys: list[QuantKey] = [
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
self.is_static_input_scheme: bool = False
|
||||
self.input_qscheme: str | None = None
|
||||
if input_config is not None:
|
||||
self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic"))
|
||||
self.input_qscheme = cast(str, input_config.get("qscheme"))
|
||||
|
||||
if self.weight_qscheme == "per_block":
|
||||
raise ValueError(
|
||||
"Quark W8A8 FP8 per-block weight quantization should use "
|
||||
"QuarkW8A8Fp8PerBlock."
|
||||
)
|
||||
per_token_activation = (
|
||||
not self.is_static_input_scheme and self.input_qscheme == "per_channel"
|
||||
)
|
||||
per_channel_weight = self.weight_qscheme == "per_channel"
|
||||
|
||||
self.activation_quant_key = (
|
||||
kFp8DynamicTokenSym if per_token_activation else kFp8StaticTensorSym
|
||||
)
|
||||
# A per-output-channel weight scale is one fp32 value per weight row
|
||||
# (length N). Tag it as ``GroupShape.PER_CHANNEL`` to match the
|
||||
# canonical compressed-tensors CHANNEL strategy, so kernel selection
|
||||
# (e.g. AITER's pre-shuffled FP8 GEMM) treats it uniformly.
|
||||
self.weight_quant_key = (
|
||||
kFp8StaticChannelSym if per_channel_weight else kFp8StaticTensorSym
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.weight_qscheme = (
|
||||
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
|
||||
)
|
||||
self.is_static_input_scheme = activation_quant_key == kFp8StaticTensorSym
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
|
||||
@@ -122,6 +114,8 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
layer.input_scale = Parameter(input_scale, requires_grad=False)
|
||||
else:
|
||||
weight_scale = layer.weight_scale.data
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
if self.activation_quant_key.scale.group_shape == GroupShape.PER_TOKEN:
|
||||
weight_scale = weight_scale.view(-1, 1)
|
||||
layer.weight = Parameter(weight.t(), requires_grad=False)
|
||||
@@ -191,6 +185,7 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
input_scale[:] = torch.finfo(torch.float32).min
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
@@ -210,22 +205,23 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
|
||||
|
||||
class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
supported_activation_quant_keys = [kFp8Dynamic128Sym]
|
||||
supported_weight_quant_keys = [
|
||||
kFp8Static128BlockSym,
|
||||
kFp8Static128BlockE8M0Sym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
if self.weight_qscheme != "per_block":
|
||||
raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.")
|
||||
block_size = weight_config.get("block_size")
|
||||
if not block_size:
|
||||
raise ValueError(
|
||||
"Quark W8A8 FP8 per-block weight quantization requires "
|
||||
"`block_size` in the weight quantization config."
|
||||
)
|
||||
self.weight_config = weight_config
|
||||
self.weight_block_size = list(block_size)
|
||||
self.activation_quant_key = kFp8Dynamic128Sym
|
||||
self.weight_quant_key = kFp8Static128BlockSym
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.weight_scale_dtype = weight_quant_key.scale.dtype
|
||||
if self.weight_quant_key == kFp8Static128BlockE8M0Sym:
|
||||
# TODO: oracle needs to support kFp8Static128BlockE8M0Sym.
|
||||
self.weight_quant_key = kFp8Static128BlockSym
|
||||
self.weight_block_size = list(weight_quant_key.scale.group_shape)
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
|
||||
@@ -272,8 +268,8 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
scale_dtype = (
|
||||
torch.float8_e8m0fnu
|
||||
if self.weight_config.get("scale_type") == "float8_e8m0fnu"
|
||||
self.weight_scale_dtype
|
||||
if self.weight_scale_dtype != torch.float32
|
||||
else None
|
||||
)
|
||||
weight_scale = create_fp8_scale_parameter(
|
||||
@@ -287,6 +283,7 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
|
||||
layer.register_parameter("weight_scale_inv", weight_scale)
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
|
||||
@@ -10,6 +10,16 @@ from vllm.model_executor.kernels.linear import (
|
||||
init_int8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8StaticTensorSym,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
@@ -21,15 +31,32 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkW8A8Int8(QuarkScheme):
|
||||
supported_activation_quant_keys = [
|
||||
kInt8StaticTensorSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
]
|
||||
supported_weight_quant_keys = [
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorSym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
qscheme: str,
|
||||
is_static_input_scheme: bool | None,
|
||||
input_symmetric: bool | None,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
self.qscheme = qscheme
|
||||
self.is_static_input_scheme = is_static_input_scheme
|
||||
self.input_symmetric = input_symmetric
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.qscheme = (
|
||||
"per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
|
||||
)
|
||||
|
||||
assert activation_quant_key is not None
|
||||
self.is_static_input_scheme = activation_quant_key.scale.static
|
||||
self.input_symmetric = activation_quant_key.symmetric
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
|
||||
Reference in New Issue
Block a user