[Quantization][Refactor][1/N] Adopt QuantKey in QuarkConfig and methods, relying on weight_quant_key, act_quant_key for quant method dispatch (#52958)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
fxmarty-amd
2026-09-01 02:55:54 -05:00
committed by GitHub
co-authored by Andreas Karatzas
parent d0e695a91b
commit 63988f3c2d
13 changed files with 1574 additions and 476 deletions
@@ -14,6 +14,11 @@ from vllm.model_executor.kernels.linear.nvfp4.emulation import (
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
QuarkScheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
@@ -36,9 +41,14 @@ class QuarkNVFP4(QuarkScheme):
- input_scale_2: bfloat16/float32, scalar (global input scale)
"""
supported_activation_quant_keys = [kNvfp4Dynamic]
supported_weight_quant_keys = [kNvfp4Static]
def __init__(
self,
activation_quant_key: QuantKey | None,
):
super().__init__(kNvfp4Static, activation_quant_key)
self.kernel = init_nvfp4_linear_kernel()
self.group_size = 16
@@ -3,7 +3,6 @@
from collections.abc import Callable
from fractions import Fraction
from typing import Any
import torch
@@ -15,15 +14,14 @@ from vllm.model_executor.kernels.linear import (
init_mxfp6_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
OCP_MX_BLOCK_SIZE,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp6E2M3Dynamic,
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
)
from vllm.model_executor.parameter import (
@@ -38,45 +36,32 @@ from .quark_scheme import QuarkScheme
logger = init_logger(__name__)
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Static,
"mxfp6_e3m2": kMxfp6E3M2Static,
"mxfp6_e2m3": kMxfp6E2M3Static,
}
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Dynamic,
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
}
class QuarkOCP_MX(QuarkScheme):
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None]
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
def __init__(
self,
weight_quant_spec: dict[str, Any],
input_quant_spec: dict[str, Any] | None,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
dynamic_mxfp4_quant: bool = False,
):
super().__init__(weight_quant_key, activation_quant_key)
self.dynamic_mxfp4_quant = dynamic_mxfp4_quant
self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp")
self.input_dtype: str | None = None
if input_quant_spec is not None:
self.input_dtype = input_quant_spec["dtype"].replace("fp", "mxfp")
if self.input_dtype not in [None, *_ACTIVATION_QUANT_KEY_MAP]:
raise ValueError(
f"Unsupported input_dtype={self.input_dtype} for QuarkOCP_MX. "
f"Supported activation dtypes are {_ACTIVATION_QUANT_KEY_MAP.keys()}, "
"or None for weight-only quantization."
self.weight_dtype = next(
dtype
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
if quant_key == weight_quant_key
)
self.input_dtype = (
next(
dtype
for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
if quant_key == activation_quant_key
)
self.weight_quant_key = _WEIGHT_QUANT_KEY_MAP[self.weight_dtype]
self.activation_quant_key = (
_ACTIVATION_QUANT_KEY_MAP[self.input_dtype]
if self.input_dtype is not None
if activation_quant_key is not None
else None
)
@@ -85,17 +70,6 @@ class QuarkOCP_MX(QuarkScheme):
else:
self.packed_factor = Fraction(numerator=8, denominator=6)
if input_quant_spec is None:
self.static_input_scales = False
else:
self.static_input_scales = not input_quant_spec.get("is_dynamic")
if self.static_input_scales:
raise NotImplementedError(
"QuarkOCP_MX with static input scales is currently not "
"implemented. Please open an issue."
)
if not current_platform.supports_mx():
logger.warning_once(
"The current platform does not support native MXFP4/MXFP6 "
@@ -5,6 +5,8 @@ from abc import ABC, abstractmethod
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
__all__ = ["QuarkScheme"]
@@ -14,6 +16,23 @@ class QuarkScheme(ABC):
of different quantization schemes supported by Quark.
"""
supported_activation_quant_keys: list[QuantKey | None] = []
supported_weight_quant_keys: list[QuantKey] = []
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
if activation_quant_key not in self.supported_activation_quant_keys:
raise ValueError(
f"Unsupported activation quant key: {activation_quant_key}"
)
if weight_quant_key not in self.supported_weight_quant_keys:
raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
self.activation_quant_key = activation_quant_key
self.weight_quant_key = weight_quant_key
@classmethod
@abstractmethod
def get_min_capability(cls) -> int:
@@ -3,7 +3,6 @@
from collections.abc import Callable
from fractions import Fraction
from typing import Any
import torch
import torch.nn.functional as F
@@ -11,7 +10,10 @@ import torch.nn.functional as F
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
get_fp8_min_max,
kFp8StaticTensorSym,
kMxfp4Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
@@ -38,39 +40,28 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
Uses the AITER Triton kernel and falls back to emulation if AITER not available.
"""
supported_activation_quant_keys = [kFp8StaticTensorSym]
supported_weight_quant_keys = [kMxfp4Static]
def __init__(
self,
weight_quant_spec: dict[str, Any],
input_quant_spec: dict[str, Any],
activation_quant_key: QuantKey | None,
):
super().__init__(kMxfp4Static, activation_quant_key)
self.out_dtype = None
self.weight_dtype = "mxfp4"
self.packed_factor: Fraction = Fraction(2, 1) # 2 FP4 values per byte
self.weight_block_size = OCP_MX_BLOCK_SIZE
self.is_static_input_scheme = not input_quant_spec.get("is_dynamic")
self.input_qscheme = input_quant_spec.get("qscheme") # "per_tensor"
self.fp8_min, self.fp8_max = get_fp8_min_max()
self.fp8_dtype = current_platform.fp8_dtype()
if not self.is_static_input_scheme:
raise NotImplementedError(
"Dynamic FP8 activation quantization is not yet supported "
"for W4A8. The current implementation expects static per-tensor "
"FP8 scales stored in the checkpoint."
)
kernel_supported_gpu = False
if current_platform.is_rocm():
kernel_supported_gpu = current_platform.supports_mx()
self.use_aiter_kernel = (
is_aiter_found_and_supported()
and self.is_static_input_scheme
and kernel_supported_gpu
)
self.use_aiter_kernel = is_aiter_found_and_supported() and kernel_supported_gpu
if not self.use_aiter_kernel:
logger.warning_once(
@@ -128,17 +119,16 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
layer.register_parameter("weight_scale", weight_scale)
# INPUT SCALE (FP8 per-tensor static scale)
if self.is_static_input_scheme:
input_scale = PerTensorScaleParameter(
data=torch.empty(
len(output_partition_sizes),
dtype=torch.float32,
),
weight_loader=weight_loader,
)
# Initialize to avoid NaN
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
input_scale = PerTensorScaleParameter(
data=torch.empty(
len(output_partition_sizes),
dtype=torch.float32,
),
weight_loader=weight_loader,
)
# Initialize to avoid NaN
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Ensuring weights & scales are non-trainable
@@ -147,16 +137,15 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
layer.weight_scale.data, requires_grad=False
)
if self.is_static_input_scheme:
input_scale = layer.input_scale.data
# For fused modules (QKV), take the max scale
if input_scale.numel() != 1:
input_scale = input_scale.max()
input_scale = layer.input_scale.data
# For fused modules (QKV), take the max scale
if input_scale.numel() != 1:
input_scale = input_scale.max()
layer.input_scale = torch.nn.Parameter(
torch.tensor(input_scale, dtype=torch.float32),
requires_grad=False,
)
layer.input_scale = torch.nn.Parameter(
torch.tensor(input_scale, dtype=torch.float32),
requires_grad=False,
)
def apply_weights(
self,
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import Any, cast
import torch
from torch.nn import Parameter
@@ -18,8 +17,11 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockE8M0Sym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
@@ -42,36 +44,26 @@ logger = init_logger(__name__)
class QuarkW8A8Fp8(QuarkScheme):
supported_activation_quant_keys: list[QuantKey | None] = [
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticTensorSym,
]
supported_weight_quant_keys: list[QuantKey] = [
kFp8StaticChannelSym,
kFp8StaticTensorSym,
]
def __init__(
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
self.is_static_input_scheme: bool = False
self.input_qscheme: str | None = None
if input_config is not None:
self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic"))
self.input_qscheme = cast(str, input_config.get("qscheme"))
if self.weight_qscheme == "per_block":
raise ValueError(
"Quark W8A8 FP8 per-block weight quantization should use "
"QuarkW8A8Fp8PerBlock."
)
per_token_activation = (
not self.is_static_input_scheme and self.input_qscheme == "per_channel"
)
per_channel_weight = self.weight_qscheme == "per_channel"
self.activation_quant_key = (
kFp8DynamicTokenSym if per_token_activation else kFp8StaticTensorSym
)
# A per-output-channel weight scale is one fp32 value per weight row
# (length N). Tag it as ``GroupShape.PER_CHANNEL`` to match the
# canonical compressed-tensors CHANNEL strategy, so kernel selection
# (e.g. AITER's pre-shuffled FP8 GEMM) treats it uniformly.
self.weight_quant_key = (
kFp8StaticChannelSym if per_channel_weight else kFp8StaticTensorSym
super().__init__(weight_quant_key, activation_quant_key)
self.weight_qscheme = (
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
)
self.is_static_input_scheme = activation_quant_key == kFp8StaticTensorSym
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@@ -122,6 +114,8 @@ class QuarkW8A8Fp8(QuarkScheme):
layer.input_scale = Parameter(input_scale, requires_grad=False)
else:
weight_scale = layer.weight_scale.data
assert self.activation_quant_key is not None
if self.activation_quant_key.scale.group_shape == GroupShape.PER_TOKEN:
weight_scale = weight_scale.view(-1, 1)
layer.weight = Parameter(weight.t(), requires_grad=False)
@@ -191,6 +185,7 @@ class QuarkW8A8Fp8(QuarkScheme):
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
@@ -210,22 +205,23 @@ class QuarkW8A8Fp8(QuarkScheme):
class QuarkW8A8Fp8PerBlock(QuarkScheme):
supported_activation_quant_keys = [kFp8Dynamic128Sym]
supported_weight_quant_keys = [
kFp8Static128BlockSym,
kFp8Static128BlockE8M0Sym,
]
def __init__(
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
if self.weight_qscheme != "per_block":
raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.")
block_size = weight_config.get("block_size")
if not block_size:
raise ValueError(
"Quark W8A8 FP8 per-block weight quantization requires "
"`block_size` in the weight quantization config."
)
self.weight_config = weight_config
self.weight_block_size = list(block_size)
self.activation_quant_key = kFp8Dynamic128Sym
self.weight_quant_key = kFp8Static128BlockSym
super().__init__(weight_quant_key, activation_quant_key)
self.weight_scale_dtype = weight_quant_key.scale.dtype
if self.weight_quant_key == kFp8Static128BlockE8M0Sym:
# TODO: oracle needs to support kFp8Static128BlockE8M0Sym.
self.weight_quant_key = kFp8Static128BlockSym
self.weight_block_size = list(weight_quant_key.scale.group_shape)
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@@ -272,8 +268,8 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
layer.register_parameter("weight", weight)
scale_dtype = (
torch.float8_e8m0fnu
if self.weight_config.get("scale_type") == "float8_e8m0fnu"
self.weight_scale_dtype
if self.weight_scale_dtype != torch.float32
else None
)
weight_scale = create_fp8_scale_parameter(
@@ -287,6 +283,7 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
layer.register_parameter("weight_scale_inv", weight_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
@@ -10,6 +10,16 @@ from vllm.model_executor.kernels.linear import (
init_int8_linear_kernel,
)
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTokenAsym,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
kInt8StaticTensorAsym,
kInt8StaticTensorSym,
)
from vllm.model_executor.parameter import (
BasevLLMParameter,
ChannelQuantScaleParameter,
@@ -21,15 +31,32 @@ logger = init_logger(__name__)
class QuarkW8A8Int8(QuarkScheme):
supported_activation_quant_keys = [
kInt8StaticTensorSym,
kInt8StaticTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTensorAsym,
kInt8DynamicTokenSym,
kInt8DynamicTokenAsym,
]
supported_weight_quant_keys = [
kInt8StaticChannelSym,
kInt8StaticTensorSym,
]
def __init__(
self,
qscheme: str,
is_static_input_scheme: bool | None,
input_symmetric: bool | None,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
self.qscheme = qscheme
self.is_static_input_scheme = is_static_input_scheme
self.input_symmetric = input_symmetric
super().__init__(weight_quant_key, activation_quant_key)
self.qscheme = (
"per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
)
assert activation_quant_key is not None
self.is_static_input_scheme = activation_quant_key.scale.static
self.input_symmetric = activation_quant_key.symmetric
@classmethod
def get_min_capability(cls) -> int: