[Quantization][Refactor][1/N] Adopt QuantKey in QuarkConfig and methods, relying on weight_quant_key, act_quant_key for quant method dispatch (#52958)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
d0e695a91b
commit
63988f3c2d
@@ -35,6 +35,10 @@ from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
mxfp4_quantize,
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import PlatformEnum, current_platform
|
||||
@@ -346,9 +350,9 @@ def test_online_mxfp4_moe_matches_quark(
|
||||
online_layer = make_layer("online_layer")
|
||||
|
||||
checkpoint_method = QuarkOCP_MX_MoEMethod(
|
||||
weight_config={"qscheme": "per_group", "dtype": "fp4"},
|
||||
input_config={"dtype": "fp4", "is_dynamic": True},
|
||||
moe=checkpoint_layer.moe_config,
|
||||
weight_quant_key=kMxfp4Static,
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
)
|
||||
online_method = Mxfp4OnlineMoEMethod(layer=online_layer)
|
||||
|
||||
@@ -566,8 +570,8 @@ def test_online_mxfp4_dense_matches_quark(
|
||||
checkpoint_weight, checkpoint_weight_scale = mxfp4_quantize(weight)
|
||||
|
||||
checkpoint_scheme = QuarkOCP_MX(
|
||||
weight_quant_spec={"qscheme": "per_group", "dtype": "fp4"},
|
||||
input_quant_spec={"dtype": "fp4", "is_dynamic": True},
|
||||
weight_quant_key=kMxfp4Static,
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
)
|
||||
checkpoint_scheme.create_weights(
|
||||
layer=checkpoint_layer,
|
||||
|
||||
@@ -10,6 +10,7 @@ See also `tests/kernels/moe/test_ocp_mx_moe.py`.
|
||||
import importlib.metadata
|
||||
from dataclasses import dataclass
|
||||
from importlib.util import find_spec
|
||||
from types import SimpleNamespace
|
||||
|
||||
import huggingface_hub
|
||||
import lm_eval
|
||||
@@ -18,22 +19,62 @@ import torch
|
||||
from packaging import version
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
|
||||
QuarkConfig,
|
||||
QuarkLinearMethod,
|
||||
QuarkNVFP4,
|
||||
QuarkOCP_MX,
|
||||
QuarkW8A8Fp8,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
QuarkW8A8Int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
|
||||
QuarkMoEMethod,
|
||||
QuarkW4A8Fp8MoEMethod,
|
||||
QuarkW8A8Int8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
is_layer_skipped,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockE8M0Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4W4A8StaticChannelSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8StaticTensorSym,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
@@ -62,6 +103,576 @@ AITER_AVAILABLE = is_aiter_found_and_supported()
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QTensorConfig:
|
||||
name: str
|
||||
weight: QuarkQTensorHint
|
||||
input_tensors: QuarkQTensorHint
|
||||
weight_quant_key: QuantKey | None = None
|
||||
act_quant_key: QuantKey | None = None
|
||||
dispatch_cls: type[QuarkScheme] | type[QuarkMoEMethod] | None = None
|
||||
expected_error: tuple[type[Exception], str] | None = None
|
||||
|
||||
|
||||
QTENSOR_CONFIGS = [
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_static_tensor",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_static_tensor_single_entry_lists",
|
||||
weight=[
|
||||
{
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
}
|
||||
],
|
||||
input_tensors=[
|
||||
{
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
}
|
||||
],
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_dynamic_tensor",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
act_quant_key=kFp8DynamicTensorSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_dynamic_token",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_channel", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kFp8StaticChannelSym,
|
||||
act_quant_key=kFp8DynamicTokenSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_channel_static_tensor",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_channel", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
weight_quant_key=kFp8StaticChannelSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_tensor_dynamic_token",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
act_quant_key=kFp8DynamicTokenSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_dynamic_block_fp32",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kFp8Static128BlockSym,
|
||||
act_quant_key=kFp8Dynamic128Sym,
|
||||
dispatch_cls=QuarkW8A8Fp8PerBlock,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_dynamic_block_e8m0",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
"scale_type": "float8_e8m0fnu",
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kFp8Static128BlockE8M0Sym,
|
||||
act_quant_key=kFp8Dynamic128Sym,
|
||||
dispatch_cls=QuarkW8A8Fp8PerBlock,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_block_static_input",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": False,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
},
|
||||
expected_error=(NotImplementedError, "No quark compatible scheme"),
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_block_group_size_mismatch",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 64,
|
||||
"symmetric": True,
|
||||
},
|
||||
expected_error=(NotImplementedError, "No quark compatible scheme"),
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_block_missing_block_size",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
},
|
||||
expected_error=(ValueError, "requires `block_size`"),
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_static_symmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kInt8StaticTensorSym,
|
||||
act_quant_key=kInt8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_static_asymmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": False,
|
||||
},
|
||||
weight_quant_key=kInt8StaticTensorSym,
|
||||
act_quant_key=kInt8StaticTensorAsym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_channel_static_symmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kInt8StaticChannelSym,
|
||||
act_quant_key=kInt8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_channel_static_asymmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": False,
|
||||
},
|
||||
weight_quant_key=kInt8StaticChannelSym,
|
||||
act_quant_key=kInt8StaticTensorAsym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_dynamic_tensor_symmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kInt8StaticTensorSym,
|
||||
act_quant_key=kInt8DynamicTensorSym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_dynamic_tensor_asymmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
"symmetric": False,
|
||||
},
|
||||
weight_quant_key=kInt8StaticTensorSym,
|
||||
act_quant_key=kInt8DynamicTensorAsym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_dynamic_token",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kInt8StaticChannelSym,
|
||||
act_quant_key=kInt8DynamicTokenSym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_dynamic_token_asymmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
"symmetric": False,
|
||||
},
|
||||
weight_quant_key=kInt8StaticChannelSym,
|
||||
act_quant_key=kInt8DynamicTokenAsym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp4_weight_only",
|
||||
weight={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors=None,
|
||||
weight_quant_key=kMxfp4Static,
|
||||
act_quant_key=None,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp4_activation",
|
||||
weight={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp4Static,
|
||||
act_quant_key=kMxfp4Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp6_e3m2",
|
||||
weight={
|
||||
"dtype": "fp6_e3m2",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp6_e3m2",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp6E3M2Static,
|
||||
act_quant_key=kMxfp6E3M2Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp4_mxfp6_e3m2_activation",
|
||||
weight={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp6_e3m2",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp4Static,
|
||||
act_quant_key=kMxfp6E3M2Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp4_mxfp6_e2m3_activation",
|
||||
weight={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp6_e2m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp4Static,
|
||||
act_quant_key=kMxfp6E2M3Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp6_e2m3",
|
||||
weight={
|
||||
"dtype": "fp6_e2m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp6_e2m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp6E2M3Static,
|
||||
act_quant_key=kMxfp6E2M3Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="nvfp4",
|
||||
weight=[
|
||||
{
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 16,
|
||||
"is_dynamic": False,
|
||||
},
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
],
|
||||
input_tensors=[
|
||||
{
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 16,
|
||||
"is_dynamic": True,
|
||||
},
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
],
|
||||
weight_quant_key=kNvfp4Static,
|
||||
act_quant_key=kNvfp4Dynamic,
|
||||
dispatch_cls=QuarkNVFP4,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="w4a8_fp8_static",
|
||||
weight=[
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
{
|
||||
"dtype": "int4",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
"ch_axis": 0,
|
||||
},
|
||||
],
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
weight_quant_key=kInt4W4A8StaticChannelSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW4A8Fp8MoEMethod,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="w4a8_fp8_dynamic",
|
||||
weight=[
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
{
|
||||
"dtype": "int4",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
"ch_axis": 0,
|
||||
},
|
||||
],
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kInt4W4A8StaticChannelSym,
|
||||
act_quant_key=kFp8DynamicTokenSym,
|
||||
dispatch_cls=QuarkW4A8Fp8MoEMethod,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="w4a8_fp8_static_single_entry_input",
|
||||
weight=[
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
{
|
||||
"dtype": "int4",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
"ch_axis": 0,
|
||||
},
|
||||
],
|
||||
input_tensors=[
|
||||
{
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
}
|
||||
],
|
||||
weight_quant_key=kInt4W4A8StaticChannelSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW4A8Fp8MoEMethod,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _make_qtensor_config(
|
||||
weight: QuarkQTensorHint,
|
||||
input_tensors: QuarkQTensorHint,
|
||||
exclude: list[str] | None = None,
|
||||
) -> QuarkConfig:
|
||||
return QuarkConfig(
|
||||
{
|
||||
"global_quant_config": {
|
||||
"weight": weight,
|
||||
"input_tensors": input_tensors,
|
||||
},
|
||||
"layer_type_quant_config": {},
|
||||
"exclude": exclude or [],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _make_test_moe_config() -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
activation=MoEActivation.SILU,
|
||||
device=current_platform.device_type,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
if QUARK_MXFP4_AVAILABLE:
|
||||
from quark.torch.export.nn.modules.realquantizer import StaticScaledRealQuantizer
|
||||
from quark.torch.kernel import mx as mx_kernel
|
||||
@@ -98,83 +709,197 @@ def test_quark_config_preserves_existing_packed_modules_mapping():
|
||||
assert config.packed_modules_mapping["custom_proj"] == ["a", "b"]
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_detects_per_block_config():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
def test_quant_method_dispatch_ignored(default_vllm_config):
|
||||
config = _make_qtensor_config(None, None, exclude=["linear", "experts"])
|
||||
|
||||
assert config._is_fp8_w8a8(weight_config, input_config)
|
||||
class TestLinear(LinearBase):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
class TestRoutedExperts(RoutedExperts):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.moe_config = _make_test_moe_config()
|
||||
|
||||
assert config.get_quant_method_target("linear", LinearBase) == (
|
||||
None,
|
||||
None,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
config.get_quant_method(TestLinear(), "linear"), UnquantizedLinearMethod
|
||||
)
|
||||
|
||||
assert config.get_quant_method_target("experts", RoutedExperts) == (
|
||||
None,
|
||||
None,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
config.get_quant_method(TestRoutedExperts(), "experts"),
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
dynamic_mxfp4_config = _make_qtensor_config(
|
||||
{
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
None,
|
||||
exclude=["self_attn.q_proj", "mlp.down_proj"],
|
||||
)
|
||||
dynamic_mxfp4_config.dynamic_mxfp4_quant = True
|
||||
|
||||
assert dynamic_mxfp4_config.get_quant_method_target(
|
||||
"self_attn.q_proj", LinearBase
|
||||
) == (kMxfp4Static, None, QuarkLinearMethod)
|
||||
attention_proj = TestLinear()
|
||||
assert isinstance(
|
||||
dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"),
|
||||
QuarkLinearMethod,
|
||||
)
|
||||
assert isinstance(attention_proj.scheme, QuarkOCP_MX)
|
||||
assert attention_proj.scheme.dynamic_mxfp4_quant
|
||||
|
||||
assert dynamic_mxfp4_config.get_quant_method_target(
|
||||
"mlp.down_proj", LinearBase
|
||||
) == (None, None, UnquantizedLinearMethod)
|
||||
assert isinstance(
|
||||
dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_rejects_per_block_static_input():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": False,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
@pytest.mark.parametrize("case", QTENSOR_CONFIGS, ids=lambda case: case.name)
|
||||
def test_quant_method_dispatch_target(case):
|
||||
config = _make_qtensor_config(case.weight, case.input_tensors)
|
||||
if case.expected_error is not None:
|
||||
error_type, error_message = case.expected_error
|
||||
with pytest.raises(error_type, match=error_message):
|
||||
config.get_quant_method_target("linear", LinearBase)
|
||||
return
|
||||
|
||||
assert not config._is_fp8_w8a8(weight_config, input_config)
|
||||
assert case.dispatch_cls is not None
|
||||
is_linear = issubclass(case.dispatch_cls, QuarkScheme)
|
||||
|
||||
weight_quant_key, act_quant_key, method_cls = config.get_quant_method_target(
|
||||
"linear" if is_linear else "experts",
|
||||
LinearBase if is_linear else RoutedExperts,
|
||||
)
|
||||
|
||||
assert weight_quant_key == case.weight_quant_key
|
||||
assert act_quant_key == case.act_quant_key
|
||||
assert method_cls is (QuarkLinearMethod if is_linear else case.dispatch_cls)
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_rejects_per_block_group_size_mismatch():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 64,
|
||||
"symmetric": True,
|
||||
}
|
||||
@pytest.mark.parametrize(
|
||||
("weight", "input_tensors"),
|
||||
[
|
||||
pytest.param(
|
||||
{
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
{
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
id="single_entry",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{"dtype": "int8", "qscheme": "per_tensor"},
|
||||
{"dtype": "int8", "qscheme": "per_tensor"},
|
||||
],
|
||||
[
|
||||
{"dtype": "int8", "qscheme": "per_tensor"},
|
||||
{"dtype": "int8", "qscheme": "per_tensor"},
|
||||
],
|
||||
id="multi_entry",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_quant_method_dispatch_unsupported(weight, input_tensors):
|
||||
config = _make_qtensor_config(weight, input_tensors)
|
||||
|
||||
assert not config._is_fp8_w8a8(weight_config, input_config)
|
||||
class TestRoutedExperts(RoutedExperts):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
with pytest.raises(RuntimeError, match="^Unsupported FusedMoe scheme$"):
|
||||
config.get_quant_method_target("experts", RoutedExperts)
|
||||
|
||||
with pytest.raises(RuntimeError, match="^Unsupported FusedMoe scheme$"):
|
||||
config.get_quant_method(TestRoutedExperts(), "experts")
|
||||
|
||||
|
||||
def test_quark_w8a8_fp8_per_block_requires_block_size():
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[case for case in QTENSOR_CONFIGS if case.expected_error is None],
|
||||
ids=lambda case: case.name,
|
||||
)
|
||||
def test_quant_method_dispatch_instantiation(case, monkeypatch, default_vllm_config):
|
||||
config = _make_qtensor_config(case.weight, case.input_tensors)
|
||||
assert case.dispatch_cls is not None
|
||||
if issubclass(case.dispatch_cls, QuarkScheme):
|
||||
|
||||
with pytest.raises(ValueError, match="requires `block_size`"):
|
||||
QuarkW8A8Fp8PerBlock(weight_config, input_config)
|
||||
class TestLinear(LinearBase):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes."
|
||||
"quark_w8a8_fp8.get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=SimpleNamespace(dtype=torch.bfloat16)),
|
||||
)
|
||||
layer = TestLinear()
|
||||
method = config.get_quant_method(layer, "linear")
|
||||
|
||||
assert isinstance(method, QuarkLinearMethod)
|
||||
assert isinstance(layer.scheme, case.dispatch_cls)
|
||||
if case.weight_quant_key == kFp8Static128BlockE8M0Sym:
|
||||
# TODO: Remove once E8M0 quant key is properly handled in oracle
|
||||
assert layer.scheme.weight_quant_key == kFp8Static128BlockSym
|
||||
else:
|
||||
assert layer.scheme.weight_quant_key == case.weight_quant_key
|
||||
assert layer.scheme.activation_quant_key == case.act_quant_key
|
||||
else:
|
||||
|
||||
class TestRoutedExperts(RoutedExperts):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.moe_config = _make_test_moe_config()
|
||||
|
||||
for target in (
|
||||
"select_fp8_moe_backend",
|
||||
"select_int8_moe_backend",
|
||||
"select_mxfp4_moe_backend",
|
||||
"backend_to_kernel_cls",
|
||||
"select_nvfp4_moe_backend",
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
f"vllm.model_executor.layers.quantization.quark.quark_moe.{target}",
|
||||
lambda *args, **kwargs: (object(), object()),
|
||||
)
|
||||
|
||||
# AssertionError: W4A8 FP8 MoE requires ROCm AITER fused MoE support
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.quark.quark_moe."
|
||||
"rocm_aiter_ops.is_fused_moe_enabled",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
layer = TestRoutedExperts()
|
||||
method = config.get_quant_method(layer, "experts")
|
||||
|
||||
assert isinstance(method, case.dispatch_cls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import fnmatch
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, cast
|
||||
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
@@ -37,9 +37,35 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
|
||||
QuarkW8A8Int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
QuarkQTensorHint,
|
||||
deep_compare,
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockE8M0Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4W4A8StaticChannelSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8StaticTensorSym,
|
||||
kMxfp4Static,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -55,6 +81,15 @@ logger = init_logger(__name__)
|
||||
_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"})
|
||||
|
||||
|
||||
class QuantKeyMatch(NamedTuple):
|
||||
matches: bool
|
||||
activation_quant_key: QuantKey | None
|
||||
weight_quant_key: QuantKey | None
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return self.matches
|
||||
|
||||
|
||||
class QuarkConfig(QuantizationConfig):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -155,41 +190,102 @@ class QuarkConfig(QuantizationConfig):
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
# Check if the layer is skipped for quantization.
|
||||
weight_quant_key, activation_quant_key, method_cls = (
|
||||
self.get_quant_method_target(prefix, type(layer))
|
||||
)
|
||||
|
||||
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
|
||||
if should_ignore_layer(
|
||||
is_ignored = should_ignore_layer(
|
||||
prefix,
|
||||
ignore=exclude_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
check_children=isinstance(layer, RoutedExperts),
|
||||
):
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if (
|
||||
"self_attn" not in prefix # only quantize attention projections
|
||||
or not getattr(self, "dynamic_mxfp4_quant", False)
|
||||
or not isinstance(layer, LinearBase) # Ignore other methods
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
)
|
||||
dynamic_mxfp4_quant = (
|
||||
is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant
|
||||
)
|
||||
|
||||
scheme = self.get_scheme(
|
||||
layer=layer,
|
||||
layer_name=prefix,
|
||||
dynamic_mxfp4_quant=True,
|
||||
if method_cls is UnquantizedFusedMoEMethod:
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if method_cls is UnquantizedLinearMethod:
|
||||
return UnquantizedLinearMethod()
|
||||
if method_cls is QuarkLinearMethod:
|
||||
_, _, scheme_cls = self.get_scheme_cls(type(layer), prefix)
|
||||
scheme = self.init_scheme(
|
||||
scheme_cls,
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
dynamic_mxfp4_quant=dynamic_mxfp4_quant,
|
||||
)
|
||||
layer.scheme = scheme
|
||||
return QuarkLinearMethod(self)
|
||||
if isinstance(layer, LinearBase):
|
||||
scheme = self.get_scheme(layer=layer, layer_name=prefix)
|
||||
layer.scheme = scheme
|
||||
return QuarkLinearMethod(self)
|
||||
if isinstance(layer, Attention):
|
||||
if method_cls is QuarkKVCacheMethod:
|
||||
return QuarkKVCacheMethod(self)
|
||||
if method_cls is not None and issubclass(method_cls, QuarkMoEMethod):
|
||||
return QuarkMoEMethod.get_moe_method(
|
||||
self,
|
||||
module=layer,
|
||||
method_cls=method_cls,
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
)
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return QuarkMoEMethod.get_moe_method(self, module=layer, layer_name=prefix)
|
||||
return None
|
||||
|
||||
def get_quant_method_target(
|
||||
self, prefix: str, layer_type: type[torch.nn.Module]
|
||||
) -> tuple[
|
||||
QuantKey | None,
|
||||
QuantKey | None,
|
||||
type[QuantizeMethodBase]
|
||||
| type[UnquantizedLinearMethod]
|
||||
| type[UnquantizedFusedMoEMethod]
|
||||
| None,
|
||||
]:
|
||||
"""Return weight key, activation key, and quant method class for
|
||||
the given ``prefix`` and ``layer_type``.
|
||||
|
||||
This is the counterpart of ``get_quant_method`` without quant method
|
||||
instantiation.
|
||||
|
||||
TODO: integrate as part of base QuantizationConfig.get_quant_method_target
|
||||
in the future.
|
||||
|
||||
``None`` denotes an unquantized/BF16 activation.
|
||||
"""
|
||||
is_routed_experts = issubclass(layer_type, RoutedExperts)
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=cast(list[str], self.quant_config.get("exclude")),
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
check_children=is_routed_experts,
|
||||
):
|
||||
if is_routed_experts:
|
||||
return None, None, UnquantizedFusedMoEMethod
|
||||
if issubclass(layer_type, LinearBase):
|
||||
if "self_attn" in prefix and self.dynamic_mxfp4_quant:
|
||||
weight_quant_key, activation_key, _ = self.get_scheme_cls(
|
||||
layer_type, prefix
|
||||
)
|
||||
return weight_quant_key, activation_key, QuarkLinearMethod
|
||||
return None, None, UnquantizedLinearMethod
|
||||
return None, None, None
|
||||
if issubclass(layer_type, LinearBase):
|
||||
weight_quant_key, activation_key, _ = self.get_scheme_cls(
|
||||
layer_type, prefix
|
||||
)
|
||||
return weight_quant_key, activation_key, QuarkLinearMethod
|
||||
if issubclass(layer_type, Attention):
|
||||
return None, None, QuarkKVCacheMethod
|
||||
if is_routed_experts:
|
||||
weight_quant_key, activation_quant_key, quant_method_cls = (
|
||||
QuarkMoEMethod.get_moe_method_target(self, layer_type, prefix)
|
||||
)
|
||||
if quant_method_cls is None:
|
||||
return None, None, None
|
||||
return weight_quant_key, activation_quant_key, quant_method_cls
|
||||
return None, None, None
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "QuarkConfig":
|
||||
export_config = config.get("export")
|
||||
@@ -290,15 +386,20 @@ class QuarkConfig(QuantizationConfig):
|
||||
|
||||
def _is_fp8_w4a8(
|
||||
self,
|
||||
weight_quant: list[dict[str, Any]] | None,
|
||||
input_quant: dict[str, Any] | None,
|
||||
) -> bool:
|
||||
weight_quant: QuarkQTensorHint,
|
||||
input_quant: QuarkQTensorHint,
|
||||
) -> QuantKeyMatch:
|
||||
# Confirm weights and input quantized.
|
||||
if weight_quant is None or input_quant is None:
|
||||
return False
|
||||
|
||||
if not isinstance(weight_quant, list) or len(weight_quant) != 2:
|
||||
return False
|
||||
if isinstance(input_quant, list):
|
||||
if len(input_quant) != 1:
|
||||
return QuantKeyMatch(False, None, None)
|
||||
input_quant = input_quant[0]
|
||||
if (
|
||||
not isinstance(weight_quant, list)
|
||||
or len(weight_quant) != 2
|
||||
or not isinstance(input_quant, dict)
|
||||
):
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Confirm weight scheme is supported
|
||||
is_w4a8_dtype = (
|
||||
@@ -321,24 +422,26 @@ class QuarkConfig(QuantizationConfig):
|
||||
and is_static_weight
|
||||
and is_per_tensor_fp8_and_per_channel_int4_weight
|
||||
):
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Dynamic quantization is always supported if weights supported.
|
||||
if input_quant.get("is_dynamic"):
|
||||
return True
|
||||
return QuantKeyMatch(True, kFp8DynamicTokenSym, kInt4W4A8StaticChannelSym)
|
||||
|
||||
# Confirm activation scheme is supported.
|
||||
is_per_tensor_activation = input_quant.get("qscheme") == "per_tensor"
|
||||
return is_per_tensor_activation
|
||||
if not is_per_tensor_activation:
|
||||
return QuantKeyMatch(False, None, None)
|
||||
return QuantKeyMatch(True, kFp8StaticTensorSym, kInt4W4A8StaticChannelSym)
|
||||
|
||||
def _is_fp8_w8a8(
|
||||
self,
|
||||
weight_quant: dict[str, Any] | None,
|
||||
input_quant: dict[str, Any] | None,
|
||||
) -> bool:
|
||||
) -> QuantKeyMatch:
|
||||
# Confirm weights and input quantized.
|
||||
if weight_quant is None or input_quant is None:
|
||||
return False
|
||||
if not isinstance(weight_quant, dict) or not isinstance(input_quant, dict):
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Confirm weight scheme is supported
|
||||
is_fp8_dtype = (
|
||||
@@ -358,64 +461,125 @@ class QuarkConfig(QuantizationConfig):
|
||||
)
|
||||
|
||||
if not is_fp8_dtype or not is_static_weight:
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
if is_per_block_weight:
|
||||
if not input_quant.get("is_dynamic"):
|
||||
return False
|
||||
return (
|
||||
return QuantKeyMatch(False, None, None)
|
||||
matches = (
|
||||
input_quant.get("qscheme") == "per_group"
|
||||
and input_quant.get("group_size") == block_size[1]
|
||||
and input_quant.get("symmetric") is True
|
||||
)
|
||||
if not matches:
|
||||
return QuantKeyMatch(False, None, None)
|
||||
weight_quant_key = (
|
||||
kFp8Static128BlockE8M0Sym
|
||||
if weight_quant.get("scale_type") == "float8_e8m0fnu"
|
||||
else kFp8Static128BlockSym
|
||||
)
|
||||
return QuantKeyMatch(True, kFp8Dynamic128Sym, weight_quant_key)
|
||||
|
||||
if not is_per_tensor_or_channel_weight:
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Dynamic quantization is always supported if tensor/channel weights
|
||||
# are supported.
|
||||
if input_quant.get("is_dynamic"):
|
||||
return True
|
||||
activation_quant_key = (
|
||||
kFp8DynamicTokenSym
|
||||
if input_quant.get("qscheme") == "per_channel"
|
||||
else kFp8DynamicTensorSym
|
||||
)
|
||||
weight_quant_key = (
|
||||
kFp8StaticChannelSym
|
||||
if weight_quant.get("qscheme") == "per_channel"
|
||||
else kFp8StaticTensorSym
|
||||
)
|
||||
return QuantKeyMatch(True, activation_quant_key, weight_quant_key)
|
||||
|
||||
# Confirm activation scheme is supported.
|
||||
is_per_tensor_activation = input_quant.get("qscheme") == "per_tensor"
|
||||
return is_per_tensor_activation
|
||||
if not is_per_tensor_activation:
|
||||
return QuantKeyMatch(False, None, None)
|
||||
weight_quant_key = (
|
||||
kFp8StaticChannelSym
|
||||
if weight_quant.get("qscheme") == "per_channel"
|
||||
else kFp8StaticTensorSym
|
||||
)
|
||||
return QuantKeyMatch(True, kFp8StaticTensorSym, weight_quant_key)
|
||||
|
||||
def _is_static_tensor_w8a8(
|
||||
def _is_w8a8_int8(
|
||||
self,
|
||||
weight_quant: dict[str, Any] | None,
|
||||
input_quant: dict[str, Any] | None,
|
||||
) -> bool:
|
||||
) -> QuantKeyMatch:
|
||||
# Confirm weights and input quantized.
|
||||
if weight_quant is None or input_quant is None:
|
||||
return False
|
||||
if not isinstance(weight_quant, dict) or not isinstance(input_quant, dict):
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
is_int8_dtype = (
|
||||
weight_quant.get("dtype") == "int8" and input_quant.get("dtype") == "int8"
|
||||
)
|
||||
|
||||
is_tensor = (
|
||||
weight_quant.get("qscheme") in ["per_tensor", "per_channel"]
|
||||
and input_quant.get("qscheme") == "per_tensor"
|
||||
is_valid_weight_scheme = weight_quant.get("qscheme") in [
|
||||
"per_tensor",
|
||||
"per_channel",
|
||||
]
|
||||
is_static_input = input_quant.get(
|
||||
"qscheme"
|
||||
) == "per_tensor" and not input_quant.get("is_dynamic")
|
||||
is_dynamic_input = (
|
||||
input_quant.get("qscheme") == "per_channel"
|
||||
and input_quant.get("is_dynamic") is True
|
||||
)
|
||||
|
||||
is_static = not weight_quant.get("is_dynamic") and not input_quant.get(
|
||||
"is_dynamic"
|
||||
)
|
||||
|
||||
is_weight_symmetric = weight_quant.get("symmetric") is True
|
||||
|
||||
# Both symmetric and asymmetric input quantization supported.
|
||||
# Only symmetric weight quantization supported.
|
||||
return is_int8_dtype and is_tensor and is_weight_symmetric and is_static
|
||||
matches = (
|
||||
is_int8_dtype
|
||||
and is_valid_weight_scheme
|
||||
and is_weight_symmetric
|
||||
and (
|
||||
(is_static_input and not weight_quant.get("is_dynamic"))
|
||||
or is_dynamic_input
|
||||
)
|
||||
)
|
||||
if not matches:
|
||||
return QuantKeyMatch(False, None, None)
|
||||
weight_quant_key = (
|
||||
kInt8StaticChannelSym
|
||||
if weight_quant.get("qscheme") == "per_channel"
|
||||
else kInt8StaticTensorSym
|
||||
)
|
||||
if is_static_input:
|
||||
activation_quant_key = (
|
||||
kInt8StaticTensorSym
|
||||
if input_quant.get("symmetric") is True
|
||||
else kInt8StaticTensorAsym
|
||||
)
|
||||
elif weight_quant.get("qscheme") == "per_channel":
|
||||
activation_quant_key = (
|
||||
kInt8DynamicTokenSym
|
||||
if input_quant.get("symmetric") is True
|
||||
else kInt8DynamicTokenAsym
|
||||
)
|
||||
else:
|
||||
activation_quant_key = (
|
||||
kInt8DynamicTensorSym
|
||||
if input_quant.get("symmetric") is True
|
||||
else kInt8DynamicTensorAsym
|
||||
)
|
||||
return QuantKeyMatch(True, activation_quant_key, weight_quant_key)
|
||||
|
||||
def _is_w4a8_mxfp4_fp8(
|
||||
self,
|
||||
weight_quant: dict[str, Any] | None,
|
||||
input_quant: dict[str, Any] | None,
|
||||
) -> bool:
|
||||
) -> QuantKeyMatch:
|
||||
if weight_quant is None or input_quant is None:
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
is_weight_mxfp4 = (
|
||||
weight_quant.get("dtype") == "fp4"
|
||||
@@ -432,53 +596,24 @@ class QuarkConfig(QuantizationConfig):
|
||||
and input_quant.get("symmetric") is True # Symmetric quantization
|
||||
)
|
||||
|
||||
return is_weight_mxfp4 and is_input_fp8
|
||||
|
||||
def _is_dynamic_per_token_w8a8(
|
||||
self,
|
||||
weight_quant: dict[str, Any] | None,
|
||||
input_quant: dict[str, Any] | None,
|
||||
) -> bool:
|
||||
"""Detect W8A8 INT8 with per-tensor or per-channel
|
||||
weights and dynamic per-token input."""
|
||||
if weight_quant is None or input_quant is None:
|
||||
return False
|
||||
|
||||
is_int8_dtype = (
|
||||
weight_quant.get("dtype") == "int8" and input_quant.get("dtype") == "int8"
|
||||
)
|
||||
|
||||
is_valid_weight_scheme = weight_quant.get("qscheme") in [
|
||||
"per_tensor",
|
||||
"per_channel",
|
||||
]
|
||||
is_per_token_input = input_quant.get("qscheme") == "per_channel"
|
||||
|
||||
is_dynamic_input = input_quant.get("is_dynamic") is True
|
||||
is_weight_symmetric = weight_quant.get("symmetric") is True
|
||||
|
||||
return (
|
||||
is_int8_dtype
|
||||
and is_valid_weight_scheme
|
||||
and is_per_token_input
|
||||
and is_dynamic_input
|
||||
and is_weight_symmetric
|
||||
)
|
||||
if not (is_weight_mxfp4 and is_input_fp8):
|
||||
return QuantKeyMatch(False, None, None)
|
||||
return QuantKeyMatch(True, kFp8StaticTensorSym, kMxfp4Static)
|
||||
|
||||
def _is_nvfp4(
|
||||
self,
|
||||
weight_quant: dict[str, Any] | list[dict[str, Any]] | None,
|
||||
input_quant: dict[str, Any] | list[dict[str, Any]] | None,
|
||||
) -> bool:
|
||||
weight_quant: QuarkQTensorHint,
|
||||
input_quant: QuarkQTensorHint,
|
||||
) -> QuantKeyMatch:
|
||||
# Confirm weights and input quantized.
|
||||
if weight_quant is None or input_quant is None:
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Confirm both weight_quant and input_quant are lists with 2 elements
|
||||
if not isinstance(weight_quant, list) or len(weight_quant) != 2:
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
if not isinstance(input_quant, list) or len(input_quant) != 2:
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# First element should be fp4 with per_group quantization
|
||||
is_fp4_per_group_weight = (
|
||||
@@ -506,36 +641,35 @@ class QuarkConfig(QuantizationConfig):
|
||||
and not input_quant[1].get("is_dynamic")
|
||||
)
|
||||
|
||||
return (
|
||||
matches = (
|
||||
is_fp4_per_group_weight # type: ignore[return-value]
|
||||
and is_fp4_per_group_input
|
||||
and is_fp8_per_tensor_weight
|
||||
and is_fp8_per_tensor_input
|
||||
)
|
||||
if not matches:
|
||||
return QuantKeyMatch(False, None, None)
|
||||
return QuantKeyMatch(True, kNvfp4Dynamic, kNvfp4Static)
|
||||
|
||||
def _is_w_ocp_mx_a_x(
|
||||
self, weight_quant: dict[str, Any] | None, input_quant: dict[str, Any] | None
|
||||
) -> bool:
|
||||
self,
|
||||
weight_quant: dict[str, Any] | None,
|
||||
input_quant: dict[str, Any] | None,
|
||||
allow_static_fp8: bool = False,
|
||||
) -> QuantKeyMatch:
|
||||
"""
|
||||
This check returns True only if it is an OCP-MX weight quantization.
|
||||
The activation can be any data type (e.g., FP16/BF16, FP8, or OCP-MX format).
|
||||
The activation can be FP16/BF16, OCP MXFP4, MXFP8, FP8.
|
||||
The rationale for checking only the weight type is that
|
||||
the model loading concept and process primarily concerns the weights themselves.
|
||||
"""
|
||||
# Confirm weights quantized.
|
||||
if weight_quant is None:
|
||||
if not isinstance(weight_quant, dict):
|
||||
logger.debug(
|
||||
"Quark model's weight quantization is incompatible with OCP_MX format: "
|
||||
"weight_quant is not set."
|
||||
"weight_quant is not a dictionary."
|
||||
)
|
||||
return False
|
||||
|
||||
if isinstance(weight_quant, list):
|
||||
logger.debug(
|
||||
"Quark model's weight quantization is incompatible with OCP_MX format: "
|
||||
"weight_quant is a list (e.g. fp8_w4a8), OCP_MX requires a single dict."
|
||||
)
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Input and weight qscheme needs to be per group.
|
||||
if weight_quant.get("qscheme") != "per_group":
|
||||
@@ -543,7 +677,7 @@ class QuarkConfig(QuantizationConfig):
|
||||
"Quark model's weight quantization is incompatible with OCP MX format: "
|
||||
"weight is not per_group."
|
||||
)
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Input and weight group size needs to be 32.
|
||||
if weight_quant.get("group_size") != 32:
|
||||
@@ -551,7 +685,7 @@ class QuarkConfig(QuantizationConfig):
|
||||
"Quark model's weight quantization is incompatible with OCP MX format: "
|
||||
"group_size of weight is not 32."
|
||||
)
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Activations and weight scales need to be in e8m0 format.
|
||||
if weight_quant.get("scale_format") != "e8m0":
|
||||
@@ -559,7 +693,7 @@ class QuarkConfig(QuantizationConfig):
|
||||
"Quark model's weight quantization is incompatible with OCP MX format: "
|
||||
"scale_format of weight is not e8m0."
|
||||
)
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Input and weight dtypes need to be any of fp4,
|
||||
# fp6_e3m2 or fp6_e3m2, possibly mixed.
|
||||
@@ -572,9 +706,48 @@ class QuarkConfig(QuantizationConfig):
|
||||
"Quark model's weight quantization is incompatible with OCP MX format: "
|
||||
"dtype is not in {fp4, fp6_e3m2, fp6_e2m3}."
|
||||
)
|
||||
return False
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
return True
|
||||
weight_dtype = weight_quant["dtype"].replace("fp", "mxfp")
|
||||
weight_quant_key = _WEIGHT_QUANT_KEY_MAP[weight_dtype]
|
||||
if input_quant is None:
|
||||
activation_quant_key = None
|
||||
elif not input_quant.get("is_dynamic"):
|
||||
if (
|
||||
allow_static_fp8
|
||||
and input_quant.get("dtype") == "fp8_e4m3"
|
||||
and input_quant.get("qscheme") == "per_tensor"
|
||||
and input_quant.get("symmetric") is True
|
||||
):
|
||||
activation_quant_key = kFp8StaticTensorSym
|
||||
else:
|
||||
logger.debug(
|
||||
"Quark model's OCP MX quantization is incompatible with static "
|
||||
"input scales."
|
||||
)
|
||||
return QuantKeyMatch(False, None, None)
|
||||
elif input_quant["dtype"] == "fp8_e4m3":
|
||||
activation_quant_key = kFp8DynamicTensorSym
|
||||
else:
|
||||
input_dtype = input_quant["dtype"].replace("fp", "mxfp")
|
||||
if input_dtype not in _ACTIVATION_QUANT_KEY_MAP:
|
||||
raise ValueError(
|
||||
f"Unsupported input_dtype={input_dtype} in Quark's vLLM "
|
||||
"integration. Supported activation dtypes are "
|
||||
f"{_ACTIVATION_QUANT_KEY_MAP.keys()}, or None for "
|
||||
"weight-only quantization."
|
||||
)
|
||||
activation_quant_key = _ACTIVATION_QUANT_KEY_MAP[input_dtype]
|
||||
return QuantKeyMatch(True, activation_quant_key, weight_quant_key)
|
||||
|
||||
@staticmethod
|
||||
def _unwrap_single_quant_config(
|
||||
quant_config: QuarkQTensorHint,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Unwrap one-entry quantization-config lists."""
|
||||
if isinstance(quant_config, list):
|
||||
return quant_config[0] if len(quant_config) == 1 else None
|
||||
return quant_config
|
||||
|
||||
def get_layer_quant_config_from_name(
|
||||
self, layer_name: str
|
||||
@@ -617,14 +790,14 @@ class QuarkConfig(QuantizationConfig):
|
||||
return None
|
||||
|
||||
def _find_matched_config(
|
||||
self, layer_name: str, module: torch.nn.Module
|
||||
self, layer_name: str, module: torch.nn.Module | type[torch.nn.Module]
|
||||
) -> dict[str, Any]:
|
||||
# Priority order:
|
||||
# 1. layer_quant_config,
|
||||
# 2. layer_type_quant_config,
|
||||
# 3. global_quant_config.
|
||||
|
||||
layer_type = cast(str, type(module))
|
||||
layer_type = cast(str, module if isinstance(module, type) else type(module))
|
||||
layer_type_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("layer_type_quant_config")
|
||||
)
|
||||
@@ -662,51 +835,71 @@ class QuarkConfig(QuantizationConfig):
|
||||
return layer_quant_config
|
||||
return fallback_config
|
||||
|
||||
def _get_scheme_from_config(
|
||||
self, config: dict[str, Any], dynamic_mxfp4_quant: bool = False
|
||||
) -> "QuarkScheme":
|
||||
def _get_scheme_cls_from_config(
|
||||
self, config: dict[str, Any]
|
||||
) -> tuple[QuantKey | None, QuantKey | None, type["QuarkScheme"]]:
|
||||
if config.get("output_tensors") or config.get("bias"):
|
||||
raise NotImplementedError(
|
||||
"Currently, Quark models with output_tensors "
|
||||
"and bias quantized are not supported"
|
||||
)
|
||||
weight_config = cast(dict[str, Any], config.get("weight"))
|
||||
input_config = cast(dict[str, Any], config.get("input_tensors"))
|
||||
weight_config = cast(QuarkQTensorHint, config.get("weight"))
|
||||
input_config = cast(QuarkQTensorHint, config.get("input_tensors"))
|
||||
|
||||
if self._is_nvfp4(weight_config, input_config):
|
||||
return QuarkNVFP4()
|
||||
elif self._is_fp8_w8a8(weight_config, input_config):
|
||||
is_fp8_w8a8_supported = self._check_scheme_supported(
|
||||
QuarkW8A8Fp8.get_min_capability(), error=False
|
||||
if (
|
||||
isinstance(weight_config, list)
|
||||
and len(weight_config) > 1
|
||||
or isinstance(input_config, list)
|
||||
and len(input_config) > 1
|
||||
):
|
||||
if match := self._is_nvfp4(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkNVFP4,
|
||||
)
|
||||
|
||||
raise NotImplementedError(
|
||||
"Multi-entry weight or activation quantization configs are only "
|
||||
"supported for NVFP4."
|
||||
)
|
||||
if is_fp8_w8a8_supported:
|
||||
if weight_config.get("qscheme") == "per_block":
|
||||
return QuarkW8A8Fp8PerBlock(weight_config, input_config)
|
||||
return QuarkW8A8Fp8(weight_config, input_config)
|
||||
elif self._is_static_tensor_w8a8(weight_config, input_config):
|
||||
weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
return QuarkW8A8Int8(
|
||||
qscheme=weight_qscheme,
|
||||
is_static_input_scheme=True,
|
||||
input_symmetric=input_config.get("symmetric"),
|
||||
|
||||
weight_config = self._unwrap_single_quant_config(weight_config)
|
||||
input_config = self._unwrap_single_quant_config(input_config)
|
||||
|
||||
if (
|
||||
isinstance(weight_config, dict)
|
||||
and weight_config.get("qscheme") == "per_block"
|
||||
and not weight_config.get("block_size")
|
||||
):
|
||||
raise ValueError(
|
||||
"Quark W8A8 FP8 per-block weight quantization requires "
|
||||
"`block_size` in the weight quantization config."
|
||||
)
|
||||
elif self._is_w4a8_mxfp4_fp8(weight_config, input_config):
|
||||
is_w4a8_supported = self._check_scheme_supported(
|
||||
QuarkW4A8_MXFP4_FP8.get_min_capability(), error=False
|
||||
|
||||
if match := self._is_fp8_w8a8(weight_config, input_config):
|
||||
assert weight_config is not None
|
||||
if weight_config.get("qscheme") == "per_block":
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
)
|
||||
return match.weight_quant_key, match.activation_quant_key, QuarkW8A8Fp8
|
||||
elif match := self._is_w8a8_int8(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkW8A8Int8,
|
||||
)
|
||||
if is_w4a8_supported:
|
||||
return QuarkW4A8_MXFP4_FP8(weight_config, input_config)
|
||||
elif self._is_dynamic_per_token_w8a8(weight_config, input_config):
|
||||
weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
return QuarkW8A8Int8(
|
||||
qscheme=weight_qscheme,
|
||||
is_static_input_scheme=False,
|
||||
input_symmetric=input_config.get("symmetric"),
|
||||
)
|
||||
elif self._is_w_ocp_mx_a_x(weight_config, input_config):
|
||||
return QuarkOCP_MX(
|
||||
weight_config, input_config, dynamic_mxfp4_quant=dynamic_mxfp4_quant
|
||||
elif match := self._is_w4a8_mxfp4_fp8(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkW4A8_MXFP4_FP8,
|
||||
)
|
||||
elif match := self._is_w_ocp_mx_a_x(weight_config, input_config):
|
||||
return match.weight_quant_key, match.activation_quant_key, QuarkOCP_MX
|
||||
|
||||
raise NotImplementedError(
|
||||
"No quark compatible scheme was found. "
|
||||
@@ -714,19 +907,51 @@ class QuarkConfig(QuantizationConfig):
|
||||
f"Input config: {input_config}"
|
||||
)
|
||||
|
||||
def get_scheme(
|
||||
self, layer: torch.nn.Module, layer_name: str, dynamic_mxfp4_quant: bool = False
|
||||
) -> "QuarkScheme":
|
||||
layer_quant_config = self._find_matched_config(layer_name, layer)
|
||||
|
||||
# Find the quant_scheme
|
||||
scheme = self._get_scheme_from_config(
|
||||
layer_quant_config, dynamic_mxfp4_quant=dynamic_mxfp4_quant
|
||||
def get_scheme_cls(
|
||||
self, layer_type: type[torch.nn.Module], layer_name: str
|
||||
) -> tuple[QuantKey | None, QuantKey | None, type["QuarkScheme"]]:
|
||||
"""Return quantization keys and scheme class without initializing it."""
|
||||
return self._get_scheme_cls_from_config(
|
||||
self._find_matched_config(layer_name, layer_type)
|
||||
)
|
||||
|
||||
def init_scheme(
|
||||
self,
|
||||
scheme_cls: type["QuarkScheme"],
|
||||
weight_quant_key: QuantKey | None,
|
||||
activation_quant_key: QuantKey | None,
|
||||
dynamic_mxfp4_quant: bool = False,
|
||||
) -> "QuarkScheme":
|
||||
"""Construct a Quark scheme selected by get_scheme_cls."""
|
||||
if scheme_cls not in (
|
||||
QuarkW8A8Fp8,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
QuarkW8A8Int8,
|
||||
QuarkOCP_MX,
|
||||
QuarkNVFP4,
|
||||
QuarkW4A8_MXFP4_FP8,
|
||||
):
|
||||
raise AssertionError(f"Unsupported Quark scheme class: {scheme_cls}")
|
||||
|
||||
kwargs: dict[str, Any] = {"activation_quant_key": activation_quant_key}
|
||||
|
||||
# These classes handle different possible `weight_quant_key`.
|
||||
if scheme_cls in (
|
||||
QuarkW8A8Fp8,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
QuarkW8A8Int8,
|
||||
QuarkOCP_MX,
|
||||
):
|
||||
kwargs["weight_quant_key"] = weight_quant_key
|
||||
|
||||
if scheme_cls is QuarkOCP_MX:
|
||||
kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant
|
||||
|
||||
scheme = scheme_cls(**kwargs)
|
||||
|
||||
# Raise error if device does not support the scheme
|
||||
# (e.g. fp8 needs ada lovelace)
|
||||
self._check_scheme_supported(scheme.get_min_capability())
|
||||
|
||||
return scheme
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
@@ -56,19 +56,27 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
make_nvfp4_moe_quant_config,
|
||||
select_nvfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
OCP_MX_BLOCK_SIZE,
|
||||
OCP_MX_Scheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4W4A8StaticChannelSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8StaticTensorSym,
|
||||
kMxfp4Dynamic,
|
||||
kNvfp4Dynamic,
|
||||
@@ -82,6 +90,9 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
@@ -93,67 +104,154 @@ __all__ = [
|
||||
|
||||
|
||||
class QuarkMoEMethod(FusedMoEMethodBase):
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
supported_activation_quant_keys: list[QuantKey | None] = []
|
||||
supported_weight_quant_keys: list[QuantKey] = []
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe: FusedMoEConfig,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
if activation_quant_key not in self.supported_activation_quant_keys:
|
||||
raise ValueError(
|
||||
f"Unsupported activation quant key: {activation_quant_key}"
|
||||
)
|
||||
if weight_quant_key not in self.supported_weight_quant_keys:
|
||||
raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
|
||||
self.activation_quant_key = activation_quant_key
|
||||
self.weight_quant_key = weight_quant_key
|
||||
self.has_bias = self.moe.has_bias
|
||||
|
||||
@staticmethod
|
||||
def get_moe_method(
|
||||
quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821
|
||||
module: RoutedExperts,
|
||||
def get_moe_method_target(
|
||||
quant_config: "QuarkConfig",
|
||||
layer_type: type[torch.nn.Module],
|
||||
layer_name: str,
|
||||
) -> "QuarkMoEMethod":
|
||||
layer_quant_config = quant_config._find_matched_config(layer_name, module)
|
||||
|
||||
) -> tuple[
|
||||
QuantKey | None,
|
||||
QuantKey | None,
|
||||
type["QuarkMoEMethod"] | None,
|
||||
]:
|
||||
"""Return the quantization target for a routed-experts layer."""
|
||||
layer_quant_config = quant_config._find_matched_config(layer_name, layer_type)
|
||||
if layer_quant_config.get("output_tensors") or layer_quant_config.get("bias"):
|
||||
raise NotImplementedError(
|
||||
"Currently, Quark models with "
|
||||
"output_tensors and bias "
|
||||
"quantized are not supported"
|
||||
)
|
||||
weight_config: QuarkQTensorHint = layer_quant_config.get("weight")
|
||||
input_config: QuarkQTensorHint = layer_quant_config.get("input_tensors")
|
||||
|
||||
weight_config = layer_quant_config.get("weight")
|
||||
input_config = layer_quant_config.get("input_tensors")
|
||||
if (
|
||||
isinstance(weight_config, list)
|
||||
and len(weight_config) > 1
|
||||
or isinstance(input_config, list)
|
||||
and len(input_config) > 1
|
||||
):
|
||||
if match := quant_config._is_fp8_w4a8(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkW4A8Fp8MoEMethod,
|
||||
)
|
||||
if match := quant_config._is_nvfp4(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkNvfp4MoEMethod,
|
||||
)
|
||||
raise RuntimeError("Unsupported FusedMoe scheme")
|
||||
|
||||
if quant_config._is_fp8_w4a8(weight_config, input_config):
|
||||
return QuarkW4A8Fp8MoEMethod(weight_config, input_config, module.moe_config)
|
||||
elif quant_config._is_nvfp4(weight_config, input_config):
|
||||
return QuarkNvfp4MoEMethod(
|
||||
weight_config, input_config, module.moe_config, quant_config
|
||||
weight_config = quant_config._unwrap_single_quant_config(weight_config)
|
||||
input_config = quant_config._unwrap_single_quant_config(input_config)
|
||||
|
||||
if match := quant_config._is_fp8_w8a8(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkW8A8Fp8MoEMethod,
|
||||
)
|
||||
elif quant_config._is_fp8_w8a8(weight_config, input_config):
|
||||
return QuarkW8A8Fp8MoEMethod(weight_config, input_config, module.moe_config)
|
||||
elif quant_config._is_w_ocp_mx_a_x(weight_config, input_config):
|
||||
if match := quant_config._is_w_ocp_mx_a_x(
|
||||
weight_config, input_config, allow_static_fp8=True
|
||||
):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkOCP_MX_MoEMethod,
|
||||
)
|
||||
if match := quant_config._is_w8a8_int8(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkW8A8Int8MoEMethod,
|
||||
)
|
||||
raise RuntimeError("Unsupported FusedMoe scheme")
|
||||
|
||||
@staticmethod
|
||||
def get_moe_method(
|
||||
quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821
|
||||
module: RoutedExperts,
|
||||
method_cls: type["QuarkMoEMethod"],
|
||||
weight_quant_key: QuantKey | None,
|
||||
activation_quant_key: QuantKey | None,
|
||||
) -> "QuarkMoEMethod":
|
||||
if weight_quant_key is None:
|
||||
raise RuntimeError("Unsupported FusedMoe scheme")
|
||||
|
||||
if method_cls is QuarkW4A8Fp8MoEMethod:
|
||||
return QuarkW4A8Fp8MoEMethod(module.moe_config, activation_quant_key)
|
||||
elif method_cls is QuarkNvfp4MoEMethod:
|
||||
return QuarkNvfp4MoEMethod(
|
||||
module.moe_config, quant_config, activation_quant_key
|
||||
)
|
||||
elif method_cls is QuarkW8A8Fp8MoEMethod:
|
||||
return QuarkW8A8Fp8MoEMethod(
|
||||
module.moe_config, weight_quant_key, activation_quant_key
|
||||
)
|
||||
elif method_cls is QuarkOCP_MX_MoEMethod:
|
||||
# All OCP MX schemes (W4A16, W4A8, etc.) handled by QuarkOCP_MX_MoEMethod
|
||||
# Backend selection happens inside via oracle
|
||||
return QuarkOCP_MX_MoEMethod(weight_config, input_config, module.moe_config)
|
||||
elif quant_config._is_static_tensor_w8a8(
|
||||
weight_config, input_config
|
||||
) or quant_config._is_dynamic_per_token_w8a8(weight_config, input_config):
|
||||
return QuarkOCP_MX_MoEMethod(
|
||||
module.moe_config, weight_quant_key, activation_quant_key
|
||||
)
|
||||
elif method_cls is QuarkW8A8Int8MoEMethod:
|
||||
return QuarkW8A8Int8MoEMethod(
|
||||
weight_config, input_config, module.moe_config
|
||||
module.moe_config, weight_quant_key, activation_quant_key
|
||||
)
|
||||
else:
|
||||
raise RuntimeError("Unsupported FusedMoe scheme")
|
||||
|
||||
|
||||
class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
supported_activation_quant_keys = [
|
||||
kFp8StaticTensorSym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
]
|
||||
supported_weight_quant_keys = [
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_config: dict[str, Any],
|
||||
input_config: dict[str, Any],
|
||||
moe: FusedMoEConfig,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.weight_quant = weight_config
|
||||
self.input_quant = input_config
|
||||
|
||||
self.weight_qscheme = self.weight_quant.get("qscheme")
|
||||
self.input_qscheme = self.input_quant.get("qscheme")
|
||||
self.weight_dtype = self.weight_quant.get("dtype", "").replace(
|
||||
"fp8_e4m3", "fp8"
|
||||
super().__init__(moe, weight_quant_key, activation_quant_key)
|
||||
self.weight_qscheme = (
|
||||
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
|
||||
)
|
||||
self.input_dtype = self.input_quant.get("dtype", "").replace("fp8_e4m3", "fp8")
|
||||
self.input_qscheme = (
|
||||
"per_channel"
|
||||
if activation_quant_key == kFp8DynamicTokenSym
|
||||
else "per_tensor"
|
||||
)
|
||||
self.static_input_scales = activation_quant_key == kFp8StaticTensorSym
|
||||
per_tensor = (
|
||||
self.weight_qscheme == "per_tensor" and self.input_qscheme == "per_tensor"
|
||||
)
|
||||
@@ -170,28 +268,10 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
f"{self.weight_qscheme}, {self.input_qscheme}"
|
||||
) # noqa E501
|
||||
|
||||
self.static_input_scales = not self.input_quant.get("is_dynamic")
|
||||
if self.static_input_scales and per_channel:
|
||||
raise ValueError(
|
||||
"For FP8 Fused MoE layer, we require either per tensor or "
|
||||
"channelwise, dynamic per token quantization."
|
||||
)
|
||||
|
||||
# Determine quant keys for oracle backend selection
|
||||
if per_channel:
|
||||
weight_key = kFp8StaticChannelSym
|
||||
activation_key = kFp8DynamicTokenSym
|
||||
elif self.static_input_scales:
|
||||
weight_key = kFp8StaticTensorSym
|
||||
activation_key = kFp8StaticTensorSym
|
||||
else:
|
||||
weight_key = kFp8StaticTensorSym
|
||||
activation_key = kFp8DynamicTensorSym
|
||||
|
||||
self.fp8_backend, self.experts_cls = select_fp8_moe_backend(
|
||||
config=moe,
|
||||
weight_key=weight_key,
|
||||
activation_key=activation_key,
|
||||
weight_key=self.weight_quant_key,
|
||||
activation_key=self.activation_quant_key,
|
||||
)
|
||||
|
||||
self.model_type = getattr(
|
||||
@@ -500,17 +580,41 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
"""Quark W8A8 INT8 MoE method."""
|
||||
|
||||
supported_activation_quant_keys = [
|
||||
kInt8StaticTensorSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
]
|
||||
supported_weight_quant_keys = [
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorSym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_config: dict[str, Any],
|
||||
input_config: dict[str, Any],
|
||||
moe: FusedMoEConfig,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.weight_quant = weight_config
|
||||
self.input_quant = input_config
|
||||
self.weight_qscheme = self.weight_quant.get("qscheme", "per_tensor")
|
||||
self.static_input_scales = not self.input_quant.get("is_dynamic", False)
|
||||
super().__init__(moe, weight_quant_key, activation_quant_key)
|
||||
|
||||
# TODO: oracle needs to support asym int8.
|
||||
if self.activation_quant_key == kInt8StaticTensorAsym:
|
||||
self.activation_quant_key = kInt8StaticTensorSym
|
||||
if self.activation_quant_key == kInt8DynamicTensorAsym:
|
||||
self.activation_quant_key = kInt8DynamicTensorSym
|
||||
if self.activation_quant_key == kInt8DynamicTokenAsym:
|
||||
self.activation_quant_key = kInt8DynamicTokenSym
|
||||
|
||||
self.weight_qscheme = (
|
||||
"per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
|
||||
)
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
self.static_input_scales = self.activation_quant_key.scale.static
|
||||
|
||||
self.moe_quant_config: FusedMoEQuantConfig | None = None
|
||||
self.moe_kernel: mk.FusedMoEKernel | None = None
|
||||
@@ -528,17 +632,10 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
# Map the Quark weight scheme to oracle quant keys. Per-channel
|
||||
# weights pair with dynamic per-token activations; per-tensor
|
||||
# weights with dynamic per-tensor activations.
|
||||
if self.weight_qscheme == "per_channel":
|
||||
weight_key = kInt8StaticChannelSym
|
||||
activation_key = kInt8DynamicTokenSym
|
||||
else:
|
||||
weight_key = kInt8StaticTensorSym
|
||||
activation_key = kInt8DynamicTensorSym
|
||||
|
||||
self.int8_backend, self.experts_cls = select_int8_moe_backend(
|
||||
config=moe,
|
||||
weight_key=weight_key,
|
||||
activation_key=activation_key,
|
||||
weight_key=self.weight_quant_key,
|
||||
activation_key=self.activation_quant_key,
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
@@ -876,15 +973,18 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
|
||||
|
||||
class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
supported_activation_quant_keys = [
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticTensorSym,
|
||||
]
|
||||
supported_weight_quant_keys = [kInt4W4A8StaticChannelSym]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_config: dict[str, Any],
|
||||
input_config: dict[str, Any],
|
||||
moe: FusedMoEConfig,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.weight_quant = weight_config
|
||||
self.input_quant = input_config
|
||||
super().__init__(moe, kInt4W4A8StaticChannelSym, activation_quant_key)
|
||||
|
||||
assert rocm_aiter_ops.is_fused_moe_enabled(), (
|
||||
"W4A8 FP8 MoE requires ROCm AITER fused MoE support."
|
||||
@@ -1032,37 +1132,36 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
|
||||
|
||||
class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
supported_activation_quant_keys = [
|
||||
*_ACTIVATION_QUANT_KEY_MAP.values(),
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8StaticTensorSym,
|
||||
None,
|
||||
]
|
||||
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_config: dict[str, Any],
|
||||
input_config: dict[str, Any] | None,
|
||||
moe: FusedMoEConfig,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.weight_quant = weight_config
|
||||
self.input_quant = input_config
|
||||
|
||||
weight_qscheme = self.weight_quant.get("qscheme")
|
||||
if not weight_qscheme == "per_group":
|
||||
raise ValueError(
|
||||
"For MX(FP4) Fused MoE layers, only per-group scales "
|
||||
f"for weights are supported. Found {weight_qscheme}."
|
||||
) # noqa E501
|
||||
|
||||
self.weight_dtype = self.weight_quant["dtype"].replace("fp", "mxfp")
|
||||
if self.input_quant is not None:
|
||||
input_quant = self.input_quant["dtype"]
|
||||
if input_quant in ["fp4", "fp6_e3m2", "fp6_e2m3"]:
|
||||
self.input_dtype = input_quant.replace("fp", "mxfp")
|
||||
elif input_quant == "fp8_e4m3":
|
||||
self.input_dtype = input_quant.replace("fp8_e4m3", "fp8")
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Current input dtype {input_quant} is not compatible \
|
||||
with OCP MX (weight) MoE quantization. Please open an issue"
|
||||
)
|
||||
else:
|
||||
super().__init__(moe, weight_quant_key, activation_quant_key)
|
||||
self.weight_dtype = next(
|
||||
dtype
|
||||
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
|
||||
if quant_key == weight_quant_key
|
||||
)
|
||||
if activation_quant_key in {kFp8DynamicTensorSym, kFp8StaticTensorSym}:
|
||||
self.input_dtype: str | None = "fp8"
|
||||
elif activation_quant_key is None:
|
||||
self.input_dtype = None
|
||||
else:
|
||||
self.input_dtype = next(
|
||||
dtype
|
||||
for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
|
||||
if quant_key == activation_quant_key
|
||||
)
|
||||
|
||||
self.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(
|
||||
self.input_dtype, self.weight_dtype
|
||||
@@ -1085,10 +1184,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
self.w13_precision_config = None
|
||||
self.w2_precision_config = None
|
||||
|
||||
if self.input_quant is not None:
|
||||
self.static_input_scales = not self.input_quant.get("is_dynamic")
|
||||
else:
|
||||
self.static_input_scales = False
|
||||
self.static_input_scales = activation_quant_key == kFp8StaticTensorSym
|
||||
|
||||
# Select backend based on OCP MX scheme
|
||||
if self.ocp_mx_scheme == "w_mxfp4":
|
||||
@@ -1388,6 +1484,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
f"in vLLM for OCP MX scheme {self.ocp_mx_scheme}. Please open an issue."
|
||||
)
|
||||
else:
|
||||
assert self.input_dtype is not None
|
||||
return ocp_mx_moe_quant_config(
|
||||
quant_dtype=self.input_dtype,
|
||||
weight_dtype=self.weight_dtype,
|
||||
@@ -1459,24 +1556,24 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
|
||||
|
||||
class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
supported_activation_quant_keys = [kNvfp4Dynamic]
|
||||
supported_weight_quant_keys = [kNvfp4Static]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_config: dict[str, Any],
|
||||
input_config: dict[str, Any],
|
||||
moe: FusedMoEConfig,
|
||||
quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.weight_quant = weight_config
|
||||
self.input_quant = input_config
|
||||
super().__init__(moe, kNvfp4Static, activation_quant_key)
|
||||
self.quant_config = quant_config
|
||||
self.group_size = 16
|
||||
|
||||
# Select experts implementation.
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=kNvfp4Dynamic,
|
||||
weight_key=self.weight_quant_key,
|
||||
activation_key=self.activation_quant_key,
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
|
||||
@@ -14,6 +14,11 @@ from vllm.model_executor.kernels.linear.nvfp4.emulation import (
|
||||
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
|
||||
QuarkScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
@@ -36,9 +41,14 @@ class QuarkNVFP4(QuarkScheme):
|
||||
- input_scale_2: bfloat16/float32, scalar (global input scale)
|
||||
"""
|
||||
|
||||
supported_activation_quant_keys = [kNvfp4Dynamic]
|
||||
supported_weight_quant_keys = [kNvfp4Static]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(kNvfp4Static, activation_quant_key)
|
||||
self.kernel = init_nvfp4_linear_kernel()
|
||||
self.group_size = 16
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -15,15 +14,14 @@ from vllm.model_executor.kernels.linear import (
|
||||
init_mxfp6_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
OCP_MX_BLOCK_SIZE,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
@@ -38,45 +36,32 @@ from .quark_scheme import QuarkScheme
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Static,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Static,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Static,
|
||||
}
|
||||
|
||||
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Dynamic,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
|
||||
}
|
||||
|
||||
|
||||
class QuarkOCP_MX(QuarkScheme):
|
||||
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
|
||||
supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None]
|
||||
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_spec: dict[str, Any],
|
||||
input_quant_spec: dict[str, Any] | None,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
dynamic_mxfp4_quant: bool = False,
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.dynamic_mxfp4_quant = dynamic_mxfp4_quant
|
||||
self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp")
|
||||
self.input_dtype: str | None = None
|
||||
if input_quant_spec is not None:
|
||||
self.input_dtype = input_quant_spec["dtype"].replace("fp", "mxfp")
|
||||
|
||||
if self.input_dtype not in [None, *_ACTIVATION_QUANT_KEY_MAP]:
|
||||
raise ValueError(
|
||||
f"Unsupported input_dtype={self.input_dtype} for QuarkOCP_MX. "
|
||||
f"Supported activation dtypes are {_ACTIVATION_QUANT_KEY_MAP.keys()}, "
|
||||
"or None for weight-only quantization."
|
||||
self.weight_dtype = next(
|
||||
dtype
|
||||
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
|
||||
if quant_key == weight_quant_key
|
||||
)
|
||||
self.input_dtype = (
|
||||
next(
|
||||
dtype
|
||||
for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
|
||||
if quant_key == activation_quant_key
|
||||
)
|
||||
|
||||
self.weight_quant_key = _WEIGHT_QUANT_KEY_MAP[self.weight_dtype]
|
||||
self.activation_quant_key = (
|
||||
_ACTIVATION_QUANT_KEY_MAP[self.input_dtype]
|
||||
if self.input_dtype is not None
|
||||
if activation_quant_key is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -85,17 +70,6 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
else:
|
||||
self.packed_factor = Fraction(numerator=8, denominator=6)
|
||||
|
||||
if input_quant_spec is None:
|
||||
self.static_input_scales = False
|
||||
else:
|
||||
self.static_input_scales = not input_quant_spec.get("is_dynamic")
|
||||
|
||||
if self.static_input_scales:
|
||||
raise NotImplementedError(
|
||||
"QuarkOCP_MX with static input scales is currently not "
|
||||
"implemented. Please open an issue."
|
||||
)
|
||||
|
||||
if not current_platform.supports_mx():
|
||||
logger.warning_once(
|
||||
"The current platform does not support native MXFP4/MXFP6 "
|
||||
|
||||
@@ -5,6 +5,8 @@ from abc import ABC, abstractmethod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
__all__ = ["QuarkScheme"]
|
||||
|
||||
|
||||
@@ -14,6 +16,23 @@ class QuarkScheme(ABC):
|
||||
of different quantization schemes supported by Quark.
|
||||
"""
|
||||
|
||||
supported_activation_quant_keys: list[QuantKey | None] = []
|
||||
supported_weight_quant_keys: list[QuantKey] = []
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
if activation_quant_key not in self.supported_activation_quant_keys:
|
||||
raise ValueError(
|
||||
f"Unsupported activation quant key: {activation_quant_key}"
|
||||
)
|
||||
if weight_quant_key not in self.supported_weight_quant_keys:
|
||||
raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
|
||||
self.activation_quant_key = activation_quant_key
|
||||
self.weight_quant_key = weight_quant_key
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -11,7 +10,10 @@ import torch.nn.functional as F
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
get_fp8_min_max,
|
||||
kFp8StaticTensorSym,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
@@ -38,39 +40,28 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
|
||||
Uses the AITER Triton kernel and falls back to emulation if AITER not available.
|
||||
"""
|
||||
|
||||
supported_activation_quant_keys = [kFp8StaticTensorSym]
|
||||
supported_weight_quant_keys = [kMxfp4Static]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_spec: dict[str, Any],
|
||||
input_quant_spec: dict[str, Any],
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(kMxfp4Static, activation_quant_key)
|
||||
self.out_dtype = None
|
||||
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.packed_factor: Fraction = Fraction(2, 1) # 2 FP4 values per byte
|
||||
self.weight_block_size = OCP_MX_BLOCK_SIZE
|
||||
|
||||
self.is_static_input_scheme = not input_quant_spec.get("is_dynamic")
|
||||
self.input_qscheme = input_quant_spec.get("qscheme") # "per_tensor"
|
||||
|
||||
self.fp8_min, self.fp8_max = get_fp8_min_max()
|
||||
self.fp8_dtype = current_platform.fp8_dtype()
|
||||
|
||||
if not self.is_static_input_scheme:
|
||||
raise NotImplementedError(
|
||||
"Dynamic FP8 activation quantization is not yet supported "
|
||||
"for W4A8. The current implementation expects static per-tensor "
|
||||
"FP8 scales stored in the checkpoint."
|
||||
)
|
||||
|
||||
kernel_supported_gpu = False
|
||||
if current_platform.is_rocm():
|
||||
kernel_supported_gpu = current_platform.supports_mx()
|
||||
|
||||
self.use_aiter_kernel = (
|
||||
is_aiter_found_and_supported()
|
||||
and self.is_static_input_scheme
|
||||
and kernel_supported_gpu
|
||||
)
|
||||
self.use_aiter_kernel = is_aiter_found_and_supported() and kernel_supported_gpu
|
||||
|
||||
if not self.use_aiter_kernel:
|
||||
logger.warning_once(
|
||||
@@ -128,17 +119,16 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
# INPUT SCALE (FP8 per-tensor static scale)
|
||||
if self.is_static_input_scheme:
|
||||
input_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(
|
||||
len(output_partition_sizes),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
# Initialize to avoid NaN
|
||||
input_scale[:] = torch.finfo(torch.float32).min
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
input_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(
|
||||
len(output_partition_sizes),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
# Initialize to avoid NaN
|
||||
input_scale[:] = torch.finfo(torch.float32).min
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Ensuring weights & scales are non-trainable
|
||||
@@ -147,16 +137,15 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
|
||||
layer.weight_scale.data, requires_grad=False
|
||||
)
|
||||
|
||||
if self.is_static_input_scheme:
|
||||
input_scale = layer.input_scale.data
|
||||
# For fused modules (QKV), take the max scale
|
||||
if input_scale.numel() != 1:
|
||||
input_scale = input_scale.max()
|
||||
input_scale = layer.input_scale.data
|
||||
# For fused modules (QKV), take the max scale
|
||||
if input_scale.numel() != 1:
|
||||
input_scale = input_scale.max()
|
||||
|
||||
layer.input_scale = torch.nn.Parameter(
|
||||
torch.tensor(input_scale, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.input_scale = torch.nn.Parameter(
|
||||
torch.tensor(input_scale, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
@@ -18,8 +17,11 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockE8M0Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
@@ -42,36 +44,26 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkW8A8Fp8(QuarkScheme):
|
||||
supported_activation_quant_keys: list[QuantKey | None] = [
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticTensorSym,
|
||||
]
|
||||
supported_weight_quant_keys: list[QuantKey] = [
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
self.is_static_input_scheme: bool = False
|
||||
self.input_qscheme: str | None = None
|
||||
if input_config is not None:
|
||||
self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic"))
|
||||
self.input_qscheme = cast(str, input_config.get("qscheme"))
|
||||
|
||||
if self.weight_qscheme == "per_block":
|
||||
raise ValueError(
|
||||
"Quark W8A8 FP8 per-block weight quantization should use "
|
||||
"QuarkW8A8Fp8PerBlock."
|
||||
)
|
||||
per_token_activation = (
|
||||
not self.is_static_input_scheme and self.input_qscheme == "per_channel"
|
||||
)
|
||||
per_channel_weight = self.weight_qscheme == "per_channel"
|
||||
|
||||
self.activation_quant_key = (
|
||||
kFp8DynamicTokenSym if per_token_activation else kFp8StaticTensorSym
|
||||
)
|
||||
# A per-output-channel weight scale is one fp32 value per weight row
|
||||
# (length N). Tag it as ``GroupShape.PER_CHANNEL`` to match the
|
||||
# canonical compressed-tensors CHANNEL strategy, so kernel selection
|
||||
# (e.g. AITER's pre-shuffled FP8 GEMM) treats it uniformly.
|
||||
self.weight_quant_key = (
|
||||
kFp8StaticChannelSym if per_channel_weight else kFp8StaticTensorSym
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.weight_qscheme = (
|
||||
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
|
||||
)
|
||||
self.is_static_input_scheme = activation_quant_key == kFp8StaticTensorSym
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
|
||||
@@ -122,6 +114,8 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
layer.input_scale = Parameter(input_scale, requires_grad=False)
|
||||
else:
|
||||
weight_scale = layer.weight_scale.data
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
if self.activation_quant_key.scale.group_shape == GroupShape.PER_TOKEN:
|
||||
weight_scale = weight_scale.view(-1, 1)
|
||||
layer.weight = Parameter(weight.t(), requires_grad=False)
|
||||
@@ -191,6 +185,7 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
input_scale[:] = torch.finfo(torch.float32).min
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
@@ -210,22 +205,23 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
|
||||
|
||||
class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
supported_activation_quant_keys = [kFp8Dynamic128Sym]
|
||||
supported_weight_quant_keys = [
|
||||
kFp8Static128BlockSym,
|
||||
kFp8Static128BlockE8M0Sym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
if self.weight_qscheme != "per_block":
|
||||
raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.")
|
||||
block_size = weight_config.get("block_size")
|
||||
if not block_size:
|
||||
raise ValueError(
|
||||
"Quark W8A8 FP8 per-block weight quantization requires "
|
||||
"`block_size` in the weight quantization config."
|
||||
)
|
||||
self.weight_config = weight_config
|
||||
self.weight_block_size = list(block_size)
|
||||
self.activation_quant_key = kFp8Dynamic128Sym
|
||||
self.weight_quant_key = kFp8Static128BlockSym
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.weight_scale_dtype = weight_quant_key.scale.dtype
|
||||
if self.weight_quant_key == kFp8Static128BlockE8M0Sym:
|
||||
# TODO: oracle needs to support kFp8Static128BlockE8M0Sym.
|
||||
self.weight_quant_key = kFp8Static128BlockSym
|
||||
self.weight_block_size = list(weight_quant_key.scale.group_shape)
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
|
||||
@@ -272,8 +268,8 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
scale_dtype = (
|
||||
torch.float8_e8m0fnu
|
||||
if self.weight_config.get("scale_type") == "float8_e8m0fnu"
|
||||
self.weight_scale_dtype
|
||||
if self.weight_scale_dtype != torch.float32
|
||||
else None
|
||||
)
|
||||
weight_scale = create_fp8_scale_parameter(
|
||||
@@ -287,6 +283,7 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
|
||||
layer.register_parameter("weight_scale_inv", weight_scale)
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
|
||||
@@ -10,6 +10,16 @@ from vllm.model_executor.kernels.linear import (
|
||||
init_int8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8StaticTensorSym,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
@@ -21,15 +31,32 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkW8A8Int8(QuarkScheme):
|
||||
supported_activation_quant_keys = [
|
||||
kInt8StaticTensorSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
]
|
||||
supported_weight_quant_keys = [
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorSym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
qscheme: str,
|
||||
is_static_input_scheme: bool | None,
|
||||
input_symmetric: bool | None,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
self.qscheme = qscheme
|
||||
self.is_static_input_scheme = is_static_input_scheme
|
||||
self.input_symmetric = input_symmetric
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.qscheme = (
|
||||
"per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
|
||||
)
|
||||
|
||||
assert activation_quant_key is not None
|
||||
self.is_static_input_scheme = activation_quant_key.scale.static
|
||||
self.input_symmetric = activation_quant_key.symmetric
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
|
||||
@@ -8,6 +8,8 @@ from typing import Any
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None
|
||||
|
||||
|
||||
def deep_compare(dict1: Any, dict2: Any) -> bool:
|
||||
if type(dict1) is not type(dict2):
|
||||
|
||||
@@ -3,11 +3,32 @@
|
||||
from enum import Enum
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
OCP_MX_BLOCK_SIZE = 32
|
||||
|
||||
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Static,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Static,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Static,
|
||||
}
|
||||
|
||||
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Dynamic,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
|
||||
}
|
||||
|
||||
OCP_MX_DTYPES = {
|
||||
"mxfp4",
|
||||
"mxfp6_e3m2",
|
||||
|
||||
@@ -222,6 +222,11 @@ kFp8Dynamic128Sym = QuantKey(FP8_DTYPE, kDynamic128Scale, symmetric=True)
|
||||
|
||||
kStatic128BlockScale = ScaleDesc(torch.float32, True, GroupShape(128, 128))
|
||||
kFp8Static128BlockSym = QuantKey(FP8_DTYPE, kStatic128BlockScale, symmetric=True)
|
||||
kFp8Static128BlockE8M0Sym = QuantKey(
|
||||
FP8_DTYPE,
|
||||
ScaleDesc(torch.float8_e8m0fnu, True, GroupShape(128, 128)),
|
||||
symmetric=True,
|
||||
)
|
||||
|
||||
kMxfp8StaticScale = ScaleDesc(torch.uint8, True, GroupShape(1, 32))
|
||||
kMxfp8Static = QuantKey(FP8_DTYPE, kMxfp8StaticScale, symmetric=True)
|
||||
@@ -284,6 +289,9 @@ kInt8StaticChannelSym = QuantKey(torch.int8, kStaticChannelScale, symmetric=True
|
||||
kInt8DynamicTokenSym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=True)
|
||||
kInt8StaticTensorSym = QuantKey(torch.int8, kStaticTensorScale, symmetric=True)
|
||||
kInt8DynamicTensorSym = QuantKey(torch.int8, kDynamicTensorScale, symmetric=True)
|
||||
kInt8DynamicTokenAsym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=False)
|
||||
kInt8StaticTensorAsym = QuantKey(torch.int8, kStaticTensorScale, symmetric=False)
|
||||
kInt8DynamicTensorAsym = QuantKey(torch.int8, kDynamicTensorScale, symmetric=False)
|
||||
|
||||
# INT4 W4A8 quantization keys
|
||||
|
||||
|
||||
Reference in New Issue
Block a user