[Quantization][Refactor][1/N] Adopt QuantKey in QuarkConfig and methods, relying on weight_quant_key, act_quant_key for quant method dispatch (#52958)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
d0e695a91b
commit
63988f3c2d
@@ -35,6 +35,10 @@ from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
mxfp4_quantize,
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import PlatformEnum, current_platform
|
||||
@@ -346,9 +350,9 @@ def test_online_mxfp4_moe_matches_quark(
|
||||
online_layer = make_layer("online_layer")
|
||||
|
||||
checkpoint_method = QuarkOCP_MX_MoEMethod(
|
||||
weight_config={"qscheme": "per_group", "dtype": "fp4"},
|
||||
input_config={"dtype": "fp4", "is_dynamic": True},
|
||||
moe=checkpoint_layer.moe_config,
|
||||
weight_quant_key=kMxfp4Static,
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
)
|
||||
online_method = Mxfp4OnlineMoEMethod(layer=online_layer)
|
||||
|
||||
@@ -566,8 +570,8 @@ def test_online_mxfp4_dense_matches_quark(
|
||||
checkpoint_weight, checkpoint_weight_scale = mxfp4_quantize(weight)
|
||||
|
||||
checkpoint_scheme = QuarkOCP_MX(
|
||||
weight_quant_spec={"qscheme": "per_group", "dtype": "fp4"},
|
||||
input_quant_spec={"dtype": "fp4", "is_dynamic": True},
|
||||
weight_quant_key=kMxfp4Static,
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
)
|
||||
checkpoint_scheme.create_weights(
|
||||
layer=checkpoint_layer,
|
||||
|
||||
@@ -10,6 +10,7 @@ See also `tests/kernels/moe/test_ocp_mx_moe.py`.
|
||||
import importlib.metadata
|
||||
from dataclasses import dataclass
|
||||
from importlib.util import find_spec
|
||||
from types import SimpleNamespace
|
||||
|
||||
import huggingface_hub
|
||||
import lm_eval
|
||||
@@ -18,22 +19,62 @@ import torch
|
||||
from packaging import version
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
|
||||
QuarkConfig,
|
||||
QuarkLinearMethod,
|
||||
QuarkNVFP4,
|
||||
QuarkOCP_MX,
|
||||
QuarkW8A8Fp8,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
QuarkW8A8Int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
|
||||
QuarkMoEMethod,
|
||||
QuarkW4A8Fp8MoEMethod,
|
||||
QuarkW8A8Int8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
is_layer_skipped,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockE8M0Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4W4A8StaticChannelSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8StaticTensorSym,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
@@ -62,6 +103,576 @@ AITER_AVAILABLE = is_aiter_found_and_supported()
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QTensorConfig:
|
||||
name: str
|
||||
weight: QuarkQTensorHint
|
||||
input_tensors: QuarkQTensorHint
|
||||
weight_quant_key: QuantKey | None = None
|
||||
act_quant_key: QuantKey | None = None
|
||||
dispatch_cls: type[QuarkScheme] | type[QuarkMoEMethod] | None = None
|
||||
expected_error: tuple[type[Exception], str] | None = None
|
||||
|
||||
|
||||
QTENSOR_CONFIGS = [
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_static_tensor",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_static_tensor_single_entry_lists",
|
||||
weight=[
|
||||
{
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
}
|
||||
],
|
||||
input_tensors=[
|
||||
{
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
}
|
||||
],
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_dynamic_tensor",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
act_quant_key=kFp8DynamicTensorSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_dynamic_token",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_channel", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kFp8StaticChannelSym,
|
||||
act_quant_key=kFp8DynamicTokenSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_channel_static_tensor",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_channel", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
weight_quant_key=kFp8StaticChannelSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_tensor_dynamic_token",
|
||||
weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
act_quant_key=kFp8DynamicTokenSym,
|
||||
dispatch_cls=QuarkW8A8Fp8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_dynamic_block_fp32",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kFp8Static128BlockSym,
|
||||
act_quant_key=kFp8Dynamic128Sym,
|
||||
dispatch_cls=QuarkW8A8Fp8PerBlock,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_dynamic_block_e8m0",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
"scale_type": "float8_e8m0fnu",
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kFp8Static128BlockE8M0Sym,
|
||||
act_quant_key=kFp8Dynamic128Sym,
|
||||
dispatch_cls=QuarkW8A8Fp8PerBlock,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_block_static_input",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": False,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
},
|
||||
expected_error=(NotImplementedError, "No quark compatible scheme"),
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_block_group_size_mismatch",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 64,
|
||||
"symmetric": True,
|
||||
},
|
||||
expected_error=(NotImplementedError, "No quark compatible scheme"),
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_block_missing_block_size",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
},
|
||||
expected_error=(ValueError, "requires `block_size`"),
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_static_symmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kInt8StaticTensorSym,
|
||||
act_quant_key=kInt8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_static_asymmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": False,
|
||||
},
|
||||
weight_quant_key=kInt8StaticTensorSym,
|
||||
act_quant_key=kInt8StaticTensorAsym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_channel_static_symmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kInt8StaticChannelSym,
|
||||
act_quant_key=kInt8StaticTensorSym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_channel_static_asymmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": False,
|
||||
},
|
||||
weight_quant_key=kInt8StaticChannelSym,
|
||||
act_quant_key=kInt8StaticTensorAsym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_dynamic_tensor_symmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kInt8StaticTensorSym,
|
||||
act_quant_key=kInt8DynamicTensorSym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_dynamic_tensor_asymmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
"symmetric": False,
|
||||
},
|
||||
weight_quant_key=kInt8StaticTensorSym,
|
||||
act_quant_key=kInt8DynamicTensorAsym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_dynamic_token",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kInt8StaticChannelSym,
|
||||
act_quant_key=kInt8DynamicTokenSym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="int8_w8a8_dynamic_token_asymmetric",
|
||||
weight={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
"symmetric": False,
|
||||
},
|
||||
weight_quant_key=kInt8StaticChannelSym,
|
||||
act_quant_key=kInt8DynamicTokenAsym,
|
||||
dispatch_cls=QuarkW8A8Int8,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp4_weight_only",
|
||||
weight={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors=None,
|
||||
weight_quant_key=kMxfp4Static,
|
||||
act_quant_key=None,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp4_activation",
|
||||
weight={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp4Static,
|
||||
act_quant_key=kMxfp4Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp6_e3m2",
|
||||
weight={
|
||||
"dtype": "fp6_e3m2",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp6_e3m2",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp6E3M2Static,
|
||||
act_quant_key=kMxfp6E3M2Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp4_mxfp6_e3m2_activation",
|
||||
weight={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp6_e3m2",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp4Static,
|
||||
act_quant_key=kMxfp6E3M2Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp4_mxfp6_e2m3_activation",
|
||||
weight={
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp6_e2m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp4Static,
|
||||
act_quant_key=kMxfp6E2M3Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="ocp_mx_mxfp6_e2m3",
|
||||
weight={
|
||||
"dtype": "fp6_e2m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp6_e2m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kMxfp6E2M3Static,
|
||||
act_quant_key=kMxfp6E2M3Dynamic,
|
||||
dispatch_cls=QuarkOCP_MX,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="nvfp4",
|
||||
weight=[
|
||||
{
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 16,
|
||||
"is_dynamic": False,
|
||||
},
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
],
|
||||
input_tensors=[
|
||||
{
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 16,
|
||||
"is_dynamic": True,
|
||||
},
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
],
|
||||
weight_quant_key=kNvfp4Static,
|
||||
act_quant_key=kNvfp4Dynamic,
|
||||
dispatch_cls=QuarkNVFP4,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="w4a8_fp8_static",
|
||||
weight=[
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
{
|
||||
"dtype": "int4",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
"ch_axis": 0,
|
||||
},
|
||||
],
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
weight_quant_key=kInt4W4A8StaticChannelSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW4A8Fp8MoEMethod,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="w4a8_fp8_dynamic",
|
||||
weight=[
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
{
|
||||
"dtype": "int4",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
"ch_axis": 0,
|
||||
},
|
||||
],
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
weight_quant_key=kInt4W4A8StaticChannelSym,
|
||||
act_quant_key=kFp8DynamicTokenSym,
|
||||
dispatch_cls=QuarkW4A8Fp8MoEMethod,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="w4a8_fp8_static_single_entry_input",
|
||||
weight=[
|
||||
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
|
||||
{
|
||||
"dtype": "int4",
|
||||
"qscheme": "per_channel",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
"ch_axis": 0,
|
||||
},
|
||||
],
|
||||
input_tensors=[
|
||||
{
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
}
|
||||
],
|
||||
weight_quant_key=kInt4W4A8StaticChannelSym,
|
||||
act_quant_key=kFp8StaticTensorSym,
|
||||
dispatch_cls=QuarkW4A8Fp8MoEMethod,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _make_qtensor_config(
|
||||
weight: QuarkQTensorHint,
|
||||
input_tensors: QuarkQTensorHint,
|
||||
exclude: list[str] | None = None,
|
||||
) -> QuarkConfig:
|
||||
return QuarkConfig(
|
||||
{
|
||||
"global_quant_config": {
|
||||
"weight": weight,
|
||||
"input_tensors": input_tensors,
|
||||
},
|
||||
"layer_type_quant_config": {},
|
||||
"exclude": exclude or [],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _make_test_moe_config() -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
activation=MoEActivation.SILU,
|
||||
device=current_platform.device_type,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
if QUARK_MXFP4_AVAILABLE:
|
||||
from quark.torch.export.nn.modules.realquantizer import StaticScaledRealQuantizer
|
||||
from quark.torch.kernel import mx as mx_kernel
|
||||
@@ -98,83 +709,197 @@ def test_quark_config_preserves_existing_packed_modules_mapping():
|
||||
assert config.packed_modules_mapping["custom_proj"] == ["a", "b"]
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_detects_per_block_config():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
def test_quant_method_dispatch_ignored(default_vllm_config):
|
||||
config = _make_qtensor_config(None, None, exclude=["linear", "experts"])
|
||||
|
||||
assert config._is_fp8_w8a8(weight_config, input_config)
|
||||
class TestLinear(LinearBase):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
class TestRoutedExperts(RoutedExperts):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.moe_config = _make_test_moe_config()
|
||||
|
||||
assert config.get_quant_method_target("linear", LinearBase) == (
|
||||
None,
|
||||
None,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
config.get_quant_method(TestLinear(), "linear"), UnquantizedLinearMethod
|
||||
)
|
||||
|
||||
assert config.get_quant_method_target("experts", RoutedExperts) == (
|
||||
None,
|
||||
None,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
config.get_quant_method(TestRoutedExperts(), "experts"),
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
dynamic_mxfp4_config = _make_qtensor_config(
|
||||
{
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
None,
|
||||
exclude=["self_attn.q_proj", "mlp.down_proj"],
|
||||
)
|
||||
dynamic_mxfp4_config.dynamic_mxfp4_quant = True
|
||||
|
||||
assert dynamic_mxfp4_config.get_quant_method_target(
|
||||
"self_attn.q_proj", LinearBase
|
||||
) == (kMxfp4Static, None, QuarkLinearMethod)
|
||||
attention_proj = TestLinear()
|
||||
assert isinstance(
|
||||
dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"),
|
||||
QuarkLinearMethod,
|
||||
)
|
||||
assert isinstance(attention_proj.scheme, QuarkOCP_MX)
|
||||
assert attention_proj.scheme.dynamic_mxfp4_quant
|
||||
|
||||
assert dynamic_mxfp4_config.get_quant_method_target(
|
||||
"mlp.down_proj", LinearBase
|
||||
) == (None, None, UnquantizedLinearMethod)
|
||||
assert isinstance(
|
||||
dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_rejects_per_block_static_input():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": False,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
@pytest.mark.parametrize("case", QTENSOR_CONFIGS, ids=lambda case: case.name)
|
||||
def test_quant_method_dispatch_target(case):
|
||||
config = _make_qtensor_config(case.weight, case.input_tensors)
|
||||
if case.expected_error is not None:
|
||||
error_type, error_message = case.expected_error
|
||||
with pytest.raises(error_type, match=error_message):
|
||||
config.get_quant_method_target("linear", LinearBase)
|
||||
return
|
||||
|
||||
assert not config._is_fp8_w8a8(weight_config, input_config)
|
||||
assert case.dispatch_cls is not None
|
||||
is_linear = issubclass(case.dispatch_cls, QuarkScheme)
|
||||
|
||||
weight_quant_key, act_quant_key, method_cls = config.get_quant_method_target(
|
||||
"linear" if is_linear else "experts",
|
||||
LinearBase if is_linear else RoutedExperts,
|
||||
)
|
||||
|
||||
assert weight_quant_key == case.weight_quant_key
|
||||
assert act_quant_key == case.act_quant_key
|
||||
assert method_cls is (QuarkLinearMethod if is_linear else case.dispatch_cls)
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_rejects_per_block_group_size_mismatch():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 64,
|
||||
"symmetric": True,
|
||||
}
|
||||
@pytest.mark.parametrize(
|
||||
("weight", "input_tensors"),
|
||||
[
|
||||
pytest.param(
|
||||
{
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
{
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
id="single_entry",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{"dtype": "int8", "qscheme": "per_tensor"},
|
||||
{"dtype": "int8", "qscheme": "per_tensor"},
|
||||
],
|
||||
[
|
||||
{"dtype": "int8", "qscheme": "per_tensor"},
|
||||
{"dtype": "int8", "qscheme": "per_tensor"},
|
||||
],
|
||||
id="multi_entry",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_quant_method_dispatch_unsupported(weight, input_tensors):
|
||||
config = _make_qtensor_config(weight, input_tensors)
|
||||
|
||||
assert not config._is_fp8_w8a8(weight_config, input_config)
|
||||
class TestRoutedExperts(RoutedExperts):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
with pytest.raises(RuntimeError, match="^Unsupported FusedMoe scheme$"):
|
||||
config.get_quant_method_target("experts", RoutedExperts)
|
||||
|
||||
with pytest.raises(RuntimeError, match="^Unsupported FusedMoe scheme$"):
|
||||
config.get_quant_method(TestRoutedExperts(), "experts")
|
||||
|
||||
|
||||
def test_quark_w8a8_fp8_per_block_requires_block_size():
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[case for case in QTENSOR_CONFIGS if case.expected_error is None],
|
||||
ids=lambda case: case.name,
|
||||
)
|
||||
def test_quant_method_dispatch_instantiation(case, monkeypatch, default_vllm_config):
|
||||
config = _make_qtensor_config(case.weight, case.input_tensors)
|
||||
assert case.dispatch_cls is not None
|
||||
if issubclass(case.dispatch_cls, QuarkScheme):
|
||||
|
||||
with pytest.raises(ValueError, match="requires `block_size`"):
|
||||
QuarkW8A8Fp8PerBlock(weight_config, input_config)
|
||||
class TestLinear(LinearBase):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes."
|
||||
"quark_w8a8_fp8.get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=SimpleNamespace(dtype=torch.bfloat16)),
|
||||
)
|
||||
layer = TestLinear()
|
||||
method = config.get_quant_method(layer, "linear")
|
||||
|
||||
assert isinstance(method, QuarkLinearMethod)
|
||||
assert isinstance(layer.scheme, case.dispatch_cls)
|
||||
if case.weight_quant_key == kFp8Static128BlockE8M0Sym:
|
||||
# TODO: Remove once E8M0 quant key is properly handled in oracle
|
||||
assert layer.scheme.weight_quant_key == kFp8Static128BlockSym
|
||||
else:
|
||||
assert layer.scheme.weight_quant_key == case.weight_quant_key
|
||||
assert layer.scheme.activation_quant_key == case.act_quant_key
|
||||
else:
|
||||
|
||||
class TestRoutedExperts(RoutedExperts):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.moe_config = _make_test_moe_config()
|
||||
|
||||
for target in (
|
||||
"select_fp8_moe_backend",
|
||||
"select_int8_moe_backend",
|
||||
"select_mxfp4_moe_backend",
|
||||
"backend_to_kernel_cls",
|
||||
"select_nvfp4_moe_backend",
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
f"vllm.model_executor.layers.quantization.quark.quark_moe.{target}",
|
||||
lambda *args, **kwargs: (object(), object()),
|
||||
)
|
||||
|
||||
# AssertionError: W4A8 FP8 MoE requires ROCm AITER fused MoE support
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.quark.quark_moe."
|
||||
"rocm_aiter_ops.is_fused_moe_enabled",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
layer = TestRoutedExperts()
|
||||
method = config.get_quant_method(layer, "experts")
|
||||
|
||||
assert isinstance(method, case.dispatch_cls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
|
||||
Reference in New Issue
Block a user