[Quantization][Refactor][1/N] Adopt QuantKey in QuarkConfig and methods, relying on weight_quant_key, act_quant_key for quant method dispatch (#52958)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
d0e695a91b
commit
63988f3c2d
@@ -3,11 +3,32 @@
|
||||
from enum import Enum
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
OCP_MX_BLOCK_SIZE = 32
|
||||
|
||||
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Static,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Static,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Static,
|
||||
}
|
||||
|
||||
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Dynamic,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
|
||||
}
|
||||
|
||||
OCP_MX_DTYPES = {
|
||||
"mxfp4",
|
||||
"mxfp6_e3m2",
|
||||
|
||||
@@ -222,6 +222,11 @@ kFp8Dynamic128Sym = QuantKey(FP8_DTYPE, kDynamic128Scale, symmetric=True)
|
||||
|
||||
kStatic128BlockScale = ScaleDesc(torch.float32, True, GroupShape(128, 128))
|
||||
kFp8Static128BlockSym = QuantKey(FP8_DTYPE, kStatic128BlockScale, symmetric=True)
|
||||
kFp8Static128BlockE8M0Sym = QuantKey(
|
||||
FP8_DTYPE,
|
||||
ScaleDesc(torch.float8_e8m0fnu, True, GroupShape(128, 128)),
|
||||
symmetric=True,
|
||||
)
|
||||
|
||||
kMxfp8StaticScale = ScaleDesc(torch.uint8, True, GroupShape(1, 32))
|
||||
kMxfp8Static = QuantKey(FP8_DTYPE, kMxfp8StaticScale, symmetric=True)
|
||||
@@ -284,6 +289,9 @@ kInt8StaticChannelSym = QuantKey(torch.int8, kStaticChannelScale, symmetric=True
|
||||
kInt8DynamicTokenSym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=True)
|
||||
kInt8StaticTensorSym = QuantKey(torch.int8, kStaticTensorScale, symmetric=True)
|
||||
kInt8DynamicTensorSym = QuantKey(torch.int8, kDynamicTensorScale, symmetric=True)
|
||||
kInt8DynamicTokenAsym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=False)
|
||||
kInt8StaticTensorAsym = QuantKey(torch.int8, kStaticTensorScale, symmetric=False)
|
||||
kInt8DynamicTensorAsym = QuantKey(torch.int8, kDynamicTensorScale, symmetric=False)
|
||||
|
||||
# INT4 W4A8 quantization keys
|
||||
|
||||
|
||||
Reference in New Issue
Block a user