[Quantization] add online fp8 ptpc (#44132)
Signed-off-by: walterbm <walter.beller.morales@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ac3409d162
commit
753e9d55e6
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Dynamic,
|
||||
@@ -24,6 +25,7 @@ QUANT_KEY_NAMES: dict[str, QuantKey] = {
|
||||
"fp8_per_tensor_static": kFp8StaticTensorSym,
|
||||
"fp8_per_tensor_dynamic": kFp8DynamicTensorSym,
|
||||
"fp8_per_token": kFp8DynamicTokenSym,
|
||||
"fp8_per_channel_static": kFp8StaticChannelSym,
|
||||
"fp8_per_block_static": kFp8Static128BlockSym,
|
||||
"fp8_per_block_dynamic": kFp8Dynamic128Sym,
|
||||
"mxfp8": kMxfp8Dynamic,
|
||||
@@ -118,6 +120,12 @@ _ONLINE_SHORTHANDS: dict[str, QuantizationConfigArgs] = {
|
||||
linear=QuantSpec(weight=kFp8Static128BlockSym),
|
||||
moe=QuantSpec(weight=kFp8Static128BlockSym),
|
||||
),
|
||||
# Per-output-channel weight scale + dynamic per-token activation.
|
||||
# Same shape as llmcompressor's FP8_DYNAMIC recipe.
|
||||
"fp8_per_channel": QuantizationConfigArgs(
|
||||
linear=QuantSpec(weight=kFp8StaticChannelSym),
|
||||
moe=QuantSpec(weight=kFp8StaticChannelSym),
|
||||
),
|
||||
"mxfp8": QuantizationConfigArgs(
|
||||
linear=QuantSpec(weight=kMxfp8Dynamic),
|
||||
moe=QuantSpec(weight=kMxfp8Dynamic),
|
||||
|
||||
Reference in New Issue
Block a user