[Quantization] add online fp8 ptpc (#44132)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Walter Beller-Morales
2026-06-08 22:42:11 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ac3409d162
commit 753e9d55e6
6 changed files with 385 additions and 8 deletions
+8
View File
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt8StaticChannelSym,
kMxfp4Dynamic,
@@ -24,6 +25,7 @@ QUANT_KEY_NAMES: dict[str, QuantKey] = {
"fp8_per_tensor_static": kFp8StaticTensorSym,
"fp8_per_tensor_dynamic": kFp8DynamicTensorSym,
"fp8_per_token": kFp8DynamicTokenSym,
"fp8_per_channel_static": kFp8StaticChannelSym,
"fp8_per_block_static": kFp8Static128BlockSym,
"fp8_per_block_dynamic": kFp8Dynamic128Sym,
"mxfp8": kMxfp8Dynamic,
@@ -118,6 +120,12 @@ _ONLINE_SHORTHANDS: dict[str, QuantizationConfigArgs] = {
linear=QuantSpec(weight=kFp8Static128BlockSym),
moe=QuantSpec(weight=kFp8Static128BlockSym),
),
# Per-output-channel weight scale + dynamic per-token activation.
# Same shape as llmcompressor's FP8_DYNAMIC recipe.
"fp8_per_channel": QuantizationConfigArgs(
linear=QuantSpec(weight=kFp8StaticChannelSym),
moe=QuantSpec(weight=kFp8StaticChannelSym),
),
"mxfp8": QuantizationConfigArgs(
linear=QuantSpec(weight=kMxfp8Dynamic),
moe=QuantSpec(weight=kMxfp8Dynamic),