[Quantization] add online fp8 ptpc (#44132)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Walter Beller-Morales
2026-06-08 22:42:11 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ac3409d162
commit 753e9d55e6
6 changed files with 385 additions and 8 deletions
@@ -40,6 +40,7 @@ QuantizationMethods = Literal[
# _ONLINE_SHORTHANDS by the assertion in get_quantization_config().
"fp8_per_tensor",
"fp8_per_block",
"fp8_per_channel",
"int8_per_channel_weight_only",
"mxfp8",
]
@@ -30,6 +30,8 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerBlockOnlineMoEMethod,
Fp8PerTensorOnlineLinearMethod,
Fp8PerTensorOnlineMoEMethod,
Fp8PtpcOnlineLinearMethod,
Fp8PtpcOnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.online.int8 import (
Int8OnlineMoEMethod,
@@ -41,6 +43,7 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import (
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt8StaticChannelSym,
kMxfp8Dynamic,
@@ -55,12 +58,14 @@ logger = init_logger(__name__)
_ONLINE_LINEAR_METHODS: dict[QuantKey, type] = {
kFp8StaticTensorSym: Fp8PerTensorOnlineLinearMethod,
kFp8Static128BlockSym: Fp8PerBlockOnlineLinearMethod,
kFp8StaticChannelSym: Fp8PtpcOnlineLinearMethod,
kMxfp8Dynamic: Mxfp8OnlineLinearMethod,
}
_ONLINE_MOE_METHODS: dict[QuantKey, type] = {
kFp8StaticTensorSym: Fp8PerTensorOnlineMoEMethod,
kFp8Static128BlockSym: Fp8PerBlockOnlineMoEMethod,
kFp8StaticChannelSym: Fp8PtpcOnlineMoEMethod,
kMxfp8Dynamic: Mxfp8OnlineMoEMethod,
kInt8StaticChannelSym: Int8OnlineMoEMethod,
}
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
import vllm.envs as envs
from vllm import _custom_ops as ops
@@ -19,6 +20,7 @@ from vllm.config import get_current_vllm_config
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
@@ -37,6 +39,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -270,6 +273,89 @@ class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
)
class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase):
"""Online PTPC FP8 linear quantization.
Per-output-channel weight scale + dynamic per-token activation scale. The
layout matches the llmcompressor's FP8_DYNAMIC recipe, so accuracy
is comparable but no pre-quantized checkpoint is required.
"""
weight_quant_key = kFp8StaticChannelSym
activation_quant_key = kFp8DynamicTokenSym
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
super().create_weights(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
)
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
weight_shape=layer.weight.shape,
input_dtype=self.input_dtype,
out_dtype=self.out_dtype,
module_name=self.__class__.__name__,
)
# PTPC requires per-token activation FP8; MarlinFP8 is W8A16 and
# would silently produce a weight-only fp8 model.
if isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel):
raise ValueError(
"FP8 PTPC online quant requires a kernel that honors "
"per-token activation quantization; MarlinFP8 is W8A16 "
"weight-only. Requires SM89+ for Cutlass FP8 or ROCm MI3xx "
"for rowwise scaled_mm."
)
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
layer.input_scale = None
qweight, weight_scale = ops.scaled_fp8_quant(
layer.weight, scale=None, use_per_token_if_dynamic=True
)
replace_parameter(layer, "weight", qweight.t())
replace_parameter(layer, "weight_scale", weight_scale)
self.fp8_linear.process_weights_after_loading(layer)
layer._already_called_process_weights_after_loading = True
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
# if batch invariant mode is enabled dequant
if envs.VLLM_BATCH_INVARIANT and not isinstance(
self.fp8_linear, CutlassFP8ScaledMMLinearKernel
):
weight_dequant = (
layer.weight.to(x.dtype) * layer.weight_scale.to(x.dtype).t()
)
return torch.nn.functional.linear(x, weight_dequant.t(), bias)
return self.fp8_linear.apply_weights(layer, x, bias)
# ---------------------------------------------------------------------------
# Online FP8 MoE Methods
# ---------------------------------------------------------------------------
@@ -284,12 +370,17 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
experts_cls: "type[mk.FusedMoEExperts] | None"
weight_scale_name: str
weight_block_size: list[int] | None
per_act_token_quant: bool = False
per_out_ch_quant: bool = False
def __init__(
self,
*,
weight_block_size: list[int] | None,
layer: torch.nn.Module,
weight_key: "QuantKey | None" = None,
activation_key: "QuantKey | None" = None,
allow_vllm_cutlass: bool = False,
):
super().__init__(layer.moe_config)
self.weight_block_size = weight_block_size
@@ -298,20 +389,22 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
"weight_scale_inv" if self.block_quant else "weight_scale"
)
# Set weight key and activation key for kernel compatibility
if self.block_quant:
weight_key = kFp8Static128BlockSym
activation_key = kFp8Dynamic128Sym
else:
weight_key = kFp8StaticTensorSym
activation_key = kFp8DynamicTensorSym
# Subclasses may pass explicit kernel keys (PTPC needs channelwise +
# per-token).
if weight_key is None or activation_key is None:
if self.block_quant:
weight_key = kFp8Static128BlockSym
activation_key = kFp8Dynamic128Sym
else:
weight_key = kFp8StaticTensorSym
activation_key = kFp8DynamicTensorSym
# Select Fp8 MoE backend
self.fp8_backend, self.experts_cls = select_fp8_moe_backend(
config=self.moe,
weight_key=weight_key,
activation_key=activation_key,
allow_vllm_cutlass=False,
allow_vllm_cutlass=allow_vllm_cutlass,
)
def _setup_kernel(
@@ -380,6 +473,8 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
block_shape=self.weight_block_size,
per_act_token_quant=self.per_act_token_quant,
per_out_ch_quant=self.per_out_ch_quant,
swiglu_limit=getattr(layer, "swiglu_limit", None),
)
@@ -511,3 +606,89 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
"""Online PTPC FP8 MoE quantization.
Quantizes each expert's weights per output channel during loading.
Activations are quantized dynamically per token at runtime.
"""
per_act_token_quant: bool = True
per_out_ch_quant: bool = True
def __init__(
self,
*,
layer: torch.nn.Module,
):
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
super().__init__(
weight_block_size=None,
layer=layer,
weight_key=kFp8StaticChannelSym,
activation_key=kFp8DynamicTokenSym,
allow_vllm_cutlass=True,
)
# Reject backends whose make_fp8_moe_quant_config branch silently
# drops per_act_token_quant / per_out_ch_quant or collapses scales:
# MARLIN / CPU route through fp8_w8a16_moe_quant_config; FLASHINFER_*
# fold scales into a per-tensor alpha (oracle/fp8.py).
if self.fp8_backend in (
Fp8MoeBackend.MARLIN,
Fp8MoeBackend.CPU,
Fp8MoeBackend.FLASHINFER_CUTLASS,
Fp8MoeBackend.FLASHINFER_TRTLLM,
):
raise ValueError(
f"FP8 PTPC online MoE quant is not supported with the "
f"{self.fp8_backend.value} backend, which does not implement "
"per-output-channel weight scales."
)
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
fp8_dtype = current_platform.fp8_dtype()
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
# Scale's leading dim is taken from the fp8 weight tensor by
# construction, so it cannot drift from the weight's expert count
# under EP / padded MoE.
n_w13 = layer.w13_weight.shape[1]
n_w2 = layer.w2_weight.shape[1]
w13_scale = torch.ones(
w13.shape[0], n_w13, 1, device=w13.device, dtype=torch.float32
)
w2_scale = torch.ones(
w2.shape[0], n_w2, 1, device=w2.device, dtype=torch.float32
)
layer.w13_input_scale = None
layer.w2_input_scale = None
for expert in range(layer.local_num_experts):
w13[expert], w13_scale[expert] = ops.scaled_fp8_quant(
layer.w13_weight[expert],
scale=None,
use_per_token_if_dynamic=True,
)
w2[expert], w2_scale[expert] = ops.scaled_fp8_quant(
layer.w2_weight[expert],
scale=None,
use_per_token_if_dynamic=True,
)
self._setup_kernel(
layer,
w13,
w2,
w13_scale,
w2_scale,
w13_input_scale=None,
w2_input_scale=None,
)
layer._already_called_process_weights_after_loading = True