[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-09-04 11:46:06 -07:00
committed by GitHub
co-authored by Codex mgoin
parent a11dfcff89
commit 5690b02c03
25 changed files with 846 additions and 175 deletions
@@ -23,6 +23,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
select_fp8_moe_backend,
)
@@ -111,7 +112,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
return is_row_parallel or (reduces_output_dim and is_column_parallel)
class _Fp8OnlineLinearBase(LinearMethodBase):
class OnlineLinearBase(LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time."""
@@ -155,7 +156,7 @@ class _Fp8OnlineLinearBase(LinearMethodBase):
initialize_online_processing(layer)
class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
@@ -256,7 +257,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
return self.fp8_linear.apply_weights(layer, x, bias)
class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
"""Online blockwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading."""
@@ -336,7 +337,7 @@ class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
)
class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
"""Online PTPC FP8 linear quantization.
Per-output-channel weight scale + dynamic per-token activation scale. The
@@ -443,12 +444,12 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
self,
*,
weight_block_size: list[int] | None,
layer: torch.nn.Module,
moe: FusedMoEConfig,
weight_key: "QuantKey | None" = None,
activation_key: "QuantKey | None" = None,
allow_vllm_cutlass: bool = False,
):
super().__init__(layer.moe_config)
super().__init__(moe)
self.weight_block_size = weight_block_size
self.block_quant: bool = self.weight_block_size is not None
self.weight_scale_name = (
@@ -555,11 +556,11 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(
weight_block_size=None,
layer=layer,
moe=moe,
)
def process_weights_after_loading(self, layer: Module) -> None:
@@ -612,11 +613,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(
weight_block_size=[128, 128],
layer=layer,
moe=moe,
)
def maybe_roundup_sizes(
@@ -716,13 +717,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
super().__init__(
weight_block_size=None,
layer=layer,
moe=moe,
weight_key=kFp8StaticChannelSym,
activation_key=kFp8DynamicTokenSym,
allow_vllm_cutlass=True,