[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
a11dfcff89
commit
5690b02c03
@@ -23,6 +23,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
select_fp8_moe_backend,
|
||||
)
|
||||
@@ -111,7 +112,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
|
||||
return is_row_parallel or (reduces_output_dim and is_column_parallel)
|
||||
|
||||
|
||||
class _Fp8OnlineLinearBase(LinearMethodBase):
|
||||
class OnlineLinearBase(LinearMethodBase):
|
||||
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
@@ -155,7 +156,7 @@ class _Fp8OnlineLinearBase(LinearMethodBase):
|
||||
initialize_online_processing(layer)
|
||||
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
|
||||
@@ -256,7 +257,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
|
||||
class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online blockwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
|
||||
@@ -336,7 +337,7 @@ class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
)
|
||||
|
||||
|
||||
class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online PTPC FP8 linear quantization.
|
||||
|
||||
Per-output-channel weight scale + dynamic per-token activation scale. The
|
||||
@@ -443,12 +444,12 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
self,
|
||||
*,
|
||||
weight_block_size: list[int] | None,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
weight_key: "QuantKey | None" = None,
|
||||
activation_key: "QuantKey | None" = None,
|
||||
allow_vllm_cutlass: bool = False,
|
||||
):
|
||||
super().__init__(layer.moe_config)
|
||||
super().__init__(moe)
|
||||
self.weight_block_size = weight_block_size
|
||||
self.block_quant: bool = self.weight_block_size is not None
|
||||
self.weight_scale_name = (
|
||||
@@ -555,11 +556,11 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(
|
||||
weight_block_size=None,
|
||||
layer=layer,
|
||||
moe=moe,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
@@ -612,11 +613,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(
|
||||
weight_block_size=[128, 128],
|
||||
layer=layer,
|
||||
moe=moe,
|
||||
)
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
@@ -716,13 +717,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
||||
|
||||
super().__init__(
|
||||
weight_block_size=None,
|
||||
layer=layer,
|
||||
moe=moe,
|
||||
weight_key=kFp8StaticChannelSym,
|
||||
activation_key=kFp8DynamicTokenSym,
|
||||
allow_vllm_cutlass=True,
|
||||
|
||||
Reference in New Issue
Block a user