[Cleanup] Remove online quantization support in fp8.py in favor of online shorthands (#53585)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fxmarty-amd
2026-09-24 12:48:14 -07:00
committed by GitHub
co-authored by mgoin mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 58aa2f1ef1
commit 90a9515006
15 changed files with 240 additions and 269 deletions
+12 -27
View File
@@ -93,7 +93,7 @@ class Fp8Config(QuantizationConfig):
def __init__(
self,
is_checkpoint_fp8_serialized: bool = False,
is_checkpoint_fp8_serialized: bool = True,
activation_scheme: str = "dynamic",
ignored_layers: list[str] | None = None,
weight_block_size: list[int] | None = None,
@@ -102,6 +102,13 @@ class Fp8Config(QuantizationConfig):
super().__init__()
self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized
if not is_checkpoint_fp8_serialized:
raise ValueError(
"The `fp8` quantization method no longer supports online "
"quantization. Please use `--quantization fp8_per_tensor` "
"instead. See "
"https://docs.vllm.ai/en/stable/features/quantization/online/"
)
if activation_scheme not in ACTIVATION_SCHEMES:
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
@@ -112,11 +119,6 @@ class Fp8Config(QuantizationConfig):
)
self.store_dtype = store_dtype
if weight_block_size is not None:
if not is_checkpoint_fp8_serialized:
raise ValueError(
"The block-wise quantization only supports fp8-serialized "
"checkpoint for now."
)
if len(weight_block_size) != 2:
raise ValueError(
"The quantization block size of weight must have 2 "
@@ -182,18 +184,9 @@ class Fp8Config(QuantizationConfig):
match_mode=self.ignored_layers_match_mode,
):
return UnquantizedLinearMethod()
if not self.is_checkpoint_fp8_serialized:
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineLinearMethod,
)
online_method = Fp8PerTensorOnlineLinearMethod()
online_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
return online_method
else:
offline_method = Fp8LinearMethod(self)
offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
return offline_method
offline_method = Fp8LinearMethod(self)
offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
return offline_method
elif isinstance(layer, RoutedExperts):
if is_layer_skipped(
prefix=prefix,
@@ -208,14 +201,7 @@ class Fp8Config(QuantizationConfig):
)
return Mxfp4MoEMethod(layer.moe_config)
if self.is_checkpoint_fp8_serialized:
return Fp8MoEMethod(self, layer)
else:
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineMoEMethod,
)
return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config)
return Fp8MoEMethod(self, layer)
elif isinstance(layer, Attention):
return Fp8KVCacheMethod(self)
return None
@@ -548,7 +534,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
layer.orig_dtype = params_dtype
layer.weight_block_size = None
assert self.quant_config.is_checkpoint_fp8_serialized
params_dtype = torch.float8_e4m3fn
if self.block_quant:
@@ -195,7 +195,11 @@ def get_quant_config(
if model_config.quantization is None:
raise ValueError("Model quantization method is not specified in the config.")
quant_cls = get_quantization_config(model_config.quantization)
from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs
from vllm.config.quantization import (
_ONLINE_SHORTHANDS,
QuantizationConfigArgs,
resolve_quantization_config,
)
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
@@ -335,6 +339,14 @@ def get_quant_config(
# If the quantization config is not found, use the default config.
if not possible_config_filenames:
if model_config.quantization == "fp8":
logger.warning_once(
"--quantization fp8 is deprecated for online quantization; "
"use --quantization fp8_per_tensor instead."
)
fp8_args = resolve_quantization_config("fp8_per_tensor", online_args)
assert fp8_args is not None
return OnlineQuantizationConfig(args=fp8_args)
if model_config.quantization in _ONLINE_SHORTHANDS:
args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]
assert isinstance(args, QuantizationConfigArgs)
+13 -1
View File
@@ -9,6 +9,12 @@ from transformers import PreTrainedConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.transformers_utils.processors.internvl import (
InternVLImageProcessor,
@@ -71,7 +77,12 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
) -> None:
super()._patch_quant_config(config, quant_config)
# ignore vit layers to preserve model performance
if isinstance(quant_config, Fp8Config):
if isinstance(quant_config, Fp8Config) or (
isinstance(quant_config, OnlineQuantizationConfig)
and quant_config.args.linear is not None
and quant_config.args.linear.weight
in (kFp8StaticTensorSym, kFp8Static128BlockSym, kFp8StaticChannelSym)
):
_FP8_IGNORED_LAYERS = [
*(
layer
@@ -87,6 +98,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
"mlp1.1",
"mlp1.3",
]
quant_config.ignored_layers = list(quant_config.ignored_layers)
for layer in _FP8_IGNORED_LAYERS:
if layer not in quant_config.ignored_layers:
quant_config.ignored_layers.append(layer)