[Cleanup] Remove online quantization support in fp8.py in favor of online shorthands (#53585)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mgoin
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
58aa2f1ef1
commit
90a9515006
@@ -93,7 +93,7 @@ class Fp8Config(QuantizationConfig):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
is_checkpoint_fp8_serialized: bool = False,
|
||||
is_checkpoint_fp8_serialized: bool = True,
|
||||
activation_scheme: str = "dynamic",
|
||||
ignored_layers: list[str] | None = None,
|
||||
weight_block_size: list[int] | None = None,
|
||||
@@ -102,6 +102,13 @@ class Fp8Config(QuantizationConfig):
|
||||
super().__init__()
|
||||
|
||||
self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized
|
||||
if not is_checkpoint_fp8_serialized:
|
||||
raise ValueError(
|
||||
"The `fp8` quantization method no longer supports online "
|
||||
"quantization. Please use `--quantization fp8_per_tensor` "
|
||||
"instead. See "
|
||||
"https://docs.vllm.ai/en/stable/features/quantization/online/"
|
||||
)
|
||||
|
||||
if activation_scheme not in ACTIVATION_SCHEMES:
|
||||
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
|
||||
@@ -112,11 +119,6 @@ class Fp8Config(QuantizationConfig):
|
||||
)
|
||||
self.store_dtype = store_dtype
|
||||
if weight_block_size is not None:
|
||||
if not is_checkpoint_fp8_serialized:
|
||||
raise ValueError(
|
||||
"The block-wise quantization only supports fp8-serialized "
|
||||
"checkpoint for now."
|
||||
)
|
||||
if len(weight_block_size) != 2:
|
||||
raise ValueError(
|
||||
"The quantization block size of weight must have 2 "
|
||||
@@ -182,18 +184,9 @@ class Fp8Config(QuantizationConfig):
|
||||
match_mode=self.ignored_layers_match_mode,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
if not self.is_checkpoint_fp8_serialized:
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
|
||||
online_method = Fp8PerTensorOnlineLinearMethod()
|
||||
online_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
|
||||
return online_method
|
||||
else:
|
||||
offline_method = Fp8LinearMethod(self)
|
||||
offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
|
||||
return offline_method
|
||||
offline_method = Fp8LinearMethod(self)
|
||||
offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
|
||||
return offline_method
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
if is_layer_skipped(
|
||||
prefix=prefix,
|
||||
@@ -208,14 +201,7 @@ class Fp8Config(QuantizationConfig):
|
||||
)
|
||||
|
||||
return Mxfp4MoEMethod(layer.moe_config)
|
||||
if self.is_checkpoint_fp8_serialized:
|
||||
return Fp8MoEMethod(self, layer)
|
||||
else:
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
)
|
||||
|
||||
return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config)
|
||||
return Fp8MoEMethod(self, layer)
|
||||
elif isinstance(layer, Attention):
|
||||
return Fp8KVCacheMethod(self)
|
||||
return None
|
||||
@@ -548,7 +534,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
|
||||
assert self.quant_config.is_checkpoint_fp8_serialized
|
||||
params_dtype = torch.float8_e4m3fn
|
||||
|
||||
if self.block_quant:
|
||||
|
||||
@@ -195,7 +195,11 @@ def get_quant_config(
|
||||
if model_config.quantization is None:
|
||||
raise ValueError("Model quantization method is not specified in the config.")
|
||||
quant_cls = get_quantization_config(model_config.quantization)
|
||||
from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs
|
||||
from vllm.config.quantization import (
|
||||
_ONLINE_SHORTHANDS,
|
||||
QuantizationConfigArgs,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
@@ -335,6 +339,14 @@ def get_quant_config(
|
||||
|
||||
# If the quantization config is not found, use the default config.
|
||||
if not possible_config_filenames:
|
||||
if model_config.quantization == "fp8":
|
||||
logger.warning_once(
|
||||
"--quantization fp8 is deprecated for online quantization; "
|
||||
"use --quantization fp8_per_tensor instead."
|
||||
)
|
||||
fp8_args = resolve_quantization_config("fp8_per_tensor", online_args)
|
||||
assert fp8_args is not None
|
||||
return OnlineQuantizationConfig(args=fp8_args)
|
||||
if model_config.quantization in _ONLINE_SHORTHANDS:
|
||||
args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]
|
||||
assert isinstance(args, QuantizationConfigArgs)
|
||||
|
||||
@@ -9,6 +9,12 @@ from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
InternVLImageProcessor,
|
||||
@@ -71,7 +77,12 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
|
||||
) -> None:
|
||||
super()._patch_quant_config(config, quant_config)
|
||||
# ignore vit layers to preserve model performance
|
||||
if isinstance(quant_config, Fp8Config):
|
||||
if isinstance(quant_config, Fp8Config) or (
|
||||
isinstance(quant_config, OnlineQuantizationConfig)
|
||||
and quant_config.args.linear is not None
|
||||
and quant_config.args.linear.weight
|
||||
in (kFp8StaticTensorSym, kFp8Static128BlockSym, kFp8StaticChannelSym)
|
||||
):
|
||||
_FP8_IGNORED_LAYERS = [
|
||||
*(
|
||||
layer
|
||||
@@ -87,6 +98,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
|
||||
"mlp1.1",
|
||||
"mlp1.3",
|
||||
]
|
||||
quant_config.ignored_layers = list(quant_config.ignored_layers)
|
||||
for layer in _FP8_IGNORED_LAYERS:
|
||||
if layer not in quant_config.ignored_layers:
|
||||
quant_config.ignored_layers.append(layer)
|
||||
|
||||
Reference in New Issue
Block a user