[Cleanup] Remove online quantization support in fp8.py in favor of online shorthands (#53585)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mgoin
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
58aa2f1ef1
commit
90a9515006
@@ -9,6 +9,12 @@ from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
InternVLImageProcessor,
|
||||
@@ -71,7 +77,12 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
|
||||
) -> None:
|
||||
super()._patch_quant_config(config, quant_config)
|
||||
# ignore vit layers to preserve model performance
|
||||
if isinstance(quant_config, Fp8Config):
|
||||
if isinstance(quant_config, Fp8Config) or (
|
||||
isinstance(quant_config, OnlineQuantizationConfig)
|
||||
and quant_config.args.linear is not None
|
||||
and quant_config.args.linear.weight
|
||||
in (kFp8StaticTensorSym, kFp8Static128BlockSym, kFp8StaticChannelSym)
|
||||
):
|
||||
_FP8_IGNORED_LAYERS = [
|
||||
*(
|
||||
layer
|
||||
@@ -87,6 +98,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
|
||||
"mlp1.1",
|
||||
"mlp1.3",
|
||||
]
|
||||
quant_config.ignored_layers = list(quant_config.ignored_layers)
|
||||
for layer in _FP8_IGNORED_LAYERS:
|
||||
if layer not in quant_config.ignored_layers:
|
||||
quant_config.ignored_layers.append(layer)
|
||||
|
||||
Reference in New Issue
Block a user