[Frontend] new online quantization frontend (#38138)
Signed-off-by: Vasiliy Kuznetsov <vasiliy@meta.com>
This commit is contained in:
@@ -21,6 +21,7 @@ from vllm.config.multimodal import (
|
||||
MultiModalConfig,
|
||||
)
|
||||
from vllm.config.pooler import PoolerConfig
|
||||
from vllm.config.quantization import OnlineQuantizationConfigArgs
|
||||
from vllm.config.scheduler import RunnerType
|
||||
from vllm.config.utils import config, getattr_iter
|
||||
from vllm.logger import init_logger
|
||||
@@ -199,6 +200,10 @@ class ModelConfig:
|
||||
`quantization_config` attribute in the model config file. If that is
|
||||
`None`, we assume the model weights are not quantized and use `dtype` to
|
||||
determine the data type of the weights."""
|
||||
quantization_config: dict[str, Any] | OnlineQuantizationConfigArgs | None = None
|
||||
"""Arguments for online quantization.
|
||||
Auto-created when `quantization` equals to one of the string values of
|
||||
the `OnlineQuantScheme` enum."""
|
||||
allow_deprecated_quantization: bool = False
|
||||
"""Whether to allow deprecated quantization methods."""
|
||||
enforce_eager: bool = False
|
||||
|
||||
Reference in New Issue
Block a user