[Frontend] new online quantization frontend (#38138)
Signed-off-by: Vasiliy Kuznetsov <vasiliy@meta.com>
This commit is contained in:
@@ -112,6 +112,7 @@ from vllm.v1.sample.logits_processor import LogitsProcessor
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.quantization import OnlineQuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.model_loader import LoadFormats
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
@@ -483,6 +484,7 @@ class EngineArgs:
|
||||
hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")
|
||||
tokenizer_revision: str | None = ModelConfig.tokenizer_revision
|
||||
quantization: QuantizationMethods | str | None = ModelConfig.quantization
|
||||
quantization_config: "dict[str, Any] | OnlineQuantizationConfigArgs | None" = None
|
||||
allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
|
||||
enforce_eager: bool = ModelConfig.enforce_eager
|
||||
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
|
||||
@@ -661,6 +663,12 @@ class EngineArgs:
|
||||
if isinstance(self.ir_op_priority, dict):
|
||||
self.ir_op_priority = IrOpPriorityConfig(**self.ir_op_priority)
|
||||
|
||||
from vllm.config.quantization import resolve_online_quant_config
|
||||
|
||||
self.quantization_config = resolve_online_quant_config(
|
||||
self.quantization, self.quantization_config
|
||||
)
|
||||
|
||||
# Setup plugins
|
||||
from vllm.plugins import load_general_plugins
|
||||
|
||||
@@ -1431,6 +1439,7 @@ class EngineArgs:
|
||||
tokenizer_revision=self.tokenizer_revision,
|
||||
max_model_len=self.max_model_len,
|
||||
quantization=self.quantization,
|
||||
quantization_config=self.quantization_config,
|
||||
allow_deprecated_quantization=self.allow_deprecated_quantization,
|
||||
enforce_eager=self.enforce_eager,
|
||||
enable_return_routed_experts=self.enable_return_routed_experts,
|
||||
|
||||
Reference in New Issue
Block a user