[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
a11dfcff89
commit
5690b02c03
@@ -24,6 +24,7 @@ from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.determinism.batch_invariant import (
|
||||
linear_batch_invariant,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import resolve_quant_method
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
@@ -282,7 +283,7 @@ class LinearBase(PluggableLayer):
|
||||
self.quant_method: QuantizeMethodBase
|
||||
if quant_config is None:
|
||||
self.quant_method = UnquantizedLinearMethod()
|
||||
elif quant_method := quant_config.get_quant_method(self, prefix=prefix):
|
||||
elif quant_method := resolve_quant_method(quant_config, self, prefix=prefix):
|
||||
self.quant_method = quant_method
|
||||
else:
|
||||
raise ValueError("All linear layers should support quant method.")
|
||||
|
||||
Reference in New Issue
Block a user