[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
a11dfcff89
commit
5690b02c03
@@ -69,7 +69,10 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_and_maybe_dequant_weights,
|
||||
)
|
||||
@@ -462,7 +465,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
|
||||
quant_method = (
|
||||
self.quant_config.get_quant_method(self, prefix=self.layer_name)
|
||||
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
|
||||
if self.quant_config
|
||||
else None
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user