[Model] Enable quantization support for transformers backend (#12960)
This commit is contained in:
@@ -28,6 +28,7 @@ from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.utils import divide
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
|
||||
@@ -37,6 +38,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsQuant
|
||||
from .utils import maybe_prefix
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -50,10 +52,10 @@ def vllm_flash_attention_forward(
|
||||
value: torch.Tensor,
|
||||
attention_mask: torch.Tensor,
|
||||
# Transformers kwargs
|
||||
scaling: float = None,
|
||||
scaling: Optional[float] = None,
|
||||
# vLLM kwargs
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
attention_instances: list[Attention] = None,
|
||||
attn_metadata: Optional[AttentionMetadata] = None,
|
||||
attention_instances: Optional[list[Attention]] = None,
|
||||
**kwargs):
|
||||
self_attn = attention_instances[module.layer_idx]
|
||||
if scaling is not None:
|
||||
@@ -99,13 +101,7 @@ def replace_linear_class(
|
||||
vllm_linear_cls = {
|
||||
"colwise": ColumnParallelLinear,
|
||||
"rowwise": RowParallelLinear,
|
||||
}.get(style)
|
||||
|
||||
if vllm_linear_cls is None:
|
||||
logger.warning(
|
||||
"Unsupported parallel style value: %s. "
|
||||
"This layer will not be tensor parallelized.", style)
|
||||
return linear
|
||||
}.get(style, ReplicatedLinear)
|
||||
|
||||
class HFCompatibleLinear(vllm_linear_cls):
|
||||
"""
|
||||
@@ -119,10 +115,11 @@ def replace_linear_class(
|
||||
input_size=linear.in_features,
|
||||
output_size=linear.out_features,
|
||||
bias=linear.bias is not None,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
|
||||
class TransformersModel(nn.Module):
|
||||
class TransformersModel(nn.Module, SupportsQuant):
|
||||
embedding_padding_modules = ["lm_head"]
|
||||
embedding_modules = ["embed_tokens"
|
||||
] # TODO transformers will have a util to get it
|
||||
@@ -133,10 +130,8 @@ class TransformersModel(nn.Module):
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.vocab_size = config.vocab_size
|
||||
self.unpadded_vocab_size = config.vocab_size
|
||||
|
||||
@@ -162,7 +157,7 @@ class TransformersModel(nn.Module):
|
||||
scale=config.head_dim**-0.5,
|
||||
num_kv_heads=divide(config.num_key_value_heads, tp_size),
|
||||
cache_config=cache_config,
|
||||
quant_config=None,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{i}.attn") for i in range(config.num_hidden_layers)
|
||||
]
|
||||
|
||||
@@ -172,7 +167,7 @@ class TransformersModel(nn.Module):
|
||||
# ForCausalLM modifications
|
||||
self.lm_head = ParallelLMHead(config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=None,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"))
|
||||
if config.tie_word_embeddings:
|
||||
self.lm_head.weight = self.model.get_input_embeddings().weight
|
||||
|
||||
Reference in New Issue
Block a user