Remove unused kwargs from model definitions (#13555)
This commit is contained in:
@@ -22,7 +22,7 @@ from torch import nn
|
||||
from transformers import AutoModel, PreTrainedModel
|
||||
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
|
||||
from vllm.attention import Attention, AttentionMetadata
|
||||
from vllm.attention import Attention
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.utils import divide
|
||||
@@ -59,7 +59,6 @@ def vllm_flash_attention_forward(
|
||||
# Transformers kwargs
|
||||
scaling: Optional[float] = None,
|
||||
# vLLM kwargs
|
||||
attn_metadata: Optional[AttentionMetadata] = None,
|
||||
attention_instances: Optional[list[Attention]] = None,
|
||||
**kwargs):
|
||||
self_attn = attention_instances[module.layer_idx]
|
||||
@@ -68,12 +67,7 @@ def vllm_flash_attention_forward(
|
||||
hidden = query.shape[-2]
|
||||
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
|
||||
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
|
||||
return self_attn.forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
kv_cache=None, # argument not used
|
||||
attn_metadata=attn_metadata), None
|
||||
return self_attn.forward(query, key, value), None
|
||||
|
||||
|
||||
ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_flash_attention_forward
|
||||
@@ -251,8 +245,6 @@ class TransformersModel(nn.Module, SupportsQuant):
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
kv_caches: list[torch.Tensor], # argument not used
|
||||
attn_metadata: AttentionMetadata,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
@@ -260,7 +252,6 @@ class TransformersModel(nn.Module, SupportsQuant):
|
||||
input_ids[None, ...],
|
||||
use_cache=False,
|
||||
position_ids=positions[None, ...],
|
||||
attn_metadata=attn_metadata,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
attention_instances=self.attention_instances,
|
||||
return_dict=False)[0][0, ...] # we remove batch dimension for now
|
||||
|
||||
Reference in New Issue
Block a user