Remove unused kwargs from model definitions (#13555)
This commit is contained in:
@@ -939,8 +939,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
hidden_states = self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
kv_caches=self.kv_caches,
|
||||
attn_metadata=None,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
@@ -1137,11 +1135,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def _dummy_run(
|
||||
self,
|
||||
num_tokens: int,
|
||||
kv_caches: Optional[List[torch.Tensor]] = None,
|
||||
) -> torch.Tensor:
|
||||
model = self.model
|
||||
if kv_caches is None:
|
||||
kv_caches = self.kv_caches
|
||||
if self.is_multimodal_model:
|
||||
input_ids = None
|
||||
inputs_embeds = self.inputs_embeds[:num_tokens]
|
||||
@@ -1172,26 +1167,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
hidden_states = model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=None,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def profile_run(self) -> None:
|
||||
# use an empty tensor instead of `None`` to force Dynamo to pass
|
||||
# it by reference, rather by specializing on the value `None`.
|
||||
# the `dtype` argument does not matter, and we use `float32` as
|
||||
# a placeholder (it has wide hardware support).
|
||||
# it is important to create tensors inside the loop, rather than
|
||||
# multiplying the list, to avoid Dynamo from treating them as
|
||||
# tensor aliasing.
|
||||
dummy_kv_caches = [
|
||||
torch.tensor((), dtype=torch.float32, device=self.device)
|
||||
for _ in range(self.num_attn_layers)
|
||||
]
|
||||
|
||||
# Profile with multimodal encoder & encoder cache.
|
||||
# TODO: handle encoder-decoder models once we support them.
|
||||
if (self.is_multimodal_model and self.max_num_encoder_input_tokens > 0
|
||||
@@ -1302,8 +1283,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
with self.maybe_profile_with_lora(self.lora_config,
|
||||
num_scheduled_tokens):
|
||||
# Trigger compilation for general shape.
|
||||
hidden_states = self._dummy_run(self.max_num_tokens,
|
||||
dummy_kv_caches)
|
||||
hidden_states = self._dummy_run(self.max_num_tokens)
|
||||
if get_pp_group().is_last_rank:
|
||||
hidden_states = hidden_states[logit_indices]
|
||||
logits = self.model.compute_logits(hidden_states, None)
|
||||
|
||||
@@ -13,11 +13,10 @@ import torch.nn as nn
|
||||
import torch_xla.core.xla_model as xm
|
||||
import torch_xla.runtime as xr
|
||||
|
||||
from vllm.attention import AttentionMetadata
|
||||
from vllm.attention.backends.abstract import AttentionType
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.sampling_params import SamplingType
|
||||
@@ -623,7 +622,6 @@ class TPUModelRunner:
|
||||
assert self.model is not None
|
||||
selected_token_ids = self.model(prompt_data.input_tokens,
|
||||
prompt_data.input_positions,
|
||||
prompt_data.attn_metadata,
|
||||
self.kv_caches)
|
||||
|
||||
# In parallel to TPU execution, prepare the next iteration
|
||||
@@ -662,7 +660,6 @@ class TPUModelRunner:
|
||||
assert self.model is not None
|
||||
selected_token_ids = self.model(decode_data.input_tokens,
|
||||
decode_data.input_positions,
|
||||
decode_data.attn_metadata,
|
||||
self.kv_caches)
|
||||
|
||||
# Transfer sampled tokens from TPU to CPU
|
||||
@@ -839,7 +836,7 @@ class TPUModelRunner:
|
||||
|
||||
with set_forward_context(attn_metadata, self.vllm_config, 0):
|
||||
assert self.model is not None
|
||||
self.model(token_ids, position_ids, attn_metadata, kv_caches)
|
||||
self.model(token_ids, position_ids, kv_caches)
|
||||
|
||||
def capture_model(self) -> None:
|
||||
"""Compile the model."""
|
||||
@@ -963,7 +960,6 @@ class ModelWrapperV1(nn.Module):
|
||||
self,
|
||||
token_ids: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
|
||||
) -> torch.Tensor:
|
||||
"""Executes the forward pass of the model and samples the next token.
|
||||
@@ -971,7 +967,6 @@ class ModelWrapperV1(nn.Module):
|
||||
Args:
|
||||
token_ids: The input token IDs of shape [batch_size, seq_len].
|
||||
position_ids: The input position IDs of shape [batch_size, seq_len].
|
||||
attn_metadata: The Pallas attention metadata.
|
||||
input_lens: The actual input lengths of shape [batch_size].
|
||||
t: The sampling temperature of shape [batch_size].
|
||||
p: The top-p probability of shape [batch_size].
|
||||
@@ -980,7 +975,8 @@ class ModelWrapperV1(nn.Module):
|
||||
memory profiling at initialization.
|
||||
"""
|
||||
# Skip this in memory profiling at initialization.
|
||||
if attn_metadata is not None and kv_caches[0][0].numel() > 0:
|
||||
if kv_caches[0][0].numel() > 0:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
# index_copy_(slot_mapping) only works when the inserted dimension
|
||||
# is 0. However, the KV cache in the Pallas backend has the shape
|
||||
# [num_kv_heads, num_blocks, block_size, head_size]. To make it
|
||||
@@ -1001,12 +997,7 @@ class ModelWrapperV1(nn.Module):
|
||||
attn_metadata.slot_mapping = slot_mapping
|
||||
|
||||
assert self.model is not None
|
||||
hidden_states = self.model(
|
||||
token_ids,
|
||||
position_ids,
|
||||
kv_caches,
|
||||
attn_metadata,
|
||||
)
|
||||
hidden_states = self.model(token_ids, position_ids)
|
||||
|
||||
hidden_states = hidden_states.flatten(0, 1)
|
||||
logits = self.model.compute_logits(hidden_states, None)
|
||||
|
||||
Reference in New Issue
Block a user