Remove unused kwargs from model definitions (#13555)
This commit is contained in:
@@ -297,10 +297,6 @@ class CPUEncoderDecoderModelRunner(
|
||||
model_input.encoder_input_tokens,
|
||||
"encoder_positions":
|
||||
model_input.encoder_input_positions,
|
||||
"kv_caches":
|
||||
kv_caches,
|
||||
"attn_metadata":
|
||||
model_input.attn_metadata,
|
||||
**MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
|
||||
device=self.device),
|
||||
"intermediate_tensors":
|
||||
|
||||
@@ -654,8 +654,6 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
|
||||
hidden_states = model_executable(
|
||||
input_ids=model_input.input_tokens,
|
||||
positions=model_input.input_positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=model_input.attn_metadata,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
**execute_model_kwargs,
|
||||
**multimodal_kwargs,
|
||||
|
||||
@@ -41,16 +41,6 @@ class CPUPoolingModelRunner(
|
||||
raise ValueError(
|
||||
"CPU worker does not support multi-step execution.")
|
||||
|
||||
num_layers = self.model_config.get_num_layers(self.parallel_config)
|
||||
# use an empty tensor instead of `None`` to force Dynamo to pass
|
||||
# it by reference, rather by specializing on the value ``None``.
|
||||
# the `dtype` argument does not matter, and we use `float32` as
|
||||
# a placeholder (it has wide hardware support).
|
||||
kv_caches = [
|
||||
torch.tensor([], dtype=torch.float32, device=self.device)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
|
||||
model_executable = self.model
|
||||
cross_enc_kwargs = {}
|
||||
if model_input.token_type_ids is not None:
|
||||
@@ -60,10 +50,6 @@ class CPUPoolingModelRunner(
|
||||
model_input.input_tokens,
|
||||
"positions":
|
||||
model_input.input_positions,
|
||||
"kv_caches":
|
||||
kv_caches,
|
||||
"attn_metadata":
|
||||
model_input.attn_metadata,
|
||||
**MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
|
||||
device=self.device),
|
||||
**cross_enc_kwargs,
|
||||
|
||||
@@ -184,8 +184,6 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
|
||||
positions=model_input.input_positions,
|
||||
encoder_input_ids=model_input.encoder_input_tokens,
|
||||
encoder_positions=model_input.encoder_input_positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=model_input.attn_metadata,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
**MultiModalKwargs.as_kwargs(multi_modal_kwargs,
|
||||
device=self.device),
|
||||
@@ -324,21 +322,11 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
|
||||
or encoder_dummy_data.multi_modal_placeholders)
|
||||
seqs.append(seq)
|
||||
|
||||
# Run the model with the dummy inputs.
|
||||
num_layers = self.model_config.get_num_layers(self.parallel_config)
|
||||
# use an empty tensor instead of `None`` to force Dynamo to pass
|
||||
# it by reference, rather by specializing on the value ``None``.
|
||||
# the `dtype` argument does not matter, and we use `float32` as
|
||||
# a placeholder (it has wide hardware support).
|
||||
kv_caches = [
|
||||
torch.tensor([], dtype=torch.float32, device=self.device)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
finished_requests_ids = [seq.request_id for seq in seqs]
|
||||
model_input = self.prepare_model_input(
|
||||
seqs, finished_requests_ids=finished_requests_ids)
|
||||
intermediate_tensors = None
|
||||
self.execute_model(model_input, kv_caches, intermediate_tensors)
|
||||
self.execute_model(model_input, None, intermediate_tensors)
|
||||
torch.cuda.synchronize()
|
||||
return
|
||||
|
||||
|
||||
@@ -384,11 +384,12 @@ class HpuModelAdapter:
|
||||
if 'virtual_engine' in kwargs:
|
||||
virtual_engine = kwargs.pop('virtual_engine')
|
||||
input_ids = kwargs['input_ids']
|
||||
kwargs['attn_metadata'] = self._update_metadata(
|
||||
kwargs['attn_metadata'], input_ids.size(0), input_ids.size(1),
|
||||
input_ids.device, self.dtype)
|
||||
attn_metadata = self._update_metadata(kwargs.pop('attn_metadata'),
|
||||
input_ids.size(0),
|
||||
input_ids.size(1),
|
||||
input_ids.device, self.dtype)
|
||||
LoraMask.setLoraMask(kwargs.pop('lora_mask'))
|
||||
with set_forward_context(kwargs['attn_metadata'], self.vllm_config,
|
||||
with set_forward_context(attn_metadata, self.vllm_config,
|
||||
virtual_engine):
|
||||
hidden_states = self.model(*args, **kwargs)
|
||||
hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
|
||||
@@ -1346,15 +1347,13 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
|
||||
max_seq_len = self.bucketing_global_state.prompt_seq_bucket_cfg[-1]
|
||||
max_batch_size = min(self.max_num_batched_tokens // max_seq_len,
|
||||
self.scheduler_config.max_num_seqs)
|
||||
self.warmup_scenario(max_batch_size, max_seq_len, True, kv_caches,
|
||||
False, True)
|
||||
self.warmup_scenario(max_batch_size, max_seq_len, True, False, True)
|
||||
return
|
||||
|
||||
def warmup_scenario(self,
|
||||
batch_size,
|
||||
seq_len,
|
||||
is_prompt,
|
||||
kv_caches,
|
||||
is_pt_profiler_run=False,
|
||||
is_lora_profile_run=False) -> None:
|
||||
use_graphs = self._use_graphs(batch_size, seq_len, is_prompt)
|
||||
@@ -1418,7 +1417,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
|
||||
profiler.start()
|
||||
for _ in range(times):
|
||||
inputs = self.prepare_model_input(seqs)
|
||||
self.execute_model(inputs, kv_caches, warmup_mode=True)
|
||||
self.execute_model(inputs, None, warmup_mode=True)
|
||||
torch.hpu.synchronize()
|
||||
if profiler:
|
||||
profiler.step()
|
||||
@@ -1470,17 +1469,16 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
|
||||
f"free_mem:{free_mem}")
|
||||
logger.info(msg)
|
||||
|
||||
def warmup_all_buckets(self, buckets, is_prompt, kv_caches):
|
||||
def warmup_all_buckets(self, buckets, is_prompt):
|
||||
for i, (batch_size, seq_len) in enumerate(reversed(buckets)):
|
||||
self.log_warmup('Prompt' if is_prompt else 'Decode', i,
|
||||
len(buckets), batch_size, seq_len)
|
||||
self.warmup_scenario(batch_size, seq_len, is_prompt, kv_caches)
|
||||
self.warmup_scenario(batch_size, seq_len, is_prompt)
|
||||
|
||||
def warmup_graphs(self,
|
||||
strategy,
|
||||
buckets,
|
||||
is_prompt,
|
||||
kv_caches,
|
||||
available_mem,
|
||||
starting_mem=0,
|
||||
total_batch_seq=0.001):
|
||||
@@ -1512,7 +1510,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
|
||||
self.graphed_buckets.add(graphed_bucket)
|
||||
self.log_warmup(phase, idx, num_candidates, batch_size, seq_len)
|
||||
with HabanaMemoryProfiler() as mem_prof:
|
||||
self.warmup_scenario(batch_size, seq_len, is_prompt, kv_caches)
|
||||
self.warmup_scenario(batch_size, seq_len, is_prompt)
|
||||
used_mem = align_workers(mem_prof.consumed_device_memory,
|
||||
torch.distributed.ReduceOp.MAX)
|
||||
available_mem -= used_mem
|
||||
@@ -1542,8 +1540,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
|
||||
graphs = graph == 't'
|
||||
if graphs:
|
||||
self.graphed_buckets.add((int(bs), int(seq_len), is_prompt))
|
||||
self.warmup_scenario(int(bs), int(seq_len), is_prompt, kv_caches,
|
||||
True)
|
||||
self.warmup_scenario(int(bs), int(seq_len), is_prompt, True)
|
||||
raise AssertionError("Finished profiling")
|
||||
if self.skip_warmup:
|
||||
logger.info("Skipping warmup...")
|
||||
@@ -1608,9 +1605,9 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
|
||||
with compile_only_mode_context(
|
||||
) if can_use_compile_only_mode else contextlib.nullcontext():
|
||||
self.warmup_all_buckets(self.bucketing_global_state.prompt_buckets,
|
||||
True, kv_caches)
|
||||
True)
|
||||
self.warmup_all_buckets(self.bucketing_global_state.decode_buckets,
|
||||
False, kv_caches)
|
||||
False)
|
||||
|
||||
if not self.enforce_eager and htorch.utils.internal.is_lazy():
|
||||
assert self.mem_margin is not None, \
|
||||
@@ -1641,11 +1638,11 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
|
||||
mem_post_prompt, prompt_batch_seq, prompt_captured_all = \
|
||||
self.warmup_graphs(
|
||||
prompt_strategy, self.bucketing_global_state.prompt_buckets,
|
||||
True, kv_caches, prompt_available_memory)
|
||||
True, prompt_available_memory)
|
||||
mem_post_decode, decode_batch_seq, decode_captured_all = \
|
||||
self.warmup_graphs(
|
||||
decode_strategy, self.bucketing_global_state.decode_buckets,
|
||||
False, kv_caches, decode_available_memory)
|
||||
False, decode_available_memory)
|
||||
|
||||
# Not all prompt buckets were captured, but all decode buckets
|
||||
# were captured and we have some free graph-allocated space
|
||||
@@ -1656,7 +1653,6 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
|
||||
self.warmup_graphs(
|
||||
prompt_strategy,
|
||||
self.bucketing_global_state.prompt_buckets, True,
|
||||
kv_caches,
|
||||
graph_free_mem - mem_post_prompt - mem_post_decode,
|
||||
mem_post_prompt, prompt_batch_seq))
|
||||
|
||||
@@ -1669,7 +1665,6 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
|
||||
mem_post_decode, _, _ = self.warmup_graphs(
|
||||
decode_strategy,
|
||||
self.bucketing_global_state.decode_buckets, False,
|
||||
kv_caches,
|
||||
graph_free_mem - mem_post_prompt - mem_post_decode,
|
||||
mem_post_decode, decode_batch_seq)
|
||||
|
||||
@@ -1982,7 +1977,6 @@ class HPUModelRunner(HPUModelRunnerBase[ModelInputForHPUWithSamplingMetadata]):
|
||||
execute_model_kwargs = {
|
||||
"input_ids": input_tokens,
|
||||
"positions": input_positions,
|
||||
"kv_caches": kv_caches,
|
||||
"attn_metadata": self.trim_attn_metadata(attn_metadata),
|
||||
"intermediate_tensors": intermediate_tensors,
|
||||
"lora_mask": lora_mask,
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.core.scheduler import SchedulerOutputs
|
||||
from vllm.distributed import get_kv_transfer_group, get_pp_group
|
||||
from vllm.distributed.parallel_state import (get_tensor_model_parallel_rank,
|
||||
graph_capture)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.inputs import INPUT_REGISTRY, InputRegistry
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.layers import LoRAMapping
|
||||
@@ -1727,8 +1727,6 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
|
||||
hidden_or_intermediate_states = model_executable(
|
||||
input_ids=model_input.input_tokens,
|
||||
positions=model_input.input_positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=model_input.attn_metadata,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
**MultiModalKwargs.as_kwargs(multi_modal_kwargs,
|
||||
device=self.device),
|
||||
@@ -1913,8 +1911,6 @@ class CUDAGraphRunner(nn.Module):
|
||||
self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=attn_metadata,
|
||||
intermediate_tensors=intermediate_inputs,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -1927,8 +1923,6 @@ class CUDAGraphRunner(nn.Module):
|
||||
output_hidden_or_intermediate_states = self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=attn_metadata,
|
||||
intermediate_tensors=intermediate_inputs,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -1976,13 +1970,10 @@ class CUDAGraphRunner(nn.Module):
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
kv_caches: List[torch.Tensor],
|
||||
attn_metadata: AttentionMetadata,
|
||||
intermediate_tensors: Optional[IntermediateTensors],
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
# KV caches are fixed tensors, so we don't need to copy them.
|
||||
del kv_caches
|
||||
attn_metadata: AttentionMetadata = get_forward_context().attn_metadata
|
||||
|
||||
# Copy the input tensors to the input buffers.
|
||||
self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
|
||||
|
||||
@@ -476,7 +476,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
|
||||
# path for warm up runs
|
||||
if not model_input.is_multi_step:
|
||||
return self._base_model_runner.execute_model(
|
||||
frozen_model_input, kv_caches, intermediate_tensors, num_steps)
|
||||
frozen_model_input, None, intermediate_tensors, num_steps)
|
||||
|
||||
# make sure we skip the sampler on the lask rank and only pythonize
|
||||
# if CPU is ahead.
|
||||
@@ -538,7 +538,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
|
||||
|
||||
# Execute the model
|
||||
output = self._base_model_runner.execute_model(frozen_model_input,
|
||||
kv_caches,
|
||||
None,
|
||||
intermediate_tensors,
|
||||
num_steps=1)
|
||||
|
||||
|
||||
@@ -346,10 +346,6 @@ class OpenVINOModelRunner(ModelRunnerBase):
|
||||
input_tokens,
|
||||
"positions":
|
||||
input_positions,
|
||||
"kv_caches":
|
||||
kv_caches,
|
||||
"attn_metadata":
|
||||
attn_metadata,
|
||||
**MultiModalKwargs.as_kwargs(multi_modal_kwargs or {},
|
||||
device=self.device),
|
||||
}
|
||||
|
||||
@@ -91,16 +91,6 @@ class PoolingModelRunner(
|
||||
else:
|
||||
model_executable = self.model
|
||||
|
||||
num_layers = self.model_config.get_num_layers(self.parallel_config)
|
||||
# use an empty tensor instead of `None`` to force Dynamo to pass
|
||||
# it by reference, rather by specializing on the value ``None``.
|
||||
# the `dtype` argument does not matter, and we use `float32` as
|
||||
# a placeholder (it has wide hardware support).
|
||||
kv_caches = [
|
||||
torch.tensor([], dtype=torch.float32, device=self.device)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
|
||||
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
|
||||
seqlen_agnostic_kwargs = {
|
||||
"finished_requests_ids": model_input.finished_requests_ids,
|
||||
@@ -121,8 +111,6 @@ class PoolingModelRunner(
|
||||
hidden_or_intermediate_states = model_executable(
|
||||
input_ids=model_input.input_tokens,
|
||||
positions=model_input.input_positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=model_input.attn_metadata,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
**MultiModalKwargs.as_kwargs(multi_modal_kwargs,
|
||||
device=self.device),
|
||||
|
||||
@@ -15,7 +15,7 @@ import torch_xla.runtime as xr
|
||||
|
||||
from vllm.attention import AttentionMetadata, get_attn_backend
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
@@ -275,8 +275,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
torch._dynamo.mark_dynamic(p, 0)
|
||||
# Dummy run.
|
||||
with set_forward_context(attn_metadata, self.vllm_config, 0):
|
||||
self.model(token_ids, position_ids, attn_metadata, input_lens, t,
|
||||
p, num_samples, kv_caches)
|
||||
self.model(token_ids, position_ids, input_lens, t, p, num_samples,
|
||||
kv_caches)
|
||||
|
||||
def warmup_model(
|
||||
self,
|
||||
@@ -679,8 +679,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
self.vllm_config,
|
||||
model_input.virtual_engine):
|
||||
output_token_ids = self.model(token_ids, position_ids,
|
||||
attn_metadata, input_lens, t,
|
||||
p, model_input.num_samples,
|
||||
input_lens, t, p,
|
||||
model_input.num_samples,
|
||||
kv_caches)
|
||||
next_token_ids.append(output_token_ids[0])
|
||||
start_idx = end_idx
|
||||
@@ -730,8 +730,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
self.vllm_config,
|
||||
model_input.virtual_engine):
|
||||
output_token_ids = self.model(token_ids, position_ids,
|
||||
attn_metadata, input_lens, t,
|
||||
p, model_input.num_samples,
|
||||
input_lens, t, p,
|
||||
model_input.num_samples,
|
||||
kv_caches)
|
||||
self.cached_step_outputs.append(output_token_ids)
|
||||
|
||||
@@ -777,7 +777,6 @@ class ModelWrapper(nn.Module):
|
||||
self,
|
||||
token_ids: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
input_lens: torch.Tensor,
|
||||
t: torch.Tensor,
|
||||
p: torch.Tensor,
|
||||
@@ -789,7 +788,6 @@ class ModelWrapper(nn.Module):
|
||||
Args:
|
||||
token_ids: The input token IDs of shape [batch_size, seq_len].
|
||||
position_ids: The input position IDs of shape [batch_size, seq_len].
|
||||
attn_metadata: The Pallas attention metadata.
|
||||
input_lens: The actual input lengths of shape [batch_size].
|
||||
t: The sampling temperature of shape [batch_size].
|
||||
p: The top-p probability of shape [batch_size].
|
||||
@@ -802,6 +800,7 @@ class ModelWrapper(nn.Module):
|
||||
start_indicies = torch.arange(
|
||||
batch_size, dtype=torch.int32, device=input_lens.device) * seq_len
|
||||
logits_indices = start_indicies + input_lens - 1
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
|
||||
# FIXME(woosuk): This is a temporary hack to avoid using the existing
|
||||
# sampler and sampling metadata.
|
||||
@@ -833,12 +832,7 @@ class ModelWrapper(nn.Module):
|
||||
slot_mapping = slot_mapping.flatten()
|
||||
attn_metadata.slot_mapping = slot_mapping
|
||||
|
||||
hidden_states = self.model(
|
||||
token_ids,
|
||||
position_ids,
|
||||
kv_caches,
|
||||
attn_metadata,
|
||||
)
|
||||
hidden_states = self.model(token_ids, position_ids)
|
||||
hidden_states = hidden_states.flatten(0, 1)
|
||||
logits = self.model.compute_logits(hidden_states, sampling_metadata)
|
||||
|
||||
|
||||
@@ -484,15 +484,6 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
|
||||
multi_modal_placeholders=dummy_data.multi_modal_placeholders)
|
||||
seqs.append(seq)
|
||||
|
||||
# Run the model with the dummy inputs.
|
||||
num_layers = self.model_config.get_num_layers(self.parallel_config)
|
||||
# use an empty tensor instead of `None`` to force Dynamo to pass
|
||||
# it by reference, rather by specializing on the value ``None``.
|
||||
# the `dtype` argument does not matter, and we use `float32` as
|
||||
# a placeholder (it has wide hardware support).
|
||||
kv_caches = [
|
||||
torch.tensor([], dtype=torch.float32, device=self.device)
|
||||
] * num_layers
|
||||
finished_requests_ids = [seq.request_id for seq in seqs]
|
||||
model_input = self.prepare_model_input(
|
||||
seqs, finished_requests_ids=finished_requests_ids)
|
||||
@@ -502,7 +493,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
|
||||
batch_size=batch_size,
|
||||
dtype=self.model_config.dtype,
|
||||
device=self.device)
|
||||
self.execute_model(model_input, kv_caches, intermediate_tensors)
|
||||
self.execute_model(model_input, None, intermediate_tensors)
|
||||
torch.xpu.synchronize()
|
||||
return
|
||||
|
||||
@@ -581,8 +572,6 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
|
||||
hidden_or_intermediate_states = model_executable(
|
||||
input_ids=model_input.input_tokens,
|
||||
positions=model_input.input_positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=model_input.attn_metadata,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
**MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs
|
||||
or {},
|
||||
|
||||
Reference in New Issue
Block a user