[Core] Support image processor (#4197)
This commit is contained in:
+62
-58
@@ -1,5 +1,6 @@
|
||||
import time
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
@@ -18,9 +19,9 @@ from vllm.lora.request import LoRARequest
|
||||
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
|
||||
from vllm.model_executor import SamplingMetadata
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
|
||||
SequenceGroupMetadata)
|
||||
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
|
||||
from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
|
||||
is_pin_memory_available, make_tensor_with_pad)
|
||||
|
||||
@@ -44,7 +45,7 @@ class ModelInput(NamedTuple):
|
||||
query_lens: List[int]
|
||||
lora_mapping: Optional[LoRAMapping]
|
||||
lora_requests: Set[LoRARequest]
|
||||
multi_modal_input: Optional[torch.Tensor]
|
||||
multi_modal_kwargs: Dict[str, torch.Tensor]
|
||||
slot_mapping: torch.Tensor
|
||||
num_prefill_tokens: int
|
||||
num_decode_tokens: int
|
||||
@@ -60,7 +61,7 @@ class ModelInput(NamedTuple):
|
||||
query_lens=[],
|
||||
lora_mapping=None,
|
||||
lora_requests=set(),
|
||||
multi_modal_input=None,
|
||||
multi_modal_kwargs={},
|
||||
slot_mapping=torch.empty(0, device=device),
|
||||
num_prefill_tokens=0,
|
||||
num_decode_tokens=0,
|
||||
@@ -122,6 +123,16 @@ class ModelRunner:
|
||||
self.block_size,
|
||||
)
|
||||
|
||||
# Create processor for multi-modal data
|
||||
if self.vision_language_config is not None:
|
||||
self.multi_modal_input_processor = MULTIMODAL_REGISTRY \
|
||||
.create_input_processor(
|
||||
self.model_config,
|
||||
self.vision_language_config,
|
||||
)
|
||||
else:
|
||||
self.multi_modal_input_processor = None
|
||||
|
||||
# Lazy initialization
|
||||
self.model: nn.Module # Set after load_model
|
||||
# Set if the backend is flashinfer.
|
||||
@@ -242,7 +253,8 @@ class ModelRunner:
|
||||
context_lens: List[int] = []
|
||||
query_lens: List[int] = []
|
||||
block_tables: List[List[int]] = []
|
||||
multi_modal_input_list: List[torch.Tensor] = []
|
||||
multi_modal_kwargs_list: Dict[str,
|
||||
List[torch.Tensor]] = defaultdict(list)
|
||||
decode_only = True
|
||||
num_prefills = 0
|
||||
num_prefill_tokens = 0
|
||||
@@ -417,9 +429,17 @@ class ModelRunner:
|
||||
and seq_group_metadata.sampling_params.prompt_logprobs
|
||||
else 1))
|
||||
|
||||
if seq_group_metadata.multi_modal_data:
|
||||
multi_modal_input_list.append(
|
||||
seq_group_metadata.multi_modal_data.data)
|
||||
mm_data = seq_group_metadata.multi_modal_data
|
||||
if mm_data is not None:
|
||||
# Process multi-modal data
|
||||
if self.multi_modal_input_processor is None:
|
||||
raise ValueError(
|
||||
"Multi-modal inputs are only supported by "
|
||||
"vision language models.")
|
||||
|
||||
mm_kwargs = self.multi_modal_input_processor(mm_data)
|
||||
for k, v in mm_kwargs.items():
|
||||
multi_modal_kwargs_list[k].append(v)
|
||||
|
||||
if _is_block_tables_empty(seq_group_metadata.block_tables):
|
||||
# During memory profiling, the block tables are not
|
||||
@@ -508,16 +528,6 @@ class ModelRunner:
|
||||
context_lens_tensor = torch.tensor(context_lens,
|
||||
dtype=torch.int,
|
||||
device=self.device)
|
||||
|
||||
if multi_modal_input_list:
|
||||
assert self.vision_language_config, (
|
||||
"Multi-modal inputs are only supported by "
|
||||
"vision language models.")
|
||||
multi_modal_input = torch.cat(multi_modal_input_list,
|
||||
dim=0).to(self.device)
|
||||
else:
|
||||
multi_modal_input = None
|
||||
|
||||
query_lens_tensor = torch.tensor(query_lens,
|
||||
dtype=torch.long,
|
||||
device=self.device)
|
||||
@@ -614,6 +624,11 @@ class ModelRunner:
|
||||
else:
|
||||
lora_mapping = None
|
||||
|
||||
multi_modal_kwargs = {
|
||||
k: torch.cat(v, dim=0).to(self.device)
|
||||
for k, v in multi_modal_kwargs_list.items()
|
||||
}
|
||||
|
||||
return ModelInput(
|
||||
input_tokens=input_tokens_tensor,
|
||||
input_positions=input_positions_tensor,
|
||||
@@ -622,7 +637,7 @@ class ModelRunner:
|
||||
query_lens=query_lens,
|
||||
lora_mapping=lora_mapping,
|
||||
lora_requests=lora_requests,
|
||||
multi_modal_input=multi_modal_input,
|
||||
multi_modal_kwargs=multi_modal_kwargs,
|
||||
slot_mapping=slot_mapping_tensor,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
@@ -633,7 +648,7 @@ class ModelRunner:
|
||||
self,
|
||||
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
|
||||
Set[LoRARequest], LoRAMapping, torch.Tensor]:
|
||||
Set[LoRARequest], LoRAMapping, Dict[str, torch.Tensor]]:
|
||||
if self.is_driver_worker:
|
||||
assert seq_group_metadata_list is not None
|
||||
# Prepare input tensors.
|
||||
@@ -645,7 +660,7 @@ class ModelRunner:
|
||||
query_lens,
|
||||
lora_mapping,
|
||||
lora_requests,
|
||||
multi_modal_input,
|
||||
multi_modal_kwargs,
|
||||
slot_mapping,
|
||||
num_prefill_tokens,
|
||||
num_decode_tokens,
|
||||
@@ -662,7 +677,7 @@ class ModelRunner:
|
||||
sampling_metadata.selected_token_indices,
|
||||
"lora_requests": lora_requests,
|
||||
"lora_mapping": lora_mapping,
|
||||
"multi_modal_input": multi_modal_input,
|
||||
"multi_modal_kwargs": multi_modal_kwargs,
|
||||
"num_prefill_tokens": num_prefill_tokens,
|
||||
"num_decode_tokens": num_decode_tokens,
|
||||
"slot_mapping": slot_mapping,
|
||||
@@ -679,7 +694,7 @@ class ModelRunner:
|
||||
"selected_token_indices")
|
||||
lora_mapping = metadata_dict.pop("lora_mapping")
|
||||
lora_requests = metadata_dict.pop("lora_requests")
|
||||
multi_modal_input = metadata_dict.pop("multi_modal_input")
|
||||
multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
|
||||
if metadata_dict:
|
||||
attn_metadata = self.attn_backend.make_metadata(
|
||||
**metadata_dict)
|
||||
@@ -694,7 +709,7 @@ class ModelRunner:
|
||||
|
||||
return (input_tokens, input_positions, attn_metadata,
|
||||
sampling_metadata, lora_requests, lora_mapping,
|
||||
multi_modal_input)
|
||||
multi_modal_kwargs)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
@@ -703,7 +718,7 @@ class ModelRunner:
|
||||
kv_caches: List[torch.Tensor],
|
||||
) -> Optional[SamplerOutput]:
|
||||
(input_tokens, input_positions, attn_metadata, sampling_metadata,
|
||||
lora_requests, lora_mapping, multi_modal_input
|
||||
lora_requests, lora_mapping, multi_modal_kwargs
|
||||
) = self.prepare_input_tensors(seq_group_metadata_list)
|
||||
|
||||
if self.lora_config:
|
||||
@@ -717,15 +732,14 @@ class ModelRunner:
|
||||
model_executable = self.graph_runners[graph_batch_size]
|
||||
else:
|
||||
model_executable = self.model
|
||||
execute_model_kwargs = {
|
||||
"input_ids": input_tokens,
|
||||
"positions": input_positions,
|
||||
"kv_caches": kv_caches,
|
||||
"attn_metadata": attn_metadata,
|
||||
}
|
||||
if self.vision_language_config:
|
||||
execute_model_kwargs.update({"image_input": multi_modal_input})
|
||||
hidden_states = model_executable(**execute_model_kwargs)
|
||||
|
||||
hidden_states = model_executable(
|
||||
input_ids=input_tokens,
|
||||
positions=input_positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=attn_metadata,
|
||||
**multi_modal_kwargs,
|
||||
)
|
||||
|
||||
# Compute the logits.
|
||||
logits = self.model.compute_logits(hidden_states, sampling_metadata)
|
||||
@@ -781,16 +795,24 @@ class ModelRunner:
|
||||
# To exercise the worst scenario for GPU memory consumption,
|
||||
# the number of seqs (batch_size) is chosen to maximize the number
|
||||
# of images processed.
|
||||
if self.vision_language_config:
|
||||
model_config = self.model_config
|
||||
vlm_config = self.vision_language_config
|
||||
|
||||
if vlm_config:
|
||||
max_num_seqs = min(
|
||||
max_num_seqs,
|
||||
int(max_num_batched_tokens /
|
||||
self.vision_language_config.image_feature_size))
|
||||
int(max_num_batched_tokens / vlm_config.image_feature_size))
|
||||
for group_id in range(max_num_seqs):
|
||||
seq_len = (max_num_batched_tokens // max_num_seqs +
|
||||
(group_id < max_num_batched_tokens % max_num_seqs))
|
||||
seq_data, fake_multi_modal_input = _prepare_fake_inputs(
|
||||
seq_len, self.vision_language_config)
|
||||
|
||||
if vlm_config is None:
|
||||
seq_data = SequenceData([0] * seq_len)
|
||||
dummy_multi_modal_data = None
|
||||
else:
|
||||
seq_data, dummy_multi_modal_data = MULTIMODAL_REGISTRY \
|
||||
.dummy_data_for_profiling(seq_len, model_config, vlm_config)
|
||||
|
||||
seq = SequenceGroupMetadata(
|
||||
request_id=str(group_id),
|
||||
is_prompt=True,
|
||||
@@ -799,7 +821,7 @@ class ModelRunner:
|
||||
block_tables=None,
|
||||
lora_request=dummy_lora_requests_per_seq[group_id]
|
||||
if dummy_lora_requests_per_seq else None,
|
||||
multi_modal_data=fake_multi_modal_input,
|
||||
multi_modal_data=dummy_multi_modal_data,
|
||||
)
|
||||
seqs.append(seq)
|
||||
|
||||
@@ -1034,24 +1056,6 @@ def _get_graph_batch_size(batch_size: int) -> int:
|
||||
_BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
|
||||
|
||||
|
||||
def _prepare_fake_inputs(
|
||||
seq_len: int, vision_language_config: Optional[VisionLanguageConfig]):
|
||||
"""Prepare fake inputs for profile run."""
|
||||
if vision_language_config:
|
||||
prompt_tokens = [
|
||||
vision_language_config.image_token_id
|
||||
] * vision_language_config.image_feature_size + [0] * (
|
||||
seq_len - vision_language_config.image_feature_size)
|
||||
fake_image_input = MultiModalData(
|
||||
type=MultiModalData.Type.IMAGE,
|
||||
data=torch.zeros(vision_language_config.image_input_shape,
|
||||
dtype=torch.float16))
|
||||
else:
|
||||
prompt_tokens = [0] * seq_len
|
||||
fake_image_input = None
|
||||
return SequenceData(prompt_tokens), fake_image_input
|
||||
|
||||
|
||||
def _is_block_tables_empty(block_tables: Union[None, Dict]):
|
||||
"""
|
||||
Check if block_tables is None or a dictionary with all None values.
|
||||
|
||||
Reference in New Issue
Block a user