[Core] Support image processor (#4197)
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
from typing import List, Optional, Tuple
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -11,6 +12,7 @@ from vllm.distributed import broadcast_tensor_dict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor import SamplingMetadata
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
|
||||
from vllm.utils import make_tensor_with_pad
|
||||
|
||||
@@ -63,6 +65,16 @@ class CPUModelRunner:
|
||||
self.block_size,
|
||||
)
|
||||
|
||||
# Create processor for multi-modal data
|
||||
if self.vision_language_config is not None:
|
||||
self.multi_modal_input_processor = MULTIMODAL_REGISTRY \
|
||||
.create_input_processor(
|
||||
self.model_config,
|
||||
self.vision_language_config,
|
||||
)
|
||||
else:
|
||||
self.multi_modal_input_processor = None
|
||||
|
||||
# Lazy initialization.
|
||||
self.model: nn.Module # Set after init_Model
|
||||
|
||||
@@ -80,14 +92,15 @@ class CPUModelRunner:
|
||||
def _prepare_prompt(
|
||||
self,
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
|
||||
Optional[torch.Tensor]]:
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int], Dict[
|
||||
str, torch.Tensor]]:
|
||||
assert len(seq_group_metadata_list) > 0
|
||||
input_tokens: List[int] = []
|
||||
input_positions: List[int] = []
|
||||
slot_mapping: List[int] = []
|
||||
seq_lens: List[int] = []
|
||||
multi_modal_input_list: List[torch.Tensor] = []
|
||||
multi_modal_kwargs_list: Dict[str,
|
||||
List[torch.Tensor]] = defaultdict(list)
|
||||
|
||||
for seq_group_metadata in seq_group_metadata_list:
|
||||
assert seq_group_metadata.is_prompt
|
||||
@@ -108,9 +121,17 @@ class CPUModelRunner:
|
||||
# is always the first token in the sequence.
|
||||
input_positions.extend(list(range(computed_len, seq_len)))
|
||||
|
||||
if seq_group_metadata.multi_modal_data:
|
||||
multi_modal_input_list.append(
|
||||
seq_group_metadata.multi_modal_data.data)
|
||||
mm_data = seq_group_metadata.multi_modal_data
|
||||
if mm_data is not None:
|
||||
# Process multi-modal data
|
||||
if self.multi_modal_input_processor is None:
|
||||
raise ValueError(
|
||||
"Multi-modal inputs are only supported by "
|
||||
"vision language models.")
|
||||
|
||||
mm_kwargs = self.multi_modal_input_processor(mm_data)
|
||||
for k, v in mm_kwargs.items():
|
||||
multi_modal_kwargs_list[k].append(v)
|
||||
|
||||
# Compute the slot mapping.
|
||||
block_table = seq_group_metadata.block_tables[seq_id]
|
||||
@@ -134,14 +155,10 @@ class CPUModelRunner:
|
||||
slot = block_number * self.block_size + block_offset
|
||||
slot_mapping.append(slot)
|
||||
|
||||
if multi_modal_input_list:
|
||||
assert self.vision_language_config, (
|
||||
"Multi-modal inputs are only supported by "
|
||||
"vision language models.")
|
||||
multi_modal_input = torch.cat(multi_modal_input_list,
|
||||
dim=0).to(self.device)
|
||||
else:
|
||||
multi_modal_input = None
|
||||
multi_modal_kwargs = {
|
||||
k: torch.cat(v, dim=0).to(self.device)
|
||||
for k, v in multi_modal_kwargs_list.items()
|
||||
}
|
||||
|
||||
num_prompt_tokens = len(input_tokens)
|
||||
|
||||
@@ -167,7 +184,7 @@ class CPUModelRunner:
|
||||
slot_mapping=slot_mapping,
|
||||
)
|
||||
return (input_tokens, input_positions, attn_metadata, seq_lens,
|
||||
multi_modal_input)
|
||||
multi_modal_kwargs)
|
||||
|
||||
def _prepare_decode(
|
||||
self,
|
||||
@@ -257,8 +274,8 @@ class CPUModelRunner:
|
||||
self,
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
|
||||
Optional[torch.Tensor]]:
|
||||
multi_modal_input = None
|
||||
Optional[Dict[str, torch.Tensor]]]:
|
||||
multi_modal_kwargs = None
|
||||
if self.is_driver_worker:
|
||||
# NOTE: We assume that all sequences in the group are all prompts or
|
||||
# all decodes.
|
||||
@@ -266,7 +283,7 @@ class CPUModelRunner:
|
||||
# Prepare input tensors.
|
||||
if is_prompt:
|
||||
(input_tokens, input_positions, attn_metadata, seq_lens,
|
||||
multi_modal_input
|
||||
multi_modal_kwargs
|
||||
) = self._prepare_prompt(seq_group_metadata_list)
|
||||
else:
|
||||
(input_tokens, input_positions,
|
||||
@@ -307,7 +324,7 @@ class CPUModelRunner:
|
||||
)
|
||||
|
||||
return (input_tokens, input_positions, attn_metadata,
|
||||
sampling_metadata, multi_modal_input)
|
||||
sampling_metadata, multi_modal_kwargs)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
|
||||
Reference in New Issue
Block a user