[V1] Scatter and gather placeholders in the model runner (#16076)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Jennifer Zhao <ai.jenniferzhao@gmail.com>
This commit is contained in:
co-authored by
DarkLight1337
mgoin
Jennifer Zhao
parent
1d01211264
commit
f2ebb6f541
@@ -56,7 +56,7 @@ from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
|
||||
VideoItem, VideoProcessorItems)
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, PromptReplacement,
|
||||
PromptUpdate)
|
||||
PromptUpdate, PromptUpdateDetails)
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -67,7 +67,6 @@ from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
|
||||
SupportsMultiModal, SupportsPP)
|
||||
from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
|
||||
merge_multimodal_embeddings)
|
||||
from .vision import scatter_patch_features, select_patch_features
|
||||
|
||||
# For profile run
|
||||
_MAX_FRAMES_PER_VIDEO = 16
|
||||
@@ -90,14 +89,6 @@ class MiniCPMVImagePixelInputs(TypedDict):
|
||||
This should be in `(height, width)` format.
|
||||
"""
|
||||
|
||||
embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
|
||||
"""
|
||||
A boolean mask indicating which image embeddings correspond
|
||||
to patch tokens.
|
||||
|
||||
Shape: `(batch_size * num_images, num_embeds)`
|
||||
"""
|
||||
|
||||
num_slices: torch.Tensor
|
||||
"""Shape: `(batch_size * num_images)`"""
|
||||
|
||||
@@ -112,14 +103,6 @@ class MiniCPMVImageEmbeddingInputs(TypedDict):
|
||||
instead of a batched tensor.
|
||||
"""
|
||||
|
||||
embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
|
||||
"""
|
||||
A boolean mask indicating which image embeddings correspond
|
||||
to patch tokens.
|
||||
|
||||
Shape: `(batch_size * num_images, num_embeds)`
|
||||
"""
|
||||
|
||||
|
||||
MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
|
||||
MiniCPMVImageEmbeddingInputs]
|
||||
@@ -245,12 +228,10 @@ def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
image_sizes=MultiModalFieldConfig.batched("image"),
|
||||
tgt_sizes=MultiModalFieldConfig.batched("image"),
|
||||
image_embeds=MultiModalFieldConfig.batched("image"),
|
||||
embed_is_patch=MultiModalFieldConfig.batched("image"),
|
||||
video_pixel_values=MultiModalFieldConfig.batched("video"),
|
||||
video_image_sizes=MultiModalFieldConfig.batched("video"),
|
||||
video_tgt_sizes=MultiModalFieldConfig.batched("video"),
|
||||
video_embeds=MultiModalFieldConfig.batched("video"),
|
||||
video_embed_is_patch=MultiModalFieldConfig.batched("video"),
|
||||
image_token_id=MultiModalFieldConfig.shared("image", num_images),
|
||||
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
|
||||
)
|
||||
@@ -398,22 +379,43 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
use_image_id=use_image_id,
|
||||
)
|
||||
|
||||
def get_sliced_grid(
|
||||
self,
|
||||
image_size: ImageSize,
|
||||
# For MiniCPM V/O 2.6
|
||||
max_slice_nums: Optional[int] = None,
|
||||
) -> Optional[tuple[int, int]]:
|
||||
image_processor = self.get_image_processor()
|
||||
version = self.get_model_version()
|
||||
|
||||
if version == (2, 0) or version == (2, 5):
|
||||
return image_processor.get_sliced_grid(image_size)
|
||||
|
||||
if max_slice_nums is None:
|
||||
max_slice_nums = image_processor.max_slice_nums
|
||||
|
||||
return image_processor.get_sliced_grid(
|
||||
image_size,
|
||||
max_slice_nums=max_slice_nums,
|
||||
)
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
image_size: ImageSize,
|
||||
max_slice_nums: Optional[int] = None,
|
||||
use_image_id: bool = True,
|
||||
) -> int:
|
||||
tokenizer = self.get_tokenizer()
|
||||
image_placeholders = self.get_slice_image_placeholder(
|
||||
image_processor = self.get_image_processor()
|
||||
|
||||
grid = self.get_sliced_grid(
|
||||
image_size,
|
||||
max_slice_nums=max_slice_nums,
|
||||
use_image_id=use_image_id,
|
||||
)
|
||||
image_token_ids = tokenizer.encode(image_placeholders,
|
||||
add_special_tokens=False)
|
||||
if grid is None:
|
||||
ncols = nrows = 0
|
||||
else:
|
||||
ncols, nrows = grid
|
||||
|
||||
return len(image_token_ids)
|
||||
return (ncols * nrows + 1) * image_processor.image_feature_size
|
||||
|
||||
def get_max_image_tokens(self) -> int:
|
||||
image_size = self.get_image_size_with_most_features()
|
||||
@@ -433,7 +435,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
return self.get_num_image_tokens(
|
||||
frame_size,
|
||||
max_slice_nums=self.get_video_max_slice_num(),
|
||||
use_image_id=False,
|
||||
)
|
||||
|
||||
def get_max_video_tokens(
|
||||
@@ -539,14 +540,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
use_image_id=False,
|
||||
) * num_frames
|
||||
|
||||
def get_embed_is_patch(
|
||||
self,
|
||||
input_ids: list[int],
|
||||
) -> torch.Tensor:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
unk_token_id = tokenizer.get_vocab()["<unk>"]
|
||||
return torch.tensor(input_ids) == unk_token_id
|
||||
|
||||
def process_images(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
@@ -570,26 +563,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
|
||||
)
|
||||
|
||||
image_sizes = [
|
||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||
]
|
||||
image_repl_features = [
|
||||
self.get_image_prompt_texts(size, idx)
|
||||
for idx, size in enumerate(image_sizes)
|
||||
]
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_repls_feature_tokens = [
|
||||
tokenizer.encode(image_repl, add_special_tokens=False)
|
||||
for image_repl in image_repl_features
|
||||
]
|
||||
|
||||
embed_is_patch = [
|
||||
self.get_embed_is_patch(image_repl_tokens)
|
||||
for image_repl_tokens in image_repls_feature_tokens
|
||||
]
|
||||
image_inputs["embed_is_patch"] = embed_is_patch
|
||||
|
||||
unk_token_id = tokenizer.get_vocab()["<unk>"]
|
||||
image_inputs["image_token_id"] = torch.tensor(unk_token_id)
|
||||
|
||||
@@ -625,31 +599,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
|
||||
)
|
||||
|
||||
frame_sizes = [
|
||||
parsed_videos.get_frame_size(i) for i in range(len(parsed_videos))
|
||||
]
|
||||
num_frames = [
|
||||
parsed_videos.get_num_frames(i) for i in range(len(parsed_videos))
|
||||
]
|
||||
video_repl_features = [
|
||||
self.get_video_prompt_texts(size, nframes)
|
||||
for size, nframes in zip(frame_sizes, num_frames)
|
||||
]
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
video_repls_feature_tokens = [
|
||||
tokenizer.encode(video_repl, add_special_tokens=False)
|
||||
for video_repl in video_repl_features
|
||||
]
|
||||
|
||||
embed_is_patch = [
|
||||
self.get_embed_is_patch(video_repl_tokens)
|
||||
for video_repl_tokens in video_repls_feature_tokens
|
||||
]
|
||||
video_inputs["embed_is_patch"] = embed_is_patch
|
||||
|
||||
video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
unk_token_id = tokenizer.get_vocab()["<unk>"]
|
||||
video_inputs["video_token_id"] = torch.tensor(unk_token_id)
|
||||
|
||||
@@ -740,7 +692,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
image_size = images.get_image_size(item_idx)
|
||||
|
||||
return self.get_image_prompt_texts(image_size, item_idx)
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_image_prompt_texts(image_size, item_idx),
|
||||
"<unk>",
|
||||
)
|
||||
|
||||
def get_video_replacement(item_idx: int):
|
||||
videos = mm_items.get_items(
|
||||
@@ -749,7 +704,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
frame_size = videos.get_frame_size(item_idx)
|
||||
num_frames = videos.get_num_frames(item_idx)
|
||||
|
||||
return self.get_video_prompt_texts(frame_size, num_frames)
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_video_prompt_texts(frame_size, num_frames),
|
||||
"<unk>",
|
||||
)
|
||||
|
||||
get_replacement = {
|
||||
"image": get_image_replacement,
|
||||
@@ -832,14 +790,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
assert isinstance(image_token_id, torch.Tensor)
|
||||
self.mm_token_ids.add(image_token_id.flatten().unique().item())
|
||||
|
||||
embed_is_patch = kwargs.pop("embed_is_patch")
|
||||
if not isinstance(embed_is_patch, (torch.Tensor, list)):
|
||||
raise ValueError(
|
||||
f"Incorrect type of embed_is_patch for {modality=}. "
|
||||
f"Got type: {type(embed_is_patch)}")
|
||||
|
||||
embed_is_patch = flatten_bn(embed_is_patch)
|
||||
|
||||
if image_embeds is not None:
|
||||
if not isinstance(image_embeds, (torch.Tensor, list)):
|
||||
raise ValueError(
|
||||
@@ -851,7 +801,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
return MiniCPMVImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds_flat,
|
||||
embed_is_patch=embed_is_patch,
|
||||
)
|
||||
|
||||
if not isinstance(pixel_values, (torch.Tensor, list)):
|
||||
@@ -879,7 +828,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values_flat,
|
||||
tgt_sizes=tgt_sizes_flat,
|
||||
embed_is_patch=embed_is_patch,
|
||||
num_slices=num_slices_flat,
|
||||
)
|
||||
|
||||
@@ -936,19 +884,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
if modality == "images":
|
||||
image_input = modalities["images"]
|
||||
image_features = self._process_vision_input(image_input)
|
||||
multimodal_embeddings += tuple(
|
||||
scatter_patch_features(
|
||||
image_features,
|
||||
image_input["embed_is_patch"],
|
||||
))
|
||||
multimodal_embeddings += tuple(image_features)
|
||||
if modality == "videos":
|
||||
video_input = modalities["videos"]
|
||||
video_features = self._process_vision_input(video_input)
|
||||
multimodal_embeddings += tuple(
|
||||
scatter_patch_features(
|
||||
video_features,
|
||||
video_input["embed_is_patch"],
|
||||
))
|
||||
multimodal_embeddings += tuple(video_features)
|
||||
|
||||
return multimodal_embeddings
|
||||
|
||||
@@ -971,7 +911,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
inputs_embeds = merge_multimodal_embeddings(
|
||||
input_ids,
|
||||
inputs_embeds,
|
||||
select_patch_features(multimodal_embeddings),
|
||||
multimodal_embeddings,
|
||||
list(self.mm_token_ids),
|
||||
)
|
||||
return inputs_embeds
|
||||
|
||||
Reference in New Issue
Block a user