Revert "[V1] Scatter and gather placeholders in the model runner" (#16075)
This commit is contained in:
@@ -385,8 +385,8 @@ class MultiModalPlaceholderMap:
|
||||
for placeholder_dict, mm_item in zip(multi_modal_placeholders,
|
||||
multi_modal_items):
|
||||
placeholder = range(
|
||||
placeholder_dict.offset,
|
||||
placeholder_dict.offset + placeholder_dict.length,
|
||||
placeholder_dict["offset"],
|
||||
placeholder_dict["offset"] + placeholder_dict["length"],
|
||||
)
|
||||
intersection = range(
|
||||
max(positions.start, placeholder.start),
|
||||
|
||||
@@ -109,8 +109,7 @@ The built-in modalities are defined by :class:`MultiModalDataBuiltins`.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlaceholderRange:
|
||||
class PlaceholderRange(TypedDict):
|
||||
"""
|
||||
Placeholder location information for multi-modal data.
|
||||
|
||||
@@ -122,8 +121,8 @@ class PlaceholderRange:
|
||||
|
||||
.. code-block::
|
||||
|
||||
A: PlaceholderRange(offset=0, length=4)
|
||||
B: PlaceholderRange(offset=5, length=4)
|
||||
A: { "offset": 0, "length": 4 }
|
||||
B: { "offset": 5, "length": 4 }
|
||||
"""
|
||||
|
||||
offset: int
|
||||
@@ -132,31 +131,6 @@ class PlaceholderRange:
|
||||
length: int
|
||||
"""The length of the placeholder."""
|
||||
|
||||
is_embed: Optional[torch.Tensor] = None
|
||||
"""
|
||||
A boolean mask of shape `(length,)` indicating which positions
|
||||
between `offset` and `offset + length` to assign embeddings to.
|
||||
"""
|
||||
|
||||
def get_num_embeds(self) -> int:
|
||||
if self.is_embed is None:
|
||||
return self.length
|
||||
|
||||
return int(self.is_embed.sum().item())
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, self.__class__):
|
||||
return False
|
||||
if not (self.offset, self.length) == (other.offset, other.length):
|
||||
return False
|
||||
|
||||
if self.is_embed is None:
|
||||
return other.is_embed is None
|
||||
if other.is_embed is None:
|
||||
return self.is_embed is None
|
||||
|
||||
return nested_tensors_equal(self.is_embed, other.is_embed)
|
||||
|
||||
|
||||
NestedTensors = Union[list["NestedTensors"], list[torch.Tensor], torch.Tensor,
|
||||
tuple[torch.Tensor, ...]]
|
||||
|
||||
@@ -108,46 +108,16 @@ class PromptUpdateDetails(Generic[_S]):
|
||||
full: _S
|
||||
"""The full content."""
|
||||
|
||||
is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]] = None
|
||||
features: _S
|
||||
"""
|
||||
Given :attr:`full`, return a boolean mask of shape `(len(full),)`
|
||||
indicating which positions of `full` to assign embeddings to.
|
||||
|
||||
`None` (default) means to assign embeddings to all positions of `full`.
|
||||
|
||||
The embeddings are obtained by calling
|
||||
:class:`SupportsMultiModal.get_multimodal_embeddings`.
|
||||
The part of the content that corresponds to feature placeholders;
|
||||
this will be replaced by the output of the vision encoder during model
|
||||
inference.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
|
||||
return PromptUpdateDetails(full=seq)
|
||||
|
||||
@staticmethod
|
||||
def select_text(
|
||||
seq: _S,
|
||||
embed_text: str,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
|
||||
def is_embed(full: "_BoundPromptSequence") -> torch.Tensor:
|
||||
embed_token_ids = encode_tokens(full.tokenizer, embed_text)
|
||||
|
||||
return torch.isin(
|
||||
torch.tensor(full.token_ids),
|
||||
torch.tensor(embed_token_ids),
|
||||
)
|
||||
|
||||
return PromptUpdateDetails(full=seq, is_embed=is_embed)
|
||||
|
||||
@staticmethod
|
||||
def select_token_id(
|
||||
seq: _S,
|
||||
embed_token_id: int,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
return PromptUpdateDetails(
|
||||
full=seq,
|
||||
is_embed=lambda f: torch.tensor(f.token_ids) == embed_token_id,
|
||||
)
|
||||
return PromptUpdateDetails(full=seq, features=seq)
|
||||
|
||||
|
||||
PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
|
||||
@@ -436,7 +406,7 @@ class _BoundPromptSequence:
|
||||
@dataclass
|
||||
class _BoundPromptContent:
|
||||
full: _BoundPromptSequence
|
||||
is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]]
|
||||
features: _BoundPromptSequence
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -496,8 +466,10 @@ class BoundPromptUpdate:
|
||||
|
||||
bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
|
||||
content.full)
|
||||
bound_features = _BoundPromptSequence.from_seq(self.tokenizer,
|
||||
content.features)
|
||||
bound_content = _BoundPromptContent(full=bound_full,
|
||||
is_embed=content.is_embed)
|
||||
features=bound_features)
|
||||
|
||||
if cache_key is not None:
|
||||
self._content_cache[cache_key] = bound_content
|
||||
@@ -633,19 +605,15 @@ class PlaceholderFeaturesInfo:
|
||||
item_idx: int
|
||||
start_idx: int
|
||||
tokens: list[int]
|
||||
is_embed: Optional[torch.Tensor]
|
||||
|
||||
@property
|
||||
def length(self) -> int:
|
||||
return len(self.tokens)
|
||||
|
||||
def to_range(self) -> PlaceholderRange:
|
||||
# TODO: Is it worth it to optimize this by stripping the
|
||||
# leading and ending positions where `is_embed=False`?
|
||||
return PlaceholderRange(
|
||||
offset=self.start_idx,
|
||||
length=self.length,
|
||||
is_embed=self.is_embed,
|
||||
)
|
||||
|
||||
|
||||
@@ -838,17 +806,22 @@ def _iter_placeholders(
|
||||
continue
|
||||
|
||||
if prompt[start_idx:end_idx_full] == content_tokens_full:
|
||||
content_is_embed = content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(content.full)
|
||||
content_tokens_feat = content.features.token_ids
|
||||
|
||||
yield PlaceholderFeaturesInfo(
|
||||
modality=modality,
|
||||
item_idx=item_idx,
|
||||
start_idx=start_idx,
|
||||
tokens=content_tokens_full,
|
||||
is_embed=content_is_embed,
|
||||
)
|
||||
try:
|
||||
match = next(
|
||||
iter_token_matches(content_tokens_full,
|
||||
content_tokens_feat))
|
||||
yield PlaceholderFeaturesInfo(
|
||||
modality=modality,
|
||||
item_idx=item_idx,
|
||||
start_idx=start_idx + match.start_idx,
|
||||
tokens=content_tokens_feat,
|
||||
)
|
||||
except StopIteration:
|
||||
raise AssertionError(
|
||||
f"{content_tokens_feat=} should be a "
|
||||
f"subsequence of {content_tokens_full=}") from None
|
||||
|
||||
# Exclude overlapping matches
|
||||
start_idx = end_idx_full
|
||||
|
||||
@@ -180,7 +180,7 @@ class MultiModalProfiler(Generic[_I]):
|
||||
placeholders_by_modality = mm_inputs["mm_placeholders"]
|
||||
|
||||
total_placeholders_by_modality = {
|
||||
modality: sum(item.get_num_embeds() for item in placeholders)
|
||||
modality: sum(item["length"] for item in placeholders)
|
||||
for modality, placeholders in placeholders_by_modality.items()
|
||||
}
|
||||
expected_placeholders_by_modality = {
|
||||
|
||||
@@ -340,7 +340,7 @@ def merge_and_sort_multimodal_metadata(
|
||||
all_items.append((modality, placeholder, hash_value))
|
||||
|
||||
# Sort all items by offset
|
||||
all_items.sort(key=lambda x: x[1].offset)
|
||||
all_items.sort(key=lambda x: x[1]['offset'])
|
||||
|
||||
# Split into separate lists
|
||||
sorted_modalities = [item[0] for item in all_items]
|
||||
|
||||
Reference in New Issue
Block a user