[V1] Scatter and gather placeholders in the model runner (#16076)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Roger Wang <ywang@roblox.com>
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Jennifer Zhao <ai.jenniferzhao@gmail.com>
This commit is contained in:
Roger Wang
2025-04-08 10:43:41 +08:00
committed by GitHub
co-authored by DarkLight1337 mgoin Jennifer Zhao
parent 1d01211264
commit f2ebb6f541
41 changed files with 521 additions and 1020 deletions
+29 -3
View File
@@ -109,7 +109,8 @@ The built-in modalities are defined by :class:`MultiModalDataBuiltins`.
"""
class PlaceholderRange(TypedDict):
@dataclass(frozen=True)
class PlaceholderRange:
"""
Placeholder location information for multi-modal data.
@@ -121,8 +122,8 @@ class PlaceholderRange(TypedDict):
.. code-block::
A: { "offset": 0, "length": 4 }
B: { "offset": 5, "length": 4 }
A: PlaceholderRange(offset=0, length=4)
B: PlaceholderRange(offset=5, length=4)
"""
offset: int
@@ -131,6 +132,31 @@ class PlaceholderRange(TypedDict):
length: int
"""The length of the placeholder."""
is_embed: Optional[torch.Tensor] = None
"""
A boolean mask of shape `(length,)` indicating which positions
between `offset` and `offset + length` to assign embeddings to.
"""
def get_num_embeds(self) -> int:
if self.is_embed is None:
return self.length
return int(self.is_embed.sum().item())
def __eq__(self, other: object) -> bool:
if not isinstance(other, self.__class__):
return False
if not (self.offset, self.length) == (other.offset, other.length):
return False
if self.is_embed is None:
return other.is_embed is None
if other.is_embed is None:
return self.is_embed is None
return nested_tensors_equal(self.is_embed, other.is_embed)
NestedTensors = Union[list["NestedTensors"], list[torch.Tensor], torch.Tensor,
tuple[torch.Tensor, ...]]