[Model] Move multimodal_cpu_fields definition to field config (#30181)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+81
-35
@@ -3,7 +3,7 @@
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import UserDict, defaultdict
|
||||
from collections.abc import Mapping, Sequence, Set
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate
|
||||
@@ -223,6 +223,23 @@ def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
|
||||
return a == b
|
||||
|
||||
|
||||
def _nested_tensors_h2d(
|
||||
tensors: NestedTensors,
|
||||
device: torch.types.Device,
|
||||
) -> NestedTensors:
|
||||
if device is None:
|
||||
return tensors
|
||||
|
||||
return json_map_leaves(
|
||||
(
|
||||
lambda x: x.to(device=device, non_blocking=True)
|
||||
if isinstance(x, torch.Tensor)
|
||||
else x
|
||||
),
|
||||
tensors,
|
||||
)
|
||||
|
||||
|
||||
BatchedTensorInputs: TypeAlias = dict[str, NestedTensors]
|
||||
"""
|
||||
A dictionary containing nested tensors which have been batched via
|
||||
@@ -334,7 +351,7 @@ class MultiModalFieldElem:
|
||||
) # noqa: E721
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class BaseMultiModalField(ABC):
|
||||
"""
|
||||
Defines how to interpret tensor data belonging to a keyword argument in
|
||||
@@ -342,6 +359,12 @@ class BaseMultiModalField(ABC):
|
||||
multi-modal items, and vice versa.
|
||||
"""
|
||||
|
||||
keep_on_cpu: bool = False
|
||||
"""
|
||||
If `True`, then this field is excluded from being moved to the accelerator
|
||||
when `MultiModalKwargsItems.get_data()` is called to batch the data.
|
||||
"""
|
||||
|
||||
def _field_factory(self, *, modality: str, key: str):
|
||||
f = partial(
|
||||
MultiModalFieldElem,
|
||||
@@ -386,6 +409,7 @@ class BaseMultiModalField(ABC):
|
||||
self,
|
||||
elems: list[MultiModalFieldElem],
|
||||
*,
|
||||
device: torch.types.Device = None,
|
||||
pin_memory: bool = False,
|
||||
) -> NestedTensors:
|
||||
"""
|
||||
@@ -399,11 +423,17 @@ class BaseMultiModalField(ABC):
|
||||
if len(set(field_types)) > 1:
|
||||
raise ValueError(f"Cannot merge different {field_types=}")
|
||||
|
||||
if device is not None and self.keep_on_cpu:
|
||||
device = "cpu"
|
||||
if pin_memory and self.keep_on_cpu:
|
||||
pin_memory = False
|
||||
|
||||
batch = [elem.data for elem in elems]
|
||||
return self._reduce_data(batch, pin_memory=pin_memory)
|
||||
out = self._reduce_data(batch, pin_memory=pin_memory)
|
||||
return _nested_tensors_h2d(out, device=device)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MultiModalBatchedField(BaseMultiModalField):
|
||||
"""
|
||||
Info:
|
||||
@@ -445,7 +475,7 @@ class MultiModalBatchedField(BaseMultiModalField):
|
||||
return batch
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MultiModalFlatField(BaseMultiModalField):
|
||||
"""
|
||||
Info:
|
||||
@@ -505,7 +535,7 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
return [e for elem in batch for e in elem]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MultiModalSharedField(BaseMultiModalField):
|
||||
"""
|
||||
Info:
|
||||
@@ -532,9 +562,10 @@ class MultiModalSharedField(BaseMultiModalField):
|
||||
return batch[0]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MultiModalFieldConfig:
|
||||
@staticmethod
|
||||
def batched(modality: str):
|
||||
def batched(modality: str, *, keep_on_cpu: bool = False):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
indexing into the first dimension of the underlying data.
|
||||
@@ -542,6 +573,7 @@ class MultiModalFieldConfig:
|
||||
Args:
|
||||
modality: The modality of the multi-modal item that uses this
|
||||
keyword argument.
|
||||
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
|
||||
|
||||
Example:
|
||||
|
||||
@@ -558,7 +590,7 @@ class MultiModalFieldConfig:
|
||||
```
|
||||
"""
|
||||
return MultiModalFieldConfig(
|
||||
field=MultiModalBatchedField(),
|
||||
field=MultiModalBatchedField(keep_on_cpu=keep_on_cpu),
|
||||
modality=modality,
|
||||
)
|
||||
|
||||
@@ -567,6 +599,8 @@ class MultiModalFieldConfig:
|
||||
modality: str,
|
||||
slices: Sequence[slice] | Sequence[Sequence[slice]],
|
||||
dim: int = 0,
|
||||
*,
|
||||
keep_on_cpu: bool = False,
|
||||
):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
@@ -579,6 +613,7 @@ class MultiModalFieldConfig:
|
||||
slices (dim>0) that is used to extract the data corresponding
|
||||
to it.
|
||||
dim: The dimension to extract data, default to 0.
|
||||
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
|
||||
|
||||
Example:
|
||||
|
||||
@@ -613,12 +648,22 @@ class MultiModalFieldConfig:
|
||||
```
|
||||
"""
|
||||
return MultiModalFieldConfig(
|
||||
field=MultiModalFlatField(slices=slices, dim=dim),
|
||||
field=MultiModalFlatField(
|
||||
slices=slices,
|
||||
dim=dim,
|
||||
keep_on_cpu=keep_on_cpu,
|
||||
),
|
||||
modality=modality,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def flat_from_sizes(modality: str, size_per_item: "torch.Tensor", dim: int = 0):
|
||||
def flat_from_sizes(
|
||||
modality: str,
|
||||
size_per_item: "torch.Tensor",
|
||||
dim: int = 0,
|
||||
*,
|
||||
keep_on_cpu: bool = False,
|
||||
):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
slicing along the first dimension of the underlying data.
|
||||
@@ -629,6 +674,7 @@ class MultiModalFieldConfig:
|
||||
size_per_item: For each multi-modal item, the size of the slice
|
||||
that is used to extract the data corresponding to it.
|
||||
dim: The dimension to slice, default to 0.
|
||||
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
|
||||
|
||||
Example:
|
||||
|
||||
@@ -676,10 +722,20 @@ class MultiModalFieldConfig:
|
||||
for i in range(len(size_per_item))
|
||||
]
|
||||
|
||||
return MultiModalFieldConfig.flat(modality, slices, dim=dim)
|
||||
return MultiModalFieldConfig.flat(
|
||||
modality,
|
||||
slices,
|
||||
dim=dim,
|
||||
keep_on_cpu=keep_on_cpu,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def shared(modality: str, batch_size: int):
|
||||
def shared(
|
||||
modality: str,
|
||||
batch_size: int,
|
||||
*,
|
||||
keep_on_cpu: bool = False,
|
||||
):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
taking the entirety of the underlying data.
|
||||
@@ -690,6 +746,7 @@ class MultiModalFieldConfig:
|
||||
modality: The modality of the multi-modal item that uses this
|
||||
keyword argument.
|
||||
batch_size: The number of multi-modal items which share this data.
|
||||
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
|
||||
|
||||
Example:
|
||||
|
||||
@@ -708,18 +765,15 @@ class MultiModalFieldConfig:
|
||||
```
|
||||
"""
|
||||
return MultiModalFieldConfig(
|
||||
field=MultiModalSharedField(batch_size),
|
||||
field=MultiModalSharedField(
|
||||
batch_size=batch_size,
|
||||
keep_on_cpu=keep_on_cpu,
|
||||
),
|
||||
modality=modality,
|
||||
)
|
||||
|
||||
def __init__(self, field: BaseMultiModalField, modality: str) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.field = field
|
||||
self.modality = modality
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"MultiModalFieldConfig(field={self.field}, modality={self.modality})"
|
||||
field: BaseMultiModalField
|
||||
modality: str
|
||||
|
||||
def build_elems(
|
||||
self,
|
||||
@@ -744,7 +798,7 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
|
||||
modality=modality,
|
||||
key="dummy",
|
||||
data=torch.empty(nbytes, dtype=torch.uint8),
|
||||
field=MultiModalSharedField(1),
|
||||
field=MultiModalSharedField(batch_size=1),
|
||||
)
|
||||
return MultiModalKwargsItem.from_elems([mm_elem])
|
||||
|
||||
@@ -844,7 +898,6 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
|
||||
*,
|
||||
device: torch.types.Device = None,
|
||||
pin_memory: bool = False,
|
||||
cpu_fields: Set[str] = frozenset(),
|
||||
) -> BatchedTensorInputs:
|
||||
"""Construct a dictionary of keyword arguments to pass to the model."""
|
||||
elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
|
||||
@@ -859,21 +912,14 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
|
||||
elems_by_key[key].append(elem)
|
||||
|
||||
data = {
|
||||
key: elems[0].field.reduce_data(elems, pin_memory=pin_memory)
|
||||
key: elems[0].field.reduce_data(
|
||||
elems,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
for key, elems in elems_by_key.items()
|
||||
}
|
||||
|
||||
if device is not None:
|
||||
for k in data.keys() - cpu_fields:
|
||||
data[k] = json_map_leaves(
|
||||
(
|
||||
lambda x: x.to(device=device, non_blocking=True)
|
||||
if isinstance(x, torch.Tensor)
|
||||
else x
|
||||
),
|
||||
data[k],
|
||||
)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user