[Model] Move multimodal_cpu_fields definition to field config (#30181)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -269,10 +269,11 @@ class MsgpackEncoder:
|
||||
name = MMF_CLASS_TO_FACTORY.get(field.__class__)
|
||||
if not name:
|
||||
raise TypeError(f"Unsupported field type: {field.__class__}")
|
||||
|
||||
# We just need to copy all of the field values in order
|
||||
# which will be then used to reconstruct the field.
|
||||
field_values = (getattr(field, f.name) for f in dataclasses.fields(field))
|
||||
return name, *field_values
|
||||
factory_kw = {f.name: getattr(field, f.name) for f in dataclasses.fields(field)}
|
||||
return name, factory_kw
|
||||
|
||||
|
||||
class MsgpackDecoder:
|
||||
@@ -392,15 +393,15 @@ class MsgpackDecoder:
|
||||
obj["data"] = self._decode_nested_tensors(obj["data"])
|
||||
|
||||
# Reconstruct the field processor using MultiModalFieldConfig
|
||||
factory_meth_name, *field_args = obj["field"]
|
||||
factory_meth_name, factory_kw = obj["field"]
|
||||
factory_meth = getattr(MultiModalFieldConfig, factory_meth_name)
|
||||
|
||||
# Special case: decode the union "slices" field of
|
||||
# MultiModalFlatField
|
||||
if factory_meth_name == "flat":
|
||||
field_args[0] = self._decode_nested_slices(field_args[0])
|
||||
factory_kw["slices"] = self._decode_nested_slices(factory_kw["slices"])
|
||||
|
||||
obj["field"] = factory_meth(None, *field_args).field
|
||||
obj["field"] = factory_meth("", **factory_kw).field
|
||||
return MultiModalFieldElem(**obj)
|
||||
|
||||
def _decode_nested_tensors(self, obj: Any) -> NestedTensors:
|
||||
|
||||
@@ -1097,7 +1097,6 @@ class GPUModelRunner(
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
merge_by_field_config=model.merge_by_field_config,
|
||||
multimodal_cpu_fields=model.multimodal_cpu_fields,
|
||||
):
|
||||
mm_kwargs_combined.update(mm_kwargs_group)
|
||||
|
||||
@@ -2109,7 +2108,6 @@ class GPUModelRunner(
|
||||
mm_kwargs,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
multimodal_cpu_fields=model.multimodal_cpu_fields,
|
||||
):
|
||||
curr_group_outputs: list[torch.Tensor] = []
|
||||
|
||||
@@ -2135,7 +2133,6 @@ class GPUModelRunner(
|
||||
[video_mm_kwargs_item],
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
multimodal_cpu_fields=model.multimodal_cpu_fields,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3887,14 +3884,12 @@ class GPUModelRunner(
|
||||
dummy_mm_item = dummy_mm_data[modality][0]
|
||||
dummy_mm_items = [dummy_mm_item] * max_items_per_batch
|
||||
|
||||
model = cast(SupportsMultiModal, self.model)
|
||||
return next(
|
||||
mm_kwargs_group
|
||||
for _, _, mm_kwargs_group in group_mm_kwargs_by_modality(
|
||||
dummy_mm_items,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
multimodal_cpu_fields=model.multimodal_cpu_fields,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -969,7 +969,6 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
mm_kwargs,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
multimodal_cpu_fields=model.multimodal_cpu_fields,
|
||||
):
|
||||
# Run the encoder.
|
||||
# `curr_group_outputs` is either of the following:
|
||||
@@ -2050,14 +2049,12 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
dummy_mm_item = dummy_mm_data[modality][0]
|
||||
dummy_mm_items = [dummy_mm_item] * max_items_per_batch
|
||||
|
||||
model = cast(SupportsMultiModal, self.model)
|
||||
return next(
|
||||
grouped_mm_kwargs
|
||||
for _, _, grouped_mm_kwargs in group_mm_kwargs_by_modality(
|
||||
dummy_mm_items,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
multimodal_cpu_fields=model.multimodal_cpu_fields,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user