[Core] Automatically cast multi-modal input dtype (#18756)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -746,11 +746,17 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
|
||||
batched_inputs: BatchedTensorInputs,
|
||||
*,
|
||||
device: torch.types.Device,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
) -> BatchedTensorInputs:
|
||||
json_inputs = cast(JSONTree[torch.Tensor], batched_inputs)
|
||||
|
||||
def maybe_cast_dtype(x: torch.Tensor):
|
||||
# This mimics the behavior of transformers.BatchFeature
|
||||
return x.to(dtype=dtype) if x.is_floating_point() else x
|
||||
|
||||
json_mapped = json_map_leaves(
|
||||
lambda x: x.to(device, non_blocking=True),
|
||||
# NOTE: Cast the dtype before sending it to device
|
||||
lambda x: maybe_cast_dtype(x).to(device=device, non_blocking=True),
|
||||
json_inputs,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user