[Perf] Avoid more h2d copies from non-pinned tensors (#54660)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-09-01 20:37:09 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3ba9907a1d
commit a566ea7e8f
5 changed files with 72 additions and 64 deletions
+37 -51
View File
@@ -293,21 +293,39 @@ def _nested_tensors_are_cpu(tensors: NestedTensors) -> bool:
)
def _is_dense(t: torch.Tensor) -> bool:
"""Whether `t`'s elements fill a contiguous storage range, possibly
permuted (e.g. a transposed matrix). Such layouts can be copied with
pitched cudaMemcpy2D/3DAsync directly from pinned memory, whereas
gapped layouts (e.g. strided slices) stage through a pageable temp."""
if not t.is_contiguous():
expected_stride = 1
for stride, size in sorted(zip(t.stride(), t.shape)):
if size > 1:
if stride != expected_stride:
return False
expected_stride *= size
return True
def _nested_tensors_h2d(
tensors: NestedTensors,
device: torch.types.Device,
*,
pin_memory: bool = False,
) -> NestedTensors:
if device is None:
return tensors
return json_map_leaves(
(
lambda x: x.to(device=device, non_blocking=True)
if isinstance(x, torch.Tensor)
else x
),
tensors,
)
def _h2d(x: torch.Tensor) -> torch.Tensor:
if not isinstance(x, torch.Tensor):
return x
if pin_memory and x.is_cpu and not (x.is_pinned() and _is_dense(x)):
# Ensure tensor is pinned and dense for non_blocking H2D copy.
x = x.new_empty(x.shape, pin_memory=True).copy_(x)
return x.to(device=device, non_blocking=True)
return json_map_leaves(_h2d, tensors)
BatchedTensorInputs: TypeAlias = dict[str, NestedTensors]
@@ -452,10 +470,7 @@ class BaseMultiModalField(ABC):
@abstractmethod
def _reduce_data(
self,
batch: list[NestedTensors],
*,
pin_memory: bool,
self, batch: list[NestedTensors], *, pin_memory: bool
) -> NestedTensors:
raise NotImplementedError
@@ -491,7 +506,7 @@ class BaseMultiModalField(ABC):
# concept, and `pin_memory()` rejects device tensors outright.
pin_memory = False
out = self._reduce_data(batch, pin_memory=pin_memory)
return _nested_tensors_h2d(out, device=device)
return _nested_tensors_h2d(out, device=device, pin_memory=pin_memory)
@dataclass(frozen=True, kw_only=True)
@@ -511,10 +526,7 @@ class MultiModalBatchedField(BaseMultiModalField):
return [field_factory(item) for item in data]
def _reduce_data(
self,
batch: list[NestedTensors],
*,
pin_memory: bool,
self, batch: list[NestedTensors], *, pin_memory: bool
) -> NestedTensors:
if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
batch = cast(list[torch.Tensor], batch)
@@ -522,20 +534,11 @@ class MultiModalBatchedField(BaseMultiModalField):
# An optimization when `batch` contains only one tensor:
# - produce exactly same result as `torch.stack(batch)`
# - will achieve zero-copy if the tensor is contiguous
out = batch[0].unsqueeze(0)
if not pin_memory:
return out.contiguous()
# Avoid extra copy - pinning unpinned memory will make it contiguous
if not out.is_contiguous() and out.is_pinned():
out = out.contiguous()
return out.pin_memory()
return batch[0].unsqueeze(0)
first_shape = batch[0].shape
if all(elem.shape == first_shape for elem in batch):
out = torch.empty(
(len(batch), *batch[0].shape),
dtype=batch[0].dtype,
device=batch[0].device,
pin_memory=pin_memory,
out = batch[0].new_empty(
(len(batch), *first_shape), pin_memory=pin_memory
)
return torch.stack(batch, out=out)
@@ -567,10 +570,7 @@ class MultiModalFlatField(BaseMultiModalField):
return [field_factory(data[cast(slice, s)]) for s in self.slices]
def _reduce_data(
self,
batch: list[NestedTensors],
*,
pin_memory: bool,
self, batch: list[NestedTensors], *, pin_memory: bool
) -> NestedTensors:
if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
batch = cast(list[torch.Tensor], batch)
@@ -578,13 +578,7 @@ class MultiModalFlatField(BaseMultiModalField):
# An optimization when `batch` contains only one tensor:
# - produce exactly same result as `torch.concat(batch)`
# - will achieve zero-copy if the tensor is contiguous
out = batch[0]
if not pin_memory:
return out.contiguous()
# Avoid extra copy - pinning unpinned memory will make it contiguous
if not out.is_contiguous() and out.is_pinned():
out = out.contiguous()
return out.pin_memory()
return batch[0]
dim = self.dim + (self.dim < 0) * len(batch[0].shape)
@@ -596,11 +590,8 @@ class MultiModalFlatField(BaseMultiModalField):
if all(_shape_before_after(elem) == first_shape for elem in batch):
shape_before, shape_after = first_shape
shape_concat = sum(item.shape[dim] for item in batch)
out = torch.empty(
(*shape_before, shape_concat, *shape_after),
dtype=batch[0].dtype,
device=batch[0].device,
pin_memory=pin_memory,
out = batch[0].new_empty(
(*shape_before, shape_concat, *shape_after), pin_memory=pin_memory
)
return torch.concat(batch, dim=self.dim, out=out)
@@ -622,12 +613,7 @@ class MultiModalFlatField(BaseMultiModalField):
max_sizes.append(max(t.shape[d] for t in batch))
# Step 2: Create zero-initialized output tensor
out = torch.zeros(
max_sizes,
dtype=batch[0].dtype,
device=batch[0].device,
pin_memory=pin_memory,
)
out = batch[0].new_zeros(max_sizes, pin_memory=pin_memory)
# Step 3: Slice-assign each tensor to its proper position
concat_offset = 0