[Perf] Avoid h2d copies from non-pinned CPU tensors (#54299)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-29 12:06:30 +08:00
committed by GitHub
parent 5182f2705b
commit 6c18a54648
14 changed files with 81 additions and 84 deletions
+11 -10
View File
@@ -158,13 +158,13 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
yield module
return
original_device_states: dict[str, torch.device] = {}
cpu_params: set[str] = set()
uva_offloaded_parameters: list[str] = []
# Store original device states and move parameters to GPU if they're on CPU
# Store which parameters are on CPU and move them to the GPU
for name, p in module.named_parameters():
if p.device.type == "cpu":
original_device_states[name] = p.device
cpu_params.add(name)
p.data = p.data.to(target_device)
if getattr(p, "_vllm_is_uva_offloaded", False):
uva_offloaded_parameters.append(name)
@@ -178,20 +178,21 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
is_pin_memory_available()
and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY
)
# Restore parameters to their original devices, ignoring new parameters
# Restore the CPU-resident parameters, ignoring new parameters.
for name, p in module.named_parameters():
if name in original_device_states:
original_device: torch.device = original_device_states[name]
p.data = p.data.to(original_device)
if name in cpu_params:
p.data = torch.empty_like(
p.data, device="cpu", pin_memory=use_pin_memory
).copy_(p.data)
# parameter is UVA offloaded, but was replaced with a new device tensor
# re-offload it to CPU using UVA
if name in uva_offloaded_parameters and not getattr(
p, "_vllm_is_uva_offloaded", False
):
cpu_data = p.data.to(device="cpu")
if use_pin_memory:
cpu_data = cpu_data.pin_memory()
cpu_data = torch.empty_like(
p.data, device="cpu", pin_memory=use_pin_memory
).copy_(p.data)
p.data = get_accelerator_view_from_cpu_tensor(cpu_data)
p._vllm_is_uva_offloaded = True