[Perf] Avoid h2d copies from non-pinned CPU tensors (#54299)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -158,13 +158,13 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
|
||||
yield module
|
||||
return
|
||||
|
||||
original_device_states: dict[str, torch.device] = {}
|
||||
cpu_params: set[str] = set()
|
||||
uva_offloaded_parameters: list[str] = []
|
||||
|
||||
# Store original device states and move parameters to GPU if they're on CPU
|
||||
# Store which parameters are on CPU and move them to the GPU
|
||||
for name, p in module.named_parameters():
|
||||
if p.device.type == "cpu":
|
||||
original_device_states[name] = p.device
|
||||
cpu_params.add(name)
|
||||
p.data = p.data.to(target_device)
|
||||
if getattr(p, "_vllm_is_uva_offloaded", False):
|
||||
uva_offloaded_parameters.append(name)
|
||||
@@ -178,20 +178,21 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
|
||||
is_pin_memory_available()
|
||||
and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY
|
||||
)
|
||||
# Restore parameters to their original devices, ignoring new parameters
|
||||
# Restore the CPU-resident parameters, ignoring new parameters.
|
||||
for name, p in module.named_parameters():
|
||||
if name in original_device_states:
|
||||
original_device: torch.device = original_device_states[name]
|
||||
p.data = p.data.to(original_device)
|
||||
if name in cpu_params:
|
||||
p.data = torch.empty_like(
|
||||
p.data, device="cpu", pin_memory=use_pin_memory
|
||||
).copy_(p.data)
|
||||
|
||||
# parameter is UVA offloaded, but was replaced with a new device tensor
|
||||
# re-offload it to CPU using UVA
|
||||
if name in uva_offloaded_parameters and not getattr(
|
||||
p, "_vllm_is_uva_offloaded", False
|
||||
):
|
||||
cpu_data = p.data.to(device="cpu")
|
||||
if use_pin_memory:
|
||||
cpu_data = cpu_data.pin_memory()
|
||||
cpu_data = torch.empty_like(
|
||||
p.data, device="cpu", pin_memory=use_pin_memory
|
||||
).copy_(p.data)
|
||||
p.data = get_accelerator_view_from_cpu_tensor(cpu_data)
|
||||
p._vllm_is_uva_offloaded = True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user