[Core] Enhance cpu<->gpu sync checking to include paged async copies (#53491)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-09-08 17:26:21 -07:00
committed by GitHub
parent 9e137d8d23
commit 4aadb0f14d
5 changed files with 332 additions and 14 deletions
+10 -7
View File
@@ -3,6 +3,7 @@
"""UVA-based CPU offloading using Unified Virtual Addressing."""
from collections.abc import Generator
from contextlib import nullcontext
import torch
import torch.nn as nn
@@ -11,6 +12,7 @@ from torch.func import functional_call
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.offloader.base import BaseOffloader, should_pin_memory
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.mem_utils import format_gib
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
@@ -126,13 +128,14 @@ class UVAOffloader(BaseOffloader):
def forward(*args, **kwargs):
module.forward = original_forward
device_state = {
# here we blindly call `to(device)`
# if the parameter is already on the device,
# it will be a no-op
k: v.to(device, non_blocking=True)
for k, v in module.state_dict().items()
}
with nullcontext() if self.pin_memory else gpu_sync_allowed():
device_state = {
# here we blindly call `to(device)`
# if the parameter is already on the device,
# it will be a no-op
k: v.to(device, non_blocking=True)
for k, v in module.state_dict().items()
}
# set `tie_weights=False` as tied weights in original model
# become untied when calling .to(device) individually