[Core] Enhance cpu<->gpu sync checking to include paged async copies (#53491)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
"""UVA-based CPU offloading using Unified Virtual Addressing."""
|
||||
|
||||
from collections.abc import Generator
|
||||
from contextlib import nullcontext
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -11,6 +12,7 @@ from torch.func import functional_call
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.offloader.base import BaseOffloader, should_pin_memory
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
@@ -126,13 +128,14 @@ class UVAOffloader(BaseOffloader):
|
||||
|
||||
def forward(*args, **kwargs):
|
||||
module.forward = original_forward
|
||||
device_state = {
|
||||
# here we blindly call `to(device)`
|
||||
# if the parameter is already on the device,
|
||||
# it will be a no-op
|
||||
k: v.to(device, non_blocking=True)
|
||||
for k, v in module.state_dict().items()
|
||||
}
|
||||
with nullcontext() if self.pin_memory else gpu_sync_allowed():
|
||||
device_state = {
|
||||
# here we blindly call `to(device)`
|
||||
# if the parameter is already on the device,
|
||||
# it will be a no-op
|
||||
k: v.to(device, non_blocking=True)
|
||||
for k, v in module.state_dict().items()
|
||||
}
|
||||
|
||||
# set `tie_weights=False` as tied weights in original model
|
||||
# become untied when calling .to(device) individually
|
||||
|
||||
Reference in New Issue
Block a user