[Bugfix] Respect VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY in prefetch offloader (#37699)
Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -20,8 +20,7 @@ import torch.nn as nn
|
||||
# Import prefetch_ops to register custom ops at module load time
|
||||
import vllm.model_executor.offloader.prefetch_ops # noqa: F401
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.offloader.base import BaseOffloader
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.model_executor.offloader.base import BaseOffloader, should_pin_memory
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -528,7 +527,7 @@ class _ModuleOffloader:
|
||||
gpu_buffer = offloader._gpu_buffer
|
||||
assert cpu_storage is not None, "CPU storage not initialized"
|
||||
assert gpu_buffer is not None, "GPU buffer not assigned"
|
||||
assert not is_pin_memory_available() or cpu_storage.is_pinned(), (
|
||||
assert not should_pin_memory() or cpu_storage.is_pinned(), (
|
||||
f"CPU storage for {name} is not pinned! "
|
||||
"non_blocking=True H2D copy from non-pinned memory "
|
||||
"causes stream synchronization that breaks "
|
||||
@@ -629,7 +628,7 @@ class _CpuParamOffloader(_BaseParamOffloader):
|
||||
original GPU tensor is garbage collected.
|
||||
"""
|
||||
param = self._param
|
||||
pin_memory = is_pin_memory_available()
|
||||
pin_memory = should_pin_memory()
|
||||
|
||||
# Create pinned CPU storage and copy current GPU data
|
||||
self._cpu_storage = torch.empty_strided(
|
||||
@@ -666,7 +665,7 @@ class _CpuParamOffloader(_BaseParamOffloader):
|
||||
param = self._param
|
||||
|
||||
if param.data.device.type == "cpu":
|
||||
if is_pin_memory_available() and not param.data.is_pinned():
|
||||
if should_pin_memory() and not param.data.is_pinned():
|
||||
pinned = torch.empty_strided(
|
||||
size=param.data.size(),
|
||||
stride=param.data.stride(),
|
||||
|
||||
Reference in New Issue
Block a user