[Bugfix] Respect VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY in prefetch offloader (#37699)
Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -10,7 +10,9 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import OffloadConfig
|
||||
@@ -18,6 +20,18 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def should_pin_memory() -> bool:
|
||||
"""Check if pinned memory should be used for weight offloading.
|
||||
|
||||
Combines the platform capability check with the user override env var.
|
||||
On unified-memory systems (e.g. GH200) pinned memory eats into GPU
|
||||
memory, so users can disable it via VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY.
|
||||
"""
|
||||
return (
|
||||
is_pin_memory_available() and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY
|
||||
)
|
||||
|
||||
|
||||
"""
|
||||
class relation:
|
||||
|
||||
|
||||
Reference in New Issue
Block a user