diff --git a/vllm/multimodal/inputs.py b/vllm/multimodal/inputs.py index 71f17a8648..dc96c366ad 100644 --- a/vllm/multimodal/inputs.py +++ b/vllm/multimodal/inputs.py @@ -457,6 +457,8 @@ class BaseMultiModalField(ABC): device = "cpu" if pin_memory and self.keep_on_cpu: pin_memory = False + if device == "cpu" or device == torch.device("cpu"): + pin_memory = False batch = [elem.data for elem in elems] out = self._reduce_data(batch, pin_memory=pin_memory) diff --git a/vllm/v1/worker/cpu/shm.py b/vllm/v1/worker/cpu/shm.py index bd1f96c71e..970e8a2d41 100644 --- a/vllm/v1/worker/cpu/shm.py +++ b/vllm/v1/worker/cpu/shm.py @@ -24,12 +24,17 @@ def fake_pin_memory(self: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tens class _EventPlaceholder: def __init__(self, *args, **kwargs) -> None: self.record = noop + self.wait = noop self.synchronize = noop class _StreamPlaceholder: def __init__(self, *args, **kwargs) -> None: self.wait_stream = noop + self.wait_event = noop + self.record_event = noop + self.synchronize = noop + self.query = lambda: True self.device = torch.device("cpu") def __enter__(self, *args, **kwargs): @@ -55,6 +60,7 @@ torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder() torch.accelerator.synchronize = noop torch.accelerator.empty_cache = noop torch.Tensor.pin_memory = fake_pin_memory +torch.Tensor.record_stream = noop torch.accelerator.get_memory_info = get_memory_info # Patch vLLM torch utils @@ -80,3 +86,9 @@ import vllm.v1.worker.gpu.buffer_utils as gpu_buffer_utils import vllm.v1.worker.cpu.buffer_utils as cpu_buffer_utils gpu_buffer_utils.UvaBuffer = cpu_buffer_utils.UvaBuffer + +# Patch Triton +from vllm.triton_utils import HAS_TRITON, tl + +if HAS_TRITON: + tl.debug_barrier = noop diff --git a/vllm/v1/worker/gpu/model_states/encoder_decoder.py b/vllm/v1/worker/gpu/model_states/encoder_decoder.py index f759c0b1e1..618984c97f 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_decoder.py +++ b/vllm/v1/worker/gpu/model_states/encoder_decoder.py @@ -9,6 +9,7 @@ import torch.nn as nn from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.kv_cache_interface import CrossAttentionSpec, KVCacheConfig from vllm.v1.worker.gpu.attn_utils import build_attn_metadata from vllm.v1.worker.gpu.input_batch import InputBatch @@ -157,7 +158,9 @@ class EncoderDecoderModelState(ModelState): for_capture: bool, num_reqs: int, ) -> dict[int, tuple[torch.Tensor, np.ndarray]]: - encoder_seq_lens = torch.zeros(num_reqs, dtype=torch.int32, pin_memory=True) + encoder_seq_lens = torch.zeros( + num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY + ) encoder_seq_lens_np = encoder_seq_lens.numpy() if not for_capture: # During normal execution, use actual encoder lengths.