[Bugfix] Fix multi-modal support on CPU MRV2 (#50073)

Signed-off-by: jiang1.li <jiang1.li@intel.com>

vllm/multimodal/inputs.py

@@ -457,6 +457,8 @@ class BaseMultiModalField(ABC):
             device = "cpu"
         if pin_memory and self.keep_on_cpu:
             pin_memory = False
+        if device == "cpu" or device == torch.device("cpu"):
+            pin_memory = False
 
         batch = [elem.data for elem in elems]
         out = self._reduce_data(batch, pin_memory=pin_memory)

vllm/v1/worker/cpu/shm.py

@@ -24,12 +24,17 @@ def fake_pin_memory(self: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tens
 class _EventPlaceholder:
     def __init__(self, *args, **kwargs) -> None:
         self.record = noop
+        self.wait = noop
         self.synchronize = noop
 
 
 class _StreamPlaceholder:
     def __init__(self, *args, **kwargs) -> None:
         self.wait_stream = noop
+        self.wait_event = noop
+        self.record_event = noop
+        self.synchronize = noop
+        self.query = lambda: True
         self.device = torch.device("cpu")
 
     def __enter__(self, *args, **kwargs):
@@ -55,6 +60,7 @@ torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder()
 torch.accelerator.synchronize = noop
 torch.accelerator.empty_cache = noop
 torch.Tensor.pin_memory = fake_pin_memory
+torch.Tensor.record_stream = noop
 torch.accelerator.get_memory_info = get_memory_info
 
 # Patch vLLM torch utils
@@ -80,3 +86,9 @@ import vllm.v1.worker.gpu.buffer_utils as gpu_buffer_utils
 import vllm.v1.worker.cpu.buffer_utils as cpu_buffer_utils
 
 gpu_buffer_utils.UvaBuffer = cpu_buffer_utils.UvaBuffer
+
+# Patch Triton
+from vllm.triton_utils import HAS_TRITON, tl
+
+if HAS_TRITON:
+    tl.debug_barrier = noop

vllm/v1/worker/gpu/model_states/encoder_decoder.py

@@ -9,6 +9,7 @@ import torch.nn as nn
 
 from vllm.config import VllmConfig
 from vllm.config.compilation import CUDAGraphMode
+from vllm.utils.torch_utils import PIN_MEMORY
 from vllm.v1.kv_cache_interface import CrossAttentionSpec, KVCacheConfig
 from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
 from vllm.v1.worker.gpu.input_batch import InputBatch
@@ -157,7 +158,9 @@ class EncoderDecoderModelState(ModelState):
         for_capture: bool,
         num_reqs: int,
     ) -> dict[int, tuple[torch.Tensor, np.ndarray]]:
-        encoder_seq_lens = torch.zeros(num_reqs, dtype=torch.int32, pin_memory=True)
+        encoder_seq_lens = torch.zeros(
+            num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY
+        )
         encoder_seq_lens_np = encoder_seq_lens.numpy()
         if not for_capture:
             # During normal execution, use actual encoder lengths.