From df802a87b77f21e0bf5f8d358633d5dd00ac5400 Mon Sep 17 00:00:00 2001 From: Jonathan Mamou Date: Wed, 1 Jul 2026 09:12:49 +0300 Subject: [PATCH] [CPU] Remove speculative decoding stream overrides from CPUModelRunner (#47162) Signed-off-by: jmamou --- vllm/v1/worker/cpu/shm.py | 1 + vllm/v1/worker/cpu_model_runner.py | 65 +----------------------------- 2 files changed, 3 insertions(+), 63 deletions(-) diff --git a/vllm/v1/worker/cpu/shm.py b/vllm/v1/worker/cpu/shm.py index d6c38ee459..deec52e44b 100644 --- a/vllm/v1/worker/cpu/shm.py +++ b/vllm/v1/worker/cpu/shm.py @@ -30,6 +30,7 @@ class _EventPlaceholder: class _StreamPlaceholder: def __init__(self, *args, **kwargs) -> None: self.wait_stream = noop + self.device = torch.device("cpu") def __enter__(self, *args, **kwargs): return self diff --git a/vllm/v1/worker/cpu_model_runner.py b/vllm/v1/worker/cpu_model_runner.py index 87f8cb154d..e6fcac8fc5 100644 --- a/vllm/v1/worker/cpu_model_runner.py +++ b/vllm/v1/worker/cpu_model_runner.py @@ -12,7 +12,6 @@ from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.model_executor.model_loader import get_model from vllm.tracing import instrument -from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig from vllm.v1.utils import CpuGpuBuffer from vllm.v1.worker.gpu_model_runner import GPUModelRunner @@ -189,67 +188,6 @@ class CPUModelRunner(GPUModelRunner): for block_id in block_ids: kv[block_id].zero_() - # ========================================================================= - # CPU-safe overrides for speculative decoding methods - # These methods override GPU-specific implementations that use CUDA streams - # ========================================================================= - - def _copy_draft_token_ids_to_cpu( - self, scheduler_output: "SchedulerOutput", zeros_only: bool = False - ) -> None: - """CPU-safe version: no async copy needed, tensors already on CPU.""" - if self.use_async_scheduling and not ( - scheduler_output.has_structured_output_requests - or self.input_batch.sampling_metadata.output_token_ids - ): - return - self._draft_token_req_ids = self.input_batch.req_ids.copy() - - draft_token_ids: torch.Tensor = self._draft_token_ids - if not torch.is_tensor(draft_token_ids): - return - - num_reqs = draft_token_ids.shape[0] - if self.draft_token_ids_cpu is not None: - if not zeros_only: - self.draft_token_ids_cpu[:num_reqs].copy_(draft_token_ids) - else: - self.draft_token_ids_cpu[:num_reqs] = 0 - - def _get_draft_token_ids_cpu(self) -> tuple[list[list[int]], list[str]]: - """CPU-safe version: no event synchronization needed.""" - if isinstance(self._draft_token_ids, list): - return self._draft_token_ids, self.input_batch.req_ids - req_ids = self._draft_token_req_ids - if req_ids is None: - return [], [] - if self.draft_token_ids_cpu is not None: - return self.draft_token_ids_cpu[: len(req_ids)].tolist(), req_ids - return [], [] - - def _copy_valid_sampled_token_count( - self, next_token_ids: torch.Tensor, valid_sampled_tokens_count: torch.Tensor - ) -> None: - """CPU-safe version: direct copy without CUDA streams.""" - if self.valid_sampled_token_count_cpu is None: - return - - counts = valid_sampled_tokens_count - counts_cpu = self.valid_sampled_token_count_cpu - counts_cpu[: counts.shape[0]].copy_(counts) - self.input_batch.prev_sampled_token_ids = next_token_ids.unsqueeze(1) - - def _get_valid_sampled_token_count(self) -> list[int]: - """CPU-safe version: no event synchronization needed.""" - prev_sampled_token_ids = self.input_batch.prev_sampled_token_ids - if prev_sampled_token_ids is None: - return [] - - counts_cpu = self.valid_sampled_token_count_cpu - if counts_cpu is None: - return [] - return counts_cpu[: prev_sampled_token_ids.shape[0]].tolist() - def _to_list(self, sampled_token_ids: torch.Tensor) -> list[list[int]]: """CPU-safe version: direct tolist() without CUDA events.""" return sampled_token_ids.tolist() @@ -264,7 +202,8 @@ def _torch_cuda_wrapper(): class _StreamPlaceholder: def __init__(self, *args, **kwargs) -> None: - pass + self.wait_stream = lambda *a, **kw: None + self.device = torch.device("cpu") cuda_event = torch.Event cuda_stream = torch.cuda.Stream