[CPU] Remove speculative decoding stream overrides from CPUModelRunner (#47162)

Signed-off-by: jmamou <jonathan.mamou@intel.com>
This commit is contained in:
Jonathan Mamou
2026-07-01 06:12:49 +00:00
committed by GitHub
parent 93d8f834dd
commit df802a87b7
2 changed files with 3 additions and 63 deletions
+1
View File
@@ -30,6 +30,7 @@ class _EventPlaceholder:
class _StreamPlaceholder:
def __init__(self, *args, **kwargs) -> None:
self.wait_stream = noop
self.device = torch.device("cpu")
def __enter__(self, *args, **kwargs):
return self
+2 -63
View File
@@ -12,7 +12,6 @@ from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.tracing import instrument
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig
from vllm.v1.utils import CpuGpuBuffer
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
@@ -189,67 +188,6 @@ class CPUModelRunner(GPUModelRunner):
for block_id in block_ids:
kv[block_id].zero_()
# =========================================================================
# CPU-safe overrides for speculative decoding methods
# These methods override GPU-specific implementations that use CUDA streams
# =========================================================================
def _copy_draft_token_ids_to_cpu(
self, scheduler_output: "SchedulerOutput", zeros_only: bool = False
) -> None:
"""CPU-safe version: no async copy needed, tensors already on CPU."""
if self.use_async_scheduling and not (
scheduler_output.has_structured_output_requests
or self.input_batch.sampling_metadata.output_token_ids
):
return
self._draft_token_req_ids = self.input_batch.req_ids.copy()
draft_token_ids: torch.Tensor = self._draft_token_ids
if not torch.is_tensor(draft_token_ids):
return
num_reqs = draft_token_ids.shape[0]
if self.draft_token_ids_cpu is not None:
if not zeros_only:
self.draft_token_ids_cpu[:num_reqs].copy_(draft_token_ids)
else:
self.draft_token_ids_cpu[:num_reqs] = 0
def _get_draft_token_ids_cpu(self) -> tuple[list[list[int]], list[str]]:
"""CPU-safe version: no event synchronization needed."""
if isinstance(self._draft_token_ids, list):
return self._draft_token_ids, self.input_batch.req_ids
req_ids = self._draft_token_req_ids
if req_ids is None:
return [], []
if self.draft_token_ids_cpu is not None:
return self.draft_token_ids_cpu[: len(req_ids)].tolist(), req_ids
return [], []
def _copy_valid_sampled_token_count(
self, next_token_ids: torch.Tensor, valid_sampled_tokens_count: torch.Tensor
) -> None:
"""CPU-safe version: direct copy without CUDA streams."""
if self.valid_sampled_token_count_cpu is None:
return
counts = valid_sampled_tokens_count
counts_cpu = self.valid_sampled_token_count_cpu
counts_cpu[: counts.shape[0]].copy_(counts)
self.input_batch.prev_sampled_token_ids = next_token_ids.unsqueeze(1)
def _get_valid_sampled_token_count(self) -> list[int]:
"""CPU-safe version: no event synchronization needed."""
prev_sampled_token_ids = self.input_batch.prev_sampled_token_ids
if prev_sampled_token_ids is None:
return []
counts_cpu = self.valid_sampled_token_count_cpu
if counts_cpu is None:
return []
return counts_cpu[: prev_sampled_token_ids.shape[0]].tolist()
def _to_list(self, sampled_token_ids: torch.Tensor) -> list[list[int]]:
"""CPU-safe version: direct tolist() without CUDA events."""
return sampled_token_ids.tolist()
@@ -264,7 +202,8 @@ def _torch_cuda_wrapper():
class _StreamPlaceholder:
def __init__(self, *args, **kwargs) -> None:
pass
self.wait_stream = lambda *a, **kw: None
self.device = torch.device("cpu")
cuda_event = torch.Event
cuda_stream = torch.cuda.Stream