feat(cpu): add CPU support for draft model speculative decoding (#32662)

Signed-off-by: R <Ganesh.R@amd.com>
This commit is contained in:
Ganesh R
2026-04-10 11:49:52 +08:00
committed by GitHub
parent 55d037e2e5
commit 445a2a4d1a
7 changed files with 945 additions and 7 deletions
+274
View File
@@ -45,3 +45,277 @@ def _compute_slot_mapping_kernel_impl(
compute_slot_mapping_kernel = _FuncWrapper(_compute_slot_mapping_kernel_impl)
def _ensure_int64(t: torch.Tensor) -> torch.Tensor:
return t if t.dtype == torch.int64 else t.to(torch.int64)
def _eagle_prepare_inputs_padded_kernel_impl(
cu_num_draft_tokens,
valid_sampled_tokens_count,
query_start_loc_gpu,
token_indices_to_sample,
num_rejected_tokens_gpu,
num_reqs,
):
# C++ expects int64 for cu_num_draft_tokens, valid_sampled_tokens_count,
# and num_rejected_tokens_gpu, but Python allocates them as int32.
orig_rejected_dtype = num_rejected_tokens_gpu.dtype
rejected_i64 = (
num_rejected_tokens_gpu
if orig_rejected_dtype == torch.int64
else num_rejected_tokens_gpu.to(torch.int64)
)
torch.ops._C.eagle_prepare_inputs_padded_kernel_impl(
_ensure_int64(cu_num_draft_tokens),
_ensure_int64(valid_sampled_tokens_count),
query_start_loc_gpu,
token_indices_to_sample,
rejected_i64,
num_reqs,
)
if orig_rejected_dtype != torch.int64:
num_rejected_tokens_gpu.copy_(rejected_i64.to(orig_rejected_dtype))
def _eagle_prepare_next_token_padded_kernel_impl(
sampled_token_ids,
discard_request_mask,
backup_next_token_ids,
next_token_ids,
valid_sampled_tokens_count,
vocab_size,
num_sampled_tokens_per_req,
num_reqs,
stride=None,
BLOCK_SIZE_TOKENS=None,
):
# C++ reads all integer tensors as int64_t*. Output tensors are written
# in-place so we create int64 copies, call C++, and copy back.
orig_next_dtype = next_token_ids.dtype
orig_valid_dtype = valid_sampled_tokens_count.dtype
next_i64 = _ensure_int64(next_token_ids)
valid_i64 = _ensure_int64(valid_sampled_tokens_count)
torch.ops._C.eagle_prepare_next_token_padded_kernel_impl(
_ensure_int64(sampled_token_ids),
discard_request_mask,
_ensure_int64(backup_next_token_ids),
next_i64,
valid_i64,
vocab_size,
num_sampled_tokens_per_req,
num_reqs,
)
if orig_next_dtype != torch.int64:
next_token_ids.copy_(next_i64.to(orig_next_dtype))
if orig_valid_dtype != torch.int64:
valid_sampled_tokens_count.copy_(valid_i64.to(orig_valid_dtype))
def _eagle_step_slot_mapping_metadata_kernel_impl(
positions,
block_table,
stride,
seq_lens,
out_clamped_positions,
out_slot_mapping,
block_size,
max_model_len,
n_blocks_per_req,
PAD_ID,
batch_size=None,
):
assert batch_size is None or batch_size == positions.shape[0], (
f"batch_size mismatch: {batch_size} vs positions.shape[0]={positions.shape[0]}"
)
torch.ops._C.eagle_step_slot_mapping_metadata_kernel_impl(
positions,
block_table,
seq_lens,
out_clamped_positions,
out_slot_mapping,
block_size,
max_model_len,
PAD_ID,
)
def _copy_and_expand_eagle_inputs_kernel_impl(
target_token_ids_ptr,
target_positions_ptr,
next_token_ids_ptr,
out_input_ids_ptr,
out_positions_ptr,
out_is_rejected_token_mask_ptr,
out_is_masked_token_mask_ptr,
out_new_token_indices_ptr,
out_hidden_state_mapping_ptr,
query_start_loc_ptr,
query_end_loc_ptr,
padding_token_id,
parallel_drafting_token_id,
total_input_tokens,
num_padding_slots_per_request,
shift_input_ids,
BLOCK_SIZE_TOKENS=None,
BLOCK_SIZE_REQS=None,
):
"""Adapter between Triton kernel call convention and C++ implementation.
The Triton kernel uses '_ptr' suffixed parameter names and compile-time
constants (BLOCK_SIZE_TOKENS, BLOCK_SIZE_REQS) which are not needed by
the C++ implementation. C++ reads token id tensors as int64_t*.
Output tensors that are int32 need copy-back after C++ writes int64.
"""
orig_ids_dtype = out_input_ids_ptr.dtype
orig_pos_dtype = out_positions_ptr.dtype
out_ids_i64 = _ensure_int64(out_input_ids_ptr)
out_pos_i64 = _ensure_int64(out_positions_ptr)
torch.ops._C.copy_and_expand_eagle_inputs_kernel_impl(
_ensure_int64(target_token_ids_ptr),
_ensure_int64(target_positions_ptr),
_ensure_int64(next_token_ids_ptr),
out_ids_i64,
out_pos_i64,
out_is_rejected_token_mask_ptr,
out_is_masked_token_mask_ptr,
out_new_token_indices_ptr,
out_hidden_state_mapping_ptr,
query_start_loc_ptr,
query_end_loc_ptr,
padding_token_id,
parallel_drafting_token_id,
total_input_tokens,
num_padding_slots_per_request,
shift_input_ids,
)
if orig_ids_dtype != torch.int64:
out_input_ids_ptr.copy_(out_ids_i64.to(orig_ids_dtype))
if orig_pos_dtype != torch.int64:
out_positions_ptr.copy_(out_pos_i64.to(orig_pos_dtype))
def _rejection_greedy_sample_kernel_impl(
output_token_ids,
cu_num_draft_tokens,
draft_token_ids,
target_argmax,
bonus_token_ids,
is_greedy,
max_spec_len,
):
# C++ kernel expects int64 for all integer tensors.
orig_dtype = output_token_ids.dtype
output_token_ids_i64 = _ensure_int64(output_token_ids)
torch.ops._C.rejection_greedy_sample_kernel_impl(
output_token_ids_i64,
_ensure_int64(cu_num_draft_tokens),
_ensure_int64(draft_token_ids),
_ensure_int64(target_argmax),
_ensure_int64(bonus_token_ids),
is_greedy,
max_spec_len,
)
if orig_dtype != torch.int64:
output_token_ids.copy_(output_token_ids_i64.to(orig_dtype))
def _rejection_random_sample_kernel_impl(
output_token_ids,
cu_num_draft_tokens,
draft_token_ids,
draft_probs,
target_probs,
bonus_token_ids,
recovered_token_ids,
uniform_probs,
is_greedy,
max_spec_len,
vocab_size,
NO_DRAFT_PROBS=False,
):
# C++ kernel expects int64 for all integer tensors and float32 for probs.
# uniform_probs is intentionally float64 in Python to avoid exact-zero
# samples; cast to float32 here for C++ compatibility.
orig_dtype = output_token_ids.dtype
output_token_ids_i64 = _ensure_int64(output_token_ids)
torch.ops._C.rejection_random_sample_kernel_impl(
output_token_ids_i64,
_ensure_int64(cu_num_draft_tokens),
_ensure_int64(draft_token_ids),
draft_probs,
target_probs,
_ensure_int64(bonus_token_ids),
_ensure_int64(recovered_token_ids),
uniform_probs.to(torch.float32),
is_greedy,
max_spec_len,
vocab_size,
NO_DRAFT_PROBS,
)
if orig_dtype != torch.int64:
output_token_ids.copy_(output_token_ids_i64.to(orig_dtype))
def _expand_kernel_impl(
output,
input_val,
cu_num_tokens,
replace_from,
replace_to,
MAX_NUM_TOKENS=None,
):
torch.ops._C.expand_kernel_impl(
_ensure_int64(output),
_ensure_int64(input_val),
_ensure_int64(cu_num_tokens),
replace_from,
replace_to,
)
def _sample_recovered_tokens_kernel_impl(
output_token_ids,
cu_num_draft_tokens,
draft_token_ids,
draft_probs,
target_probs,
inv_q,
vocab_size,
BLOCK_SIZE=None,
NO_DRAFT_PROBS=False,
):
# C++ reads integer tensors as int64_t*; ensure correct dtype.
orig_dtype = output_token_ids.dtype
output_i64 = _ensure_int64(output_token_ids)
torch.ops._C.sample_recovered_tokens_kernel_impl(
output_i64,
_ensure_int64(cu_num_draft_tokens),
_ensure_int64(draft_token_ids),
draft_probs,
target_probs,
inv_q,
vocab_size,
NO_DRAFT_PROBS,
)
if orig_dtype != torch.int64:
output_token_ids.copy_(output_i64.to(orig_dtype))
eagle_prepare_inputs_padded_kernel = _FuncWrapper(
_eagle_prepare_inputs_padded_kernel_impl
)
eagle_prepare_next_token_padded_kernel = _FuncWrapper(
_eagle_prepare_next_token_padded_kernel_impl
)
copy_and_expand_eagle_inputs_kernel = _FuncWrapper(
_copy_and_expand_eagle_inputs_kernel_impl
)
eagle_step_slot_mapping_metadata_kernel = _FuncWrapper(
_eagle_step_slot_mapping_metadata_kernel_impl
)
rejection_greedy_sample_kernel = _FuncWrapper(_rejection_greedy_sample_kernel_impl)
rejection_random_sample_kernel = _FuncWrapper(_rejection_random_sample_kernel_impl)
expand_kernel = _FuncWrapper(_expand_kernel_impl)
sample_recovered_tokens_kernel = _FuncWrapper(_sample_recovered_tokens_kernel_impl)
+4 -5
View File
@@ -26,7 +26,6 @@ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.triton_utils import triton
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -48,6 +47,7 @@ from vllm.v1.spec_decode.utils import (
eagle_prepare_next_token_padded_kernel,
eagle_step_update_slot_mapping_and_metadata,
extend_all_queries_by_N,
next_power_of_2,
)
from vllm.v1.utils import CpuGpuBuffer
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
@@ -689,9 +689,7 @@ class SpecDecodeBaseProposer:
max_num_tokens_per_request = (
cad.max_query_len + self.net_num_new_slots_per_request
)
BLOCK_SIZE_TOKENS = min(
256, triton.next_power_of_2(max_num_tokens_per_request)
)
BLOCK_SIZE_TOKENS = min(256, next_power_of_2(max_num_tokens_per_request))
num_blocks = (
max_num_tokens_per_request + BLOCK_SIZE_TOKENS - 1
) // BLOCK_SIZE_TOKENS
@@ -717,6 +715,7 @@ class SpecDecodeBaseProposer:
query_end_loc = cad.query_start_loc[1:] - 1
if num_rejected_tokens_gpu is not None:
query_end_loc = query_end_loc - num_rejected_tokens_gpu
copy_and_expand_eagle_inputs_kernel[grid](
# (Padded) Inputs from the target model
target_token_ids_ptr=target_token_ids,
@@ -899,7 +898,7 @@ class SpecDecodeBaseProposer:
grid = (batch_size,)
# Find the next power of 2 for block sizes
BLOCK_SIZE_TOKENS = triton.next_power_of_2(num_tokens)
BLOCK_SIZE_TOKENS = next_power_of_2(num_tokens)
eagle_prepare_next_token_padded_kernel[grid](
sampled_token_ids,
discard_request_mask,
+15 -1
View File
@@ -11,6 +11,20 @@ from vllm.v1.attention.backends.utils import (
PADDING_SLOT_ID = -1
def next_power_of_2(n: int) -> int:
"""Return the smallest power of 2 >= n."""
if n <= 0:
return 1
n -= 1
n |= n >> 1
n |= n >> 2
n |= n >> 4
n |= n >> 8
n |= n >> 16
n |= n >> 32
return n + 1
@triton.jit
def eagle_step_slot_mapping_metadata_kernel(
positions_ptr, # [batch_size] - current positions (1D view for M-RoPE)
@@ -102,8 +116,8 @@ def eagle_step_update_slot_mapping_and_metadata(
batch_size = positions_1d.shape[0]
if input_batch_size is None:
input_batch_size = batch_size
n_blocks_per_req = block_table_tensor.shape[1]
n_blocks_per_req = block_table_tensor.shape[1]
eagle_step_slot_mapping_metadata_kernel[(input_batch_size,)](
positions_1d,
block_table_tensor,
+121 -1
View File
@@ -11,6 +11,8 @@ from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.tracing import instrument
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.utils import CpuGpuBuffer
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
@@ -23,7 +25,7 @@ class CPUModelRunner(GPUModelRunner):
super().__init__(vllm_config, device)
assert device == torch.device("cpu")
assert self.speculative_config is None, "spec decode is not supported."
# Note: speculative decoding is now supported on CPU with C++ native impls
self.use_cuda_graph = False
self.cascade_attn_enabled = False
@@ -61,6 +63,34 @@ class CPUModelRunner(GPUModelRunner):
cpu_tl.compute_slot_mapping_kernel
)
# Speculative decoding fallbacks
import vllm.v1.sample.rejection_sampler
import vllm.v1.spec_decode.eagle
import vllm.v1.spec_decode.utils
vllm.v1.spec_decode.eagle.eagle_prepare_inputs_padded_kernel = (
cpu_tl.eagle_prepare_inputs_padded_kernel
)
vllm.v1.spec_decode.eagle.eagle_prepare_next_token_padded_kernel = (
cpu_tl.eagle_prepare_next_token_padded_kernel
)
vllm.v1.spec_decode.eagle.copy_and_expand_eagle_inputs_kernel = (
cpu_tl.copy_and_expand_eagle_inputs_kernel
)
vllm.v1.spec_decode.utils.eagle_step_slot_mapping_metadata_kernel = (
cpu_tl.eagle_step_slot_mapping_metadata_kernel
)
vllm.v1.sample.rejection_sampler.rejection_greedy_sample_kernel = (
cpu_tl.rejection_greedy_sample_kernel
)
vllm.v1.sample.rejection_sampler.rejection_random_sample_kernel = (
cpu_tl.rejection_random_sample_kernel
)
vllm.v1.sample.rejection_sampler.expand_kernel = cpu_tl.expand_kernel
vllm.v1.sample.rejection_sampler.sample_recovered_tokens_kernel = (
cpu_tl.sample_recovered_tokens_kernel
)
@instrument(span_name="Loading (CPU)")
def load_model(self, load_dummy_weights: bool = False) -> None:
if load_dummy_weights:
@@ -74,6 +104,10 @@ class CPUModelRunner(GPUModelRunner):
if self.lora_config:
self.model = self.load_lora_model(self.model, self.vllm_config, self.device)
if hasattr(self, "drafter"):
logger.info_once("Loading drafter model...")
self.drafter.load_model(self.model)
def get_model(self) -> nn.Module:
return self.model
@@ -89,8 +123,29 @@ class CPUModelRunner(GPUModelRunner):
)
)
# Warm up drafter for speculative decoding
if self.speculative_config and (self.speculative_config.uses_draft_model()):
from vllm.v1.spec_decode.draft_model import DraftModelProposer
if isinstance(self.drafter, (DraftModelProposer)):
logger.info("Warming up drafter model...")
self.drafter.dummy_run(max(16, self.max_num_reqs))
logger.info("Warming up done.")
def initialize_kv_cache(
self,
kv_cache_config: KVCacheConfig,
is_profiling: bool = False,
) -> None:
super().initialize_kv_cache(kv_cache_config, is_profiling)
if self.speculative_config:
if self.speculative_config.use_eagle():
logger.info("EAGLE drafter KV cache initialized for CPU backend")
elif self.speculative_config.uses_draft_model():
logger.info("Draft model KV cache initialized for CPU backend")
def _init_device_properties(self) -> None:
pass
@@ -102,6 +157,71 @@ class CPUModelRunner(GPUModelRunner):
# so stale KV cache data never affects computation.
pass
# =========================================================================
# CPU-safe overrides for speculative decoding methods
# These methods override GPU-specific implementations that use CUDA streams
# =========================================================================
def _copy_draft_token_ids_to_cpu(
self, scheduler_output: "SchedulerOutput", zeros_only: bool = False
) -> None:
"""CPU-safe version: no async copy needed, tensors already on CPU."""
if self.use_async_scheduling and not (
scheduler_output.has_structured_output_requests
or self.input_batch.sampling_metadata.output_token_ids
):
return
self._draft_token_req_ids = self.input_batch.req_ids.copy()
draft_token_ids: torch.Tensor = self._draft_token_ids
if not torch.is_tensor(draft_token_ids):
return
num_reqs = draft_token_ids.shape[0]
if self.draft_token_ids_cpu is not None:
if not zeros_only:
self.draft_token_ids_cpu[:num_reqs].copy_(draft_token_ids)
else:
self.draft_token_ids_cpu[:num_reqs] = 0
def _get_draft_token_ids_cpu(self) -> tuple[list[list[int]], list[str]]:
"""CPU-safe version: no event synchronization needed."""
if isinstance(self._draft_token_ids, list):
return self._draft_token_ids, self.input_batch.req_ids
req_ids = self._draft_token_req_ids
if req_ids is None:
return [], []
if self.draft_token_ids_cpu is not None:
return self.draft_token_ids_cpu[: len(req_ids)].tolist(), req_ids
return [], []
def _copy_valid_sampled_token_count(
self, next_token_ids: torch.Tensor, valid_sampled_tokens_count: torch.Tensor
) -> None:
"""CPU-safe version: direct copy without CUDA streams."""
if self.valid_sampled_token_count_cpu is None:
return
counts = valid_sampled_tokens_count
counts_cpu = self.valid_sampled_token_count_cpu
counts_cpu[: counts.shape[0]].copy_(counts)
self.input_batch.prev_sampled_token_ids = next_token_ids.unsqueeze(1)
def _get_valid_sampled_token_count(self) -> list[int]:
"""CPU-safe version: no event synchronization needed."""
prev_sampled_token_ids = self.input_batch.prev_sampled_token_ids
if prev_sampled_token_ids is None:
return []
counts_cpu = self.valid_sampled_token_count_cpu
if counts_cpu is None:
return []
return counts_cpu[: prev_sampled_token_ids.shape[0]].tolist()
def _to_list(self, sampled_token_ids: torch.Tensor) -> list[list[int]]:
"""CPU-safe version: direct tolist() without CUDA events."""
return sampled_token_ids.tolist()
@contextmanager
def _torch_cuda_wrapper():