feat(cpu): add CPU support for draft model speculative decoding (#32662)

Signed-off-by: R <Ganesh.R@amd.com>
This commit is contained in:
Ganesh R
2026-04-10 11:49:52 +08:00
committed by GitHub
parent 55d037e2e5
commit 445a2a4d1a
7 changed files with 945 additions and 7 deletions
+4 -5
View File
@@ -26,7 +26,6 @@ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.triton_utils import triton
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -48,6 +47,7 @@ from vllm.v1.spec_decode.utils import (
eagle_prepare_next_token_padded_kernel,
eagle_step_update_slot_mapping_and_metadata,
extend_all_queries_by_N,
next_power_of_2,
)
from vllm.v1.utils import CpuGpuBuffer
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
@@ -689,9 +689,7 @@ class SpecDecodeBaseProposer:
max_num_tokens_per_request = (
cad.max_query_len + self.net_num_new_slots_per_request
)
BLOCK_SIZE_TOKENS = min(
256, triton.next_power_of_2(max_num_tokens_per_request)
)
BLOCK_SIZE_TOKENS = min(256, next_power_of_2(max_num_tokens_per_request))
num_blocks = (
max_num_tokens_per_request + BLOCK_SIZE_TOKENS - 1
) // BLOCK_SIZE_TOKENS
@@ -717,6 +715,7 @@ class SpecDecodeBaseProposer:
query_end_loc = cad.query_start_loc[1:] - 1
if num_rejected_tokens_gpu is not None:
query_end_loc = query_end_loc - num_rejected_tokens_gpu
copy_and_expand_eagle_inputs_kernel[grid](
# (Padded) Inputs from the target model
target_token_ids_ptr=target_token_ids,
@@ -899,7 +898,7 @@ class SpecDecodeBaseProposer:
grid = (batch_size,)
# Find the next power of 2 for block sizes
BLOCK_SIZE_TOKENS = triton.next_power_of_2(num_tokens)
BLOCK_SIZE_TOKENS = next_power_of_2(num_tokens)
eagle_prepare_next_token_padded_kernel[grid](
sampled_token_ids,
discard_request_mask,
+15 -1
View File
@@ -11,6 +11,20 @@ from vllm.v1.attention.backends.utils import (
PADDING_SLOT_ID = -1
def next_power_of_2(n: int) -> int:
"""Return the smallest power of 2 >= n."""
if n <= 0:
return 1
n -= 1
n |= n >> 1
n |= n >> 2
n |= n >> 4
n |= n >> 8
n |= n >> 16
n |= n >> 32
return n + 1
@triton.jit
def eagle_step_slot_mapping_metadata_kernel(
positions_ptr, # [batch_size] - current positions (1D view for M-RoPE)
@@ -102,8 +116,8 @@ def eagle_step_update_slot_mapping_and_metadata(
batch_size = positions_1d.shape[0]
if input_batch_size is None:
input_batch_size = batch_size
n_blocks_per_req = block_table_tensor.shape[1]
n_blocks_per_req = block_table_tensor.shape[1]
eagle_step_slot_mapping_metadata_kernel[(input_batch_size,)](
positions_1d,
block_table_tensor,