feat(cpu): add CPU support for draft model speculative decoding (#32662)
Signed-off-by: R <Ganesh.R@amd.com>
This commit is contained in:
@@ -26,7 +26,6 @@ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
|
||||
from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -48,6 +47,7 @@ from vllm.v1.spec_decode.utils import (
|
||||
eagle_prepare_next_token_padded_kernel,
|
||||
eagle_step_update_slot_mapping_and_metadata,
|
||||
extend_all_queries_by_N,
|
||||
next_power_of_2,
|
||||
)
|
||||
from vllm.v1.utils import CpuGpuBuffer
|
||||
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
|
||||
@@ -689,9 +689,7 @@ class SpecDecodeBaseProposer:
|
||||
max_num_tokens_per_request = (
|
||||
cad.max_query_len + self.net_num_new_slots_per_request
|
||||
)
|
||||
BLOCK_SIZE_TOKENS = min(
|
||||
256, triton.next_power_of_2(max_num_tokens_per_request)
|
||||
)
|
||||
BLOCK_SIZE_TOKENS = min(256, next_power_of_2(max_num_tokens_per_request))
|
||||
num_blocks = (
|
||||
max_num_tokens_per_request + BLOCK_SIZE_TOKENS - 1
|
||||
) // BLOCK_SIZE_TOKENS
|
||||
@@ -717,6 +715,7 @@ class SpecDecodeBaseProposer:
|
||||
query_end_loc = cad.query_start_loc[1:] - 1
|
||||
if num_rejected_tokens_gpu is not None:
|
||||
query_end_loc = query_end_loc - num_rejected_tokens_gpu
|
||||
|
||||
copy_and_expand_eagle_inputs_kernel[grid](
|
||||
# (Padded) Inputs from the target model
|
||||
target_token_ids_ptr=target_token_ids,
|
||||
@@ -899,7 +898,7 @@ class SpecDecodeBaseProposer:
|
||||
grid = (batch_size,)
|
||||
|
||||
# Find the next power of 2 for block sizes
|
||||
BLOCK_SIZE_TOKENS = triton.next_power_of_2(num_tokens)
|
||||
BLOCK_SIZE_TOKENS = next_power_of_2(num_tokens)
|
||||
eagle_prepare_next_token_padded_kernel[grid](
|
||||
sampled_token_ids,
|
||||
discard_request_mask,
|
||||
|
||||
@@ -11,6 +11,20 @@ from vllm.v1.attention.backends.utils import (
|
||||
PADDING_SLOT_ID = -1
|
||||
|
||||
|
||||
def next_power_of_2(n: int) -> int:
|
||||
"""Return the smallest power of 2 >= n."""
|
||||
if n <= 0:
|
||||
return 1
|
||||
n -= 1
|
||||
n |= n >> 1
|
||||
n |= n >> 2
|
||||
n |= n >> 4
|
||||
n |= n >> 8
|
||||
n |= n >> 16
|
||||
n |= n >> 32
|
||||
return n + 1
|
||||
|
||||
|
||||
@triton.jit
|
||||
def eagle_step_slot_mapping_metadata_kernel(
|
||||
positions_ptr, # [batch_size] - current positions (1D view for M-RoPE)
|
||||
@@ -102,8 +116,8 @@ def eagle_step_update_slot_mapping_and_metadata(
|
||||
batch_size = positions_1d.shape[0]
|
||||
if input_batch_size is None:
|
||||
input_batch_size = batch_size
|
||||
n_blocks_per_req = block_table_tensor.shape[1]
|
||||
|
||||
n_blocks_per_req = block_table_tensor.shape[1]
|
||||
eagle_step_slot_mapping_metadata_kernel[(input_batch_size,)](
|
||||
positions_1d,
|
||||
block_table_tensor,
|
||||
|
||||
Reference in New Issue
Block a user