[CPU][Spec Decode] Enable DFlash SD for CPU (#44029)

Signed-off-by: guybd <guy.boudoukh@intel.com>
Signed-off-by: Guy Boudoukh <guy.boudoukh@intel.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
guybd
2026-06-25 15:32:48 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9b215ae60b
commit 2396d91e93
8 changed files with 266 additions and 8 deletions
+1 -1
View File
@@ -131,7 +131,7 @@ class DFlashQwen3Attention(nn.Module):
with the context K/V from the target model's hidden states. This forward op
computes attention for the query tokens only.
See also: precompute_and_store_context_kv"""
qkv = F.linear(hidden_states, self.qkv_proj.weight, self.qkv_proj.bias)
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
# Per-head RMSNorm
+134
View File
@@ -197,6 +197,133 @@ def _copy_and_expand_eagle_inputs_kernel_impl(
out_positions_ptr.copy_(out_pos_i64.to(orig_pos_dtype))
def _copy_and_expand_dflash_inputs_kernel_impl(
next_token_ids_ptr,
target_positions_ptr,
out_input_ids_ptr,
out_context_positions_ptr,
out_query_positions_ptr,
out_context_slot_mapping_ptr,
out_query_slot_mapping_ptr,
out_token_indices_ptr,
block_table_ptr,
block_table_stride,
query_start_loc_ptr,
num_rejected_tokens_ptr,
parallel_drafting_token_id,
block_size,
num_query_per_req,
num_speculative_tokens,
total_input_tokens,
BLOCK_SIZE=None,
HAS_NUM_REJECTED=False,
):
"""Adapter between the DFlash Triton launch and the C++ CPU op."""
assert block_table_stride == block_table_ptr.stride(0), (
"block_table_stride mismatch: "
f"{block_table_stride} vs {block_table_ptr.stride(0)}"
)
orig_ids_dtype = out_input_ids_ptr.dtype
orig_context_positions_dtype = out_context_positions_ptr.dtype
orig_query_positions_dtype = out_query_positions_ptr.dtype
orig_context_slot_mapping_dtype = out_context_slot_mapping_ptr.dtype
orig_query_slot_mapping_dtype = out_query_slot_mapping_ptr.dtype
out_ids_i64 = _ensure_int64(out_input_ids_ptr)
out_context_positions_i64 = _ensure_int64(out_context_positions_ptr)
out_query_positions_i64 = _ensure_int64(out_query_positions_ptr)
out_context_slot_mapping_i64 = _ensure_int64(out_context_slot_mapping_ptr)
out_query_slot_mapping_i64 = _ensure_int64(out_query_slot_mapping_ptr)
rejected_i64 = _ensure_int64(num_rejected_tokens_ptr) if HAS_NUM_REJECTED else None
if hasattr(torch.ops._C, "copy_and_expand_dflash_inputs_kernel_impl"):
torch.ops._C.copy_and_expand_dflash_inputs_kernel_impl(
_ensure_int64(next_token_ids_ptr),
_ensure_int64(target_positions_ptr),
out_ids_i64,
out_context_positions_i64,
out_query_positions_i64,
out_context_slot_mapping_i64,
out_query_slot_mapping_i64,
out_token_indices_ptr,
block_table_ptr,
query_start_loc_ptr,
rejected_i64,
parallel_drafting_token_id,
block_size,
num_query_per_req,
num_speculative_tokens,
total_input_tokens,
HAS_NUM_REJECTED,
)
else:
next_ids_i64 = _ensure_int64(next_token_ids_ptr)
target_positions_i64 = _ensure_int64(target_positions_ptr)
block_table_stride = block_table_ptr.stride(0)
num_reqs = query_start_loc_ptr.shape[0] - 1
for req_idx in range(num_reqs):
ctx_start = int(query_start_loc_ptr[req_idx].item())
ctx_end = int(query_start_loc_ptr[req_idx + 1].item())
num_ctx = ctx_end - ctx_start
valid_ctx_end = ctx_end
if rejected_i64 is not None:
valid_ctx_end -= int(rejected_i64[req_idx].item())
# Guard against out-of-bounds: ensure valid_ctx_end > ctx_start.
valid_ctx_end = max(valid_ctx_end, ctx_start + 1)
last_pos = int(target_positions_i64[valid_ctx_end - 1].item())
for j in range(num_ctx):
ctx_idx = ctx_start + j
ctx_pos_idx = min(ctx_idx, total_input_tokens - 1)
position = int(target_positions_i64[ctx_pos_idx].item())
block_num = min(position // block_size, block_table_stride - 1)
block_id = int(block_table_ptr[req_idx, block_num].item())
slot = block_id * block_size + (position % block_size)
out_context_positions_i64[ctx_idx] = position
out_context_slot_mapping_i64[ctx_idx] = slot
for query_off in range(num_query_per_req):
query_out = req_idx * num_query_per_req + query_off
position = last_pos + 1 + query_off
block_num = min(position // block_size, block_table_stride - 1)
block_id = int(block_table_ptr[req_idx, block_num].item())
slot = block_id * block_size + (position % block_size)
out_query_positions_i64[query_out] = position
out_query_slot_mapping_i64[query_out] = slot
out_ids_i64[query_out] = (
int(next_ids_i64[req_idx].item())
if query_off == 0
else parallel_drafting_token_id
)
if query_off > 0:
sample_out_idx = req_idx * num_speculative_tokens + (query_off - 1)
out_token_indices_ptr[sample_out_idx] = query_out
if orig_ids_dtype != torch.int64:
out_input_ids_ptr.copy_(out_ids_i64.to(orig_ids_dtype))
if orig_context_positions_dtype != torch.int64:
out_context_positions_ptr.copy_(
out_context_positions_i64.to(orig_context_positions_dtype)
)
if orig_query_positions_dtype != torch.int64:
out_query_positions_ptr.copy_(
out_query_positions_i64.to(orig_query_positions_dtype)
)
if orig_context_slot_mapping_dtype != torch.int64:
out_context_slot_mapping_ptr.copy_(
out_context_slot_mapping_i64.to(orig_context_slot_mapping_dtype)
)
if orig_query_slot_mapping_dtype != torch.int64:
out_query_slot_mapping_ptr.copy_(
out_query_slot_mapping_i64.to(orig_query_slot_mapping_dtype)
)
def _rejection_greedy_sample_kernel_impl(
output_token_ids,
cu_num_draft_tokens,
@@ -303,6 +430,10 @@ def _sample_recovered_tokens_kernel_impl(
NO_DRAFT_PROBS=False,
USE_FP64_GUMBEL=False,
):
# USE_FP64_GUMBEL only controls the gumbel-noise precision, which the caller
# has already applied to `inv_q` (fp64 vs fp32). The CPU kernel consumes
# `inv_q` directly, so the flag is accepted for interface parity and the
# value is read at its existing dtype.
# C++ reads integer tensors as int64_t*; ensure correct dtype.
orig_dtype = output_token_ids.dtype
output_i64 = _ensure_int64(output_token_ids)
@@ -330,6 +461,9 @@ eagle_prepare_next_token_padded_kernel = _FuncWrapper(
copy_and_expand_eagle_inputs_kernel = _FuncWrapper(
_copy_and_expand_eagle_inputs_kernel_impl
)
copy_and_expand_dflash_inputs_kernel = _FuncWrapper(
_copy_and_expand_dflash_inputs_kernel_impl
)
eagle_step_slot_mapping_metadata_kernel = _FuncWrapper(
_eagle_step_slot_mapping_metadata_kernel_impl
)
+4
View File
@@ -63,6 +63,10 @@ class CPUAttentionBackend(AttentionBackend):
def get_name() -> str:
return "CPU_ATTN"
@classmethod
def supports_non_causal(cls) -> bool:
return True
@classmethod
def supports_attn_type(cls, attn_type: str) -> bool:
"""CPU attention supports decoder,
+6 -4
View File
@@ -10,10 +10,12 @@ from typing_extensions import override
from vllm.config import VllmConfig
from vllm.forward_context import set_forward_context
from vllm.logger import init_logger
from vllm.triton_utils import triton
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer
from vllm.v1.spec_decode.utils import copy_and_expand_dflash_inputs_kernel
from vllm.v1.spec_decode.utils import (
copy_and_expand_dflash_inputs_kernel,
next_power_of_2,
)
logger = init_logger(__name__)
@@ -126,8 +128,8 @@ class DFlashProposer(SpecDecodeBaseProposer):
# and token_indices_to_sample
max_ctx_per_req = cad.max_query_len
max_tokens_per_req = max_ctx_per_req + num_query_per_req
BLOCK_SIZE = min(256, triton.next_power_of_2(max_tokens_per_req))
num_blocks = triton.cdiv(max_tokens_per_req, BLOCK_SIZE)
BLOCK_SIZE = min(256, next_power_of_2(max_tokens_per_req))
num_blocks = (max_tokens_per_req + BLOCK_SIZE - 1) // BLOCK_SIZE
grid = (batch_size, num_blocks)
has_num_rejected = num_rejected_tokens_gpu is not None
+14 -2
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from contextlib import contextmanager
from typing import Any
@@ -78,7 +79,7 @@ class CPUModelRunner(GPUModelRunner):
# Speculative decoding fallbacks
import vllm.v1.sample.rejection_sampler
import vllm.v1.spec_decode.llm_base_proposer
import vllm.v1.spec_decode.utils
import vllm.v1.spec_decode.utils as spec_decode_utils
vllm.v1.spec_decode.llm_base_proposer.eagle_prepare_inputs_padded_kernel = (
cpu_tl.eagle_prepare_inputs_padded_kernel
@@ -89,7 +90,18 @@ class CPUModelRunner(GPUModelRunner):
vllm.v1.spec_decode.llm_base_proposer.copy_and_expand_eagle_inputs_kernel = (
cpu_tl.copy_and_expand_eagle_inputs_kernel
)
vllm.v1.spec_decode.utils.eagle_step_slot_mapping_metadata_kernel = (
spec_decode_utils.copy_and_expand_dflash_inputs_kernel = (
cpu_tl.copy_and_expand_dflash_inputs_kernel
)
dflash_module = sys.modules.get("vllm.v1.spec_decode.dflash")
if dflash_module is not None:
dflash_kernel_name = "copy_and_expand_dflash_inputs_kernel"
setattr(
dflash_module,
dflash_kernel_name,
cpu_tl.copy_and_expand_dflash_inputs_kernel,
)
spec_decode_utils.eagle_step_slot_mapping_metadata_kernel = (
cpu_tl.eagle_step_slot_mapping_metadata_kernel
)
vllm.v1.sample.rejection_sampler.rejection_greedy_sample_kernel = (