[CPU][Spec Decode] Enable DFlash SD for CPU (#44029)

Signed-off-by: guybd <guy.boudoukh@intel.com>
Signed-off-by: Guy Boudoukh <guy.boudoukh@intel.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
guybd
2026-06-25 15:32:48 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9b215ae60b
commit 2396d91e93
8 changed files with 266 additions and 8 deletions
+134
View File
@@ -197,6 +197,133 @@ def _copy_and_expand_eagle_inputs_kernel_impl(
out_positions_ptr.copy_(out_pos_i64.to(orig_pos_dtype))
def _copy_and_expand_dflash_inputs_kernel_impl(
next_token_ids_ptr,
target_positions_ptr,
out_input_ids_ptr,
out_context_positions_ptr,
out_query_positions_ptr,
out_context_slot_mapping_ptr,
out_query_slot_mapping_ptr,
out_token_indices_ptr,
block_table_ptr,
block_table_stride,
query_start_loc_ptr,
num_rejected_tokens_ptr,
parallel_drafting_token_id,
block_size,
num_query_per_req,
num_speculative_tokens,
total_input_tokens,
BLOCK_SIZE=None,
HAS_NUM_REJECTED=False,
):
"""Adapter between the DFlash Triton launch and the C++ CPU op."""
assert block_table_stride == block_table_ptr.stride(0), (
"block_table_stride mismatch: "
f"{block_table_stride} vs {block_table_ptr.stride(0)}"
)
orig_ids_dtype = out_input_ids_ptr.dtype
orig_context_positions_dtype = out_context_positions_ptr.dtype
orig_query_positions_dtype = out_query_positions_ptr.dtype
orig_context_slot_mapping_dtype = out_context_slot_mapping_ptr.dtype
orig_query_slot_mapping_dtype = out_query_slot_mapping_ptr.dtype
out_ids_i64 = _ensure_int64(out_input_ids_ptr)
out_context_positions_i64 = _ensure_int64(out_context_positions_ptr)
out_query_positions_i64 = _ensure_int64(out_query_positions_ptr)
out_context_slot_mapping_i64 = _ensure_int64(out_context_slot_mapping_ptr)
out_query_slot_mapping_i64 = _ensure_int64(out_query_slot_mapping_ptr)
rejected_i64 = _ensure_int64(num_rejected_tokens_ptr) if HAS_NUM_REJECTED else None
if hasattr(torch.ops._C, "copy_and_expand_dflash_inputs_kernel_impl"):
torch.ops._C.copy_and_expand_dflash_inputs_kernel_impl(
_ensure_int64(next_token_ids_ptr),
_ensure_int64(target_positions_ptr),
out_ids_i64,
out_context_positions_i64,
out_query_positions_i64,
out_context_slot_mapping_i64,
out_query_slot_mapping_i64,
out_token_indices_ptr,
block_table_ptr,
query_start_loc_ptr,
rejected_i64,
parallel_drafting_token_id,
block_size,
num_query_per_req,
num_speculative_tokens,
total_input_tokens,
HAS_NUM_REJECTED,
)
else:
next_ids_i64 = _ensure_int64(next_token_ids_ptr)
target_positions_i64 = _ensure_int64(target_positions_ptr)
block_table_stride = block_table_ptr.stride(0)
num_reqs = query_start_loc_ptr.shape[0] - 1
for req_idx in range(num_reqs):
ctx_start = int(query_start_loc_ptr[req_idx].item())
ctx_end = int(query_start_loc_ptr[req_idx + 1].item())
num_ctx = ctx_end - ctx_start
valid_ctx_end = ctx_end
if rejected_i64 is not None:
valid_ctx_end -= int(rejected_i64[req_idx].item())
# Guard against out-of-bounds: ensure valid_ctx_end > ctx_start.
valid_ctx_end = max(valid_ctx_end, ctx_start + 1)
last_pos = int(target_positions_i64[valid_ctx_end - 1].item())
for j in range(num_ctx):
ctx_idx = ctx_start + j
ctx_pos_idx = min(ctx_idx, total_input_tokens - 1)
position = int(target_positions_i64[ctx_pos_idx].item())
block_num = min(position // block_size, block_table_stride - 1)
block_id = int(block_table_ptr[req_idx, block_num].item())
slot = block_id * block_size + (position % block_size)
out_context_positions_i64[ctx_idx] = position
out_context_slot_mapping_i64[ctx_idx] = slot
for query_off in range(num_query_per_req):
query_out = req_idx * num_query_per_req + query_off
position = last_pos + 1 + query_off
block_num = min(position // block_size, block_table_stride - 1)
block_id = int(block_table_ptr[req_idx, block_num].item())
slot = block_id * block_size + (position % block_size)
out_query_positions_i64[query_out] = position
out_query_slot_mapping_i64[query_out] = slot
out_ids_i64[query_out] = (
int(next_ids_i64[req_idx].item())
if query_off == 0
else parallel_drafting_token_id
)
if query_off > 0:
sample_out_idx = req_idx * num_speculative_tokens + (query_off - 1)
out_token_indices_ptr[sample_out_idx] = query_out
if orig_ids_dtype != torch.int64:
out_input_ids_ptr.copy_(out_ids_i64.to(orig_ids_dtype))
if orig_context_positions_dtype != torch.int64:
out_context_positions_ptr.copy_(
out_context_positions_i64.to(orig_context_positions_dtype)
)
if orig_query_positions_dtype != torch.int64:
out_query_positions_ptr.copy_(
out_query_positions_i64.to(orig_query_positions_dtype)
)
if orig_context_slot_mapping_dtype != torch.int64:
out_context_slot_mapping_ptr.copy_(
out_context_slot_mapping_i64.to(orig_context_slot_mapping_dtype)
)
if orig_query_slot_mapping_dtype != torch.int64:
out_query_slot_mapping_ptr.copy_(
out_query_slot_mapping_i64.to(orig_query_slot_mapping_dtype)
)
def _rejection_greedy_sample_kernel_impl(
output_token_ids,
cu_num_draft_tokens,
@@ -303,6 +430,10 @@ def _sample_recovered_tokens_kernel_impl(
NO_DRAFT_PROBS=False,
USE_FP64_GUMBEL=False,
):
# USE_FP64_GUMBEL only controls the gumbel-noise precision, which the caller
# has already applied to `inv_q` (fp64 vs fp32). The CPU kernel consumes
# `inv_q` directly, so the flag is accepted for interface parity and the
# value is read at its existing dtype.
# C++ reads integer tensors as int64_t*; ensure correct dtype.
orig_dtype = output_token_ids.dtype
output_i64 = _ensure_int64(output_token_ids)
@@ -330,6 +461,9 @@ eagle_prepare_next_token_padded_kernel = _FuncWrapper(
copy_and_expand_eagle_inputs_kernel = _FuncWrapper(
_copy_and_expand_eagle_inputs_kernel_impl
)
copy_and_expand_dflash_inputs_kernel = _FuncWrapper(
_copy_and_expand_dflash_inputs_kernel_impl
)
eagle_step_slot_mapping_metadata_kernel = _FuncWrapper(
_eagle_step_slot_mapping_metadata_kernel_impl
)