[Spec decode] Support Kimi-K3 DCP with DSpark (#52188)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
Wei Zhao
2026-08-17 13:08:45 -07:00
committed by GitHub
parent 9633933dd8
commit d1e3eee6fb
15 changed files with 341 additions and 51 deletions
@@ -140,26 +140,3 @@ def test_dspark_mla_speculative_config_preserves_architecture(tmp_path):
assert speculative_config.draft_model_config.architectures == ["K3DSparkModel"]
assert speculative_config.draft_model_config.hf_config.model_type == "k3_dspark"
assert speculative_config.draft_model_config.use_mla
def test_dspark_mla_rejects_decode_context_parallelism(tmp_path):
target_path = tmp_path / "target"
draft_path = tmp_path / "draft"
_write_target_config(target_path)
_write_dspark_config(draft_path)
target_config = ModelConfig(
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
)
with pytest.raises(ValueError, match="does not currently support decode context"):
SpeculativeConfig(
model=str(draft_path),
method="dspark",
num_speculative_tokens=8,
target_model_config=target_config,
target_parallel_config=ParallelConfig(
tensor_parallel_size=2,
decode_context_parallel_size=2,
distributed_executor_backend="external_launcher",
),
)
@@ -45,6 +45,7 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
impl.bmm1_scale = 1.0
impl.bmm2_scale = 1.0
impl.need_to_return_lse_for_decode = True
impl.dcp_world_size = 2
impl.num_heads = 6
impl.qk_nope_head_dim = 128
impl.kv_lora_rank = 512
+77
View File
@@ -837,9 +837,86 @@ def test_mock_mla_dcp_fp8_decode_gathers_quantized_query(
def test_tokenspeed_mla_noncausal_capability():
builder = tokenspeed_mla_module.TokenspeedMLAMetadataBuilder
assert builder.supports_non_causal_multi_token_decode
assert builder.supports_non_causal_multi_token_dcp
assert tokenspeed_mla_module.TokenspeedMLABackend.supports_non_causal()
def test_flashinfer_mla_dcp_multi_token_decode_uses_per_query_bounds(monkeypatch):
flashinfer_mla_module = pytest.importorskip(
"vllm.v1.attention.backends.mla.flashinfer_mla"
)
decode_call = None
def fake_decode(**kwargs):
nonlocal decode_call
decode_call = kwargs
query = kwargs["query"]
output = torch.empty(*query.shape[:-1], 512, dtype=torch.bfloat16)
lse = torch.empty(query.shape[0], query.shape[-2], dtype=torch.float32)
return output, lse
monkeypatch.setattr(
flashinfer_mla_module,
"trtllm_batch_decode_with_kv_cache_mla",
fake_decode,
)
monkeypatch.setattr(
flashinfer_mla_module,
"_get_workspace_buffer",
lambda return_lse: torch.empty(1, dtype=torch.int8),
)
impl = object.__new__(flashinfer_mla_module.FlashInferMLAImpl)
impl.dcp_world_size = 2
impl.dcp_rank = 1
impl.cp_kv_cache_interleave_size = 1
impl.need_to_return_lse_for_decode = True
impl.kv_lora_rank = 512
impl.qk_nope_head_dim = 128
impl.qk_rope_head_dim = 64
impl.bmm1_scale = 1.0
impl.bmm2_scale = 1.0
block_table = torch.tensor([[1], [2]], dtype=torch.int32)
metadata = SimpleNamespace(
num_decodes=2,
num_decode_tokens=6,
max_seq_len=7,
causal=True,
decode=SimpleNamespace(
block_table=block_table,
seq_lens=torch.tensor([5, 6], dtype=torch.int32),
dcp_tot_seq_lens=torch.tensor([10, 13], dtype=torch.int32),
flattened_block_table=None,
flattened_seq_lens=None,
query_len=0,
),
)
query = torch.empty(6, 2, 576, dtype=torch.bfloat16)
kv_cache = torch.empty(3, 16, 576, dtype=torch.bfloat16)
output, lse = impl.forward_mqa(
query,
kv_cache,
metadata,
SimpleNamespace(),
)
assert output.shape == (6, 2, 512)
assert lse is not None
assert lse.shape == (6, 2)
assert decode_call is not None
assert decode_call["query"].shape == (6, 1, 2, 576)
torch.testing.assert_close(
decode_call["seq_lens"],
torch.tensor([4, 4, 5, 5, 6, 6], dtype=torch.int32),
)
torch.testing.assert_close(
decode_call["block_tables"], block_table.repeat_interleave(3, dim=0)
)
@pytest.mark.parametrize(
("causal", "tokens_per_decode", "dcp_world_size", "dcp_rank"),
[
@@ -17,7 +17,14 @@ pytestmark = pytest.mark.skipif(
)
def _run_prepare(*, target_positions: list[int], block_table_values: list[int]):
def _run_prepare(
*,
target_positions: list[int],
block_table_values: list[int],
cp_rank: int = 0,
cp_size: int = 1,
cp_interleave: int = 1,
):
device = torch.device("cuda")
max_num_reqs = 4
max_num_tokens = 16
@@ -86,6 +93,9 @@ def _run_prepare(*, target_positions: list[int], block_table_values: list[int]):
input_seeds,
block_table,
4,
cp_rank,
cp_size,
cp_interleave,
123,
num_speculative_steps,
num_speculative_steps,
@@ -131,6 +141,20 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
assert out.seeds[2].item() == 17
def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp():
out = _run_prepare(
target_positions=[10, 11, 12, 13],
block_table_values=[0, 7, 8, 9],
cp_rank=1,
cp_size=2,
cp_interleave=2,
)
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
assert out.context_slot_mapping[:4].tolist() == [28, 29, PAD_SLOT_ID, PAD_SLOT_ID]
assert out.query_slot_mapping[:3].tolist() == [PAD_SLOT_ID, PAD_SLOT_ID, 30]
def test_prepare_dflash_inputs_never_writes_the_null_block():
# The valid context uses logical block 0 and the replacement query uses
# logical block 1. Both map to the null block and must remain unwritable.
-10
View File
@@ -1059,16 +1059,6 @@ class SpeculativeConfig:
if self.method in ("dflash", "dspark"):
self.parallel_drafting = True
if (
self.method == "dspark"
and "K3DSparkModel" in self.draft_model_config.architectures
and self.target_parallel_config.decode_context_parallel_size > 1
):
raise ValueError(
"MLA DSpark does not currently support decode context "
"parallelism; set decode_context_parallel_size=1."
)
if self.num_speculative_tokens is not None and hasattr(
self.draft_model_config.hf_config, "num_lookahead_tokens"
):
@@ -1984,6 +1984,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
# Whether this builder can flatten a non-causal query block into decode rows.
supports_non_causal_multi_token_decode: ClassVar[bool] = False
# Whether can support non-causal multi-token decode with DCP KV cache.
supports_non_causal_multi_token_dcp: ClassVar[bool] = False
# The threshold for reordering the batch into decode and prefill requests.
# If > 1, the batch will be reordered such that requests with
# query length <= threshold are classified as decode requests.
@@ -1991,6 +1994,31 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
# when speculative decoding is enabled.
reorder_batch_threshold: int = 1
def _validate_dspark_dcp_support(self, supports_dcp_with_varlen: bool) -> None:
speculative_config = getattr(self.vllm_config, "speculative_config", None)
parallel_config = self.vllm_config.parallel_config
if (
speculative_config is None
or getattr(speculative_config, "method", None) != "dspark"
or parallel_config.decode_context_parallel_size <= 1
):
return
if self.non_causal_multi_token_decode:
supported = self.supports_non_causal_multi_token_dcp
query_mode = "non-causal draft"
else:
supported = supports_dcp_with_varlen
query_mode = "causal multi-token"
if not supported:
raise ValueError(
f"{type(self).__name__} does not support {query_mode} MLA "
"attention for DSpark with decode context parallelism. Select "
"a backend with explicit DSpark DCP support or set "
"decode_context_parallel_size=1."
)
@staticmethod
def determine_chunked_prefill_workspace_size(vllm_config: VllmConfig) -> int:
scheduler_config = vllm_config.scheduler_config
@@ -2084,6 +2112,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.non_causal_multi_token_decode = getattr(
kv_cache_spec, "non_causal_multi_token_decode", False
)
self._validate_dspark_dcp_support(supports_dcp_with_varlen)
# A draft cache group can have a different head count from the target.
self.num_heads = get_num_attention_heads_from_layers(
-5
View File
@@ -341,11 +341,6 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
"parallelism."
)
self.dcp_world_size = parallel_config.decode_context_parallel_size
assert self.dcp_world_size <= 1 or self.rotary_emb is None, (
"Kimi-K3 MultiHeadLatentAttention does not support RoPE with decode "
"context parallelism because gathered queries require gathered "
"positions."
)
self.dcp_manager: MLADCPManager | None = None
if self.dcp_world_size > 1:
query_dtype = (
+8
View File
@@ -330,6 +330,11 @@ class AttentionBackend(ABC):
except NotImplementedError:
return False
@classmethod
def supports_non_causal_dcp(cls) -> bool:
builder_cls = cls.get_builder_cls()
return bool(getattr(builder_cls, "supports_non_causal_multi_token_dcp", False))
@classmethod
def supports_attn_type(cls, attn_type: str) -> bool:
"""Check if backend supports a given attention type.
@@ -378,6 +383,7 @@ class AttentionBackend(ABC):
use_kv_connector: bool = False,
use_pcp: bool = False,
use_adaptive_verification: bool = False,
use_dcp: bool = False,
) -> list[str]:
invalid_reasons = []
if not cls.supports_head_size(head_size):
@@ -414,6 +420,8 @@ class AttentionBackend(ABC):
invalid_reasons.append("sliding window not supported")
if use_non_causal and not cls.supports_non_causal():
invalid_reasons.append("non-causal attention not supported")
if use_mla and use_non_causal and use_dcp and not cls.supports_non_causal_dcp():
invalid_reasons.append("non-causal MLA attention with DCP not supported")
if use_batch_invariant and not cls.supports_batch_invariance():
invalid_reasons.append("batch invariance not supported")
if use_kv_connector and not cls.supports_kv_connector():
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import ClassVar
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
import torch
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
@@ -15,6 +16,7 @@ from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBackend,
MLACommonDecodeMetadata,
MLACommonImpl,
MLACommonMetadata,
MLACommonMetadataBuilder,
@@ -30,6 +32,10 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.utils import KVCacheLayoutType
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import AttentionSpec
logger = init_logger(__name__)
@@ -112,12 +118,56 @@ def _get_multi_ctas_kv_counter_buffer(
return _fi_multi_ctas_kv_counter
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
@dataclass
class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata):
flattened_block_table: torch.Tensor | None = None
flattened_seq_lens: torch.Tensor | None = None
query_len: int = 0
@dataclass
class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]):
pass
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
# Non-causal DSpark blocks are flattened to single-token rows in forward_mqa.
supports_non_causal_multi_token_decode: ClassVar[bool] = True
def __init__(
self,
kv_cache_spec: "AttentionSpec",
layer_names: list[str],
vllm_config: "VllmConfig",
device: torch.device,
) -> None:
super().__init__(
kv_cache_spec,
layer_names,
vllm_config,
device,
FlashInferMLAMetadata,
supports_dcp_with_varlen=True,
)
def _build_decode(
self,
block_table_tensor: torch.Tensor,
seq_lens_device: torch.Tensor,
max_seq_len: int,
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> FlashInferMLADecodeMetadata:
return FlashInferMLADecodeMetadata(
block_table=block_table_tensor,
seq_lens=seq_lens_device,
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
)
class FlashInferMLABackend(MLACommonBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
@@ -193,7 +243,7 @@ class FlashInferMLABackend(MLACommonBackend):
return "HND"
class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
can_return_lse_for_decode: bool = True
# trtllm-gen MLA decode emits LSE in log2 (per flashinfer's own
# reference at flashinfer/trace/templates/attention.py:81:
@@ -264,7 +314,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: MLACommonMetadata,
attn_metadata: FlashInferMLAMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert kv_c_and_k_pe_cache.numel() > 0
@@ -276,16 +326,28 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
block_table = attn_metadata.decode.block_table
seq_lens = attn_metadata.decode.seq_lens
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
if not attn_metadata.causal:
# Non-causal DSpark block: flatten to single-token decode rows with
# per-row context seq_lens (trtllm-gen has no causal flag and would
# otherwise mask the block causally).
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
q = q.unsqueeze(1)
if query_len > 1:
block_table = block_table.repeat_interleave(query_len, dim=0)
seq_lens = seq_lens.repeat_interleave(query_len)
block_table, seq_lens = self._prepare_flattened_decode_metadata(
attn_metadata,
query_len,
causal=False,
)
elif self.dcp_world_size > 1 and query_len > 1:
# Causal DCP block: flatten to single-token decode rows with
# per-row rank-local seq_lens for each query's visible prefix.
block_table, seq_lens = self._prepare_flattened_decode_metadata(
attn_metadata,
query_len,
causal=True,
)
q = q.unsqueeze(1)
# trtllm API requires extra dimension q_len_per_request for MTP
elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0:
logger.warning_once(
@@ -358,3 +420,49 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
o = o.view(-1, o.shape[-2], o.shape[-1])
return o, lse
def _prepare_flattened_decode_metadata(
self,
attn_metadata: FlashInferMLAMetadata,
query_len: int,
*,
causal: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Prepare flattened decode metadata once for all layers in the group."""
decode = attn_metadata.decode
assert decode is not None
if decode.query_len:
assert decode.query_len == query_len
assert decode.flattened_block_table is not None
assert decode.flattened_seq_lens is not None
return decode.flattened_block_table, decode.flattened_seq_lens
block_table = decode.block_table.repeat_interleave(query_len, dim=0)
if causal:
global_seq_lens = decode.dcp_tot_seq_lens
assert global_seq_lens is not None
offsets = torch.arange(
query_len - 1,
-1,
-1,
device=global_seq_lens.device,
dtype=global_seq_lens.dtype,
)
per_query_global_lens = torch.clamp(
(global_seq_lens.unsqueeze(1) - offsets).reshape(-1), min=0
)
interleave = self.cp_kv_cache_interleave_size
dcp_span = self.dcp_world_size * interleave
remainder = torch.clamp(
per_query_global_lens % dcp_span - self.dcp_rank * interleave,
min=0,
max=interleave,
)
seq_lens = per_query_global_lens // dcp_span * interleave + remainder
else:
seq_lens = decode.seq_lens.repeat_interleave(query_len)
decode.flattened_block_table = block_table
decode.flattened_seq_lens = seq_lens
decode.query_len = query_len
return block_table, seq_lens
@@ -61,6 +61,7 @@ class TokenspeedMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
# The kernel accepts an explicit causal mask, so a non-causal DSpark
# block can remain fused instead of being flattened to single tokens.
supports_non_causal_multi_token_decode: ClassVar[bool] = True
supports_non_causal_multi_token_dcp: ClassVar[bool] = True
def __init__(
self,
+4 -1
View File
@@ -38,6 +38,7 @@ class AttentionSelectorConfig(NamedTuple):
use_kv_connector: bool = False
use_pcp: bool = False
use_adaptive_verification: bool = False
use_dcp: bool = False
def __repr__(self):
return (
@@ -56,7 +57,8 @@ class AttentionSelectorConfig(NamedTuple):
f"use_batch_invariant={self.use_batch_invariant}, "
f"use_kv_connector={self.use_kv_connector}, "
f"use_adaptive_verification={self.use_adaptive_verification}, "
f"use_pcp={self.use_pcp})"
f"use_pcp={self.use_pcp}, "
f"use_dcp={self.use_dcp})"
)
@@ -168,6 +170,7 @@ def get_attn_backend(
use_kv_connector=use_kv_connector,
use_pcp=vllm_config.parallel_config.prefill_context_parallel_size > 1,
use_adaptive_verification=use_adaptive_verification,
use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1,
)
# A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over
+21
View File
@@ -59,3 +59,24 @@ def _dcp_local_seq_lens_kernel(
# For [num_reqs, max_num_reqs), pad with 0
local_seq_lens = tl.where(block < num_reqs, local_seq_lens, 0)
tl.store(out_ptr + block, local_seq_lens, mask=block < max_num_reqs)
@triton.jit
def cp_local_slot(
positions,
block_numbers,
block_size,
cp_rank,
CP_SIZE: tl.constexpr,
CP_INTERLEAVE: tl.constexpr,
PAD_ID: tl.constexpr,
):
"""Return rank-local KV slots, or PAD_ID for positions not owned by this rank."""
block_offsets = positions % (block_size * CP_SIZE)
if CP_SIZE == 1:
return block_numbers * block_size + block_offsets
is_local = block_offsets // CP_INTERLEAVE % CP_SIZE == cp_rank
rounds = block_offsets // (CP_INTERLEAVE * CP_SIZE)
remainder = block_offsets % CP_INTERLEAVE
local_offsets = rounds * CP_INTERLEAVE + remainder
return tl.where(is_local, block_numbers * block_size + local_offsets, PAD_ID)
@@ -11,6 +11,7 @@ from vllm.v1.worker.gpu.attn_utils import (
build_slot_mappings_by_layer,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionState,
BatchExecutionDescriptor,
@@ -41,6 +42,17 @@ def _prepare_dflash_inputs_to_capture(
attn_metadata = None
if not skip_attn:
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
dcp_local_seq_lens = None
if block_tables.cp_size > 1:
prepare_dcp_local_seq_lens(
input_buffers.dcp_local_seq_lens,
input_buffers.seq_lens,
num_reqs,
block_tables.cp_size,
block_tables.cp_rank,
block_tables.cp_interleave,
)
dcp_local_seq_lens = input_buffers.dcp_local_seq_lens
attn_metadata = build_attn_metadata(
attn_groups=attn_groups,
num_reqs=num_reqs,
@@ -49,6 +61,7 @@ def _prepare_dflash_inputs_to_capture(
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=num_tokens // num_reqs,
seq_lens=input_batch.seq_lens,
dcp_local_seq_lens=dcp_local_seq_lens,
max_seq_len=max_model_len,
block_tables=input_block_tables,
slot_mappings=slot_mappings,
@@ -17,6 +17,7 @@ from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import cp_local_slot, prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
@@ -285,10 +286,21 @@ class DFlashSpeculator(DraftModelSpeculator):
num_query_per_req: int | None = None,
causal: bool | Mapping[int, bool] = False,
query_start_loc_np: np.ndarray | None = None,
dcp_local_seq_lens: torch.Tensor | None = None,
) -> dict[str, Any] | None:
if not self.draft_attn_layer_names:
return None
assert num_query_per_req is None # Omitted for DFlash, read from self instead
if dcp_local_seq_lens is None and self.block_tables.cp_size > 1:
prepare_dcp_local_seq_lens(
self.input_buffers.dcp_local_seq_lens,
self.input_buffers.seq_lens,
num_reqs,
self.block_tables.cp_size,
self.block_tables.cp_rank,
self.block_tables.cp_interleave,
)
dcp_local_seq_lens = self.input_buffers.dcp_local_seq_lens
return super()._build_draft_attn_metadata(
num_reqs,
num_reqs_padded,
@@ -298,6 +310,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_query_per_req=self.num_query_per_req,
causal=causal,
query_start_loc_np=query_start_loc_np,
dcp_local_seq_lens=dcp_local_seq_lens,
)
@torch.inference_mode()
@@ -393,6 +406,9 @@ class DFlashSpeculator(DraftModelSpeculator):
seeds,
self.block_tables.input_block_tables[gid],
self.block_tables.kernel_block_sizes[gid],
self.block_tables.cp_rank,
self.block_tables.cp_size,
self.block_tables.cp_interleave,
self.parallel_drafting_token_id,
self.num_query_per_req,
self.num_speculative_steps,
@@ -507,8 +523,11 @@ def _prepare_dflash_inputs_kernel(
max_num_reqs,
max_num_tokens,
max_model_len,
cp_rank,
SAMPLE_FROM_ANCHOR: tl.constexpr,
PAD_SLOT_ID: tl.constexpr,
CP_SIZE: tl.constexpr,
CP_INTERLEAVE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
@@ -543,7 +562,7 @@ def _prepare_dflash_inputs_kernel(
# --- Context positions / slots ---
ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0)
ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_valid_ctx, other=0)
ctx_block_num = ctx_pos // block_size
ctx_block_num = ctx_pos // (block_size * CP_SIZE)
ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1)
ctx_block_id = tl.load(
block_table_ptr + req_idx * block_table_stride + ctx_block_num,
@@ -554,9 +573,12 @@ def _prepare_dflash_inputs_kernel(
# to it after eviction; rejected suffix rows are invalid context as well.
# Neither kind of row may write draft KV into physical block 0.
ctx_resident = is_valid_ctx & (ctx_block_id != 0)
local_ctx_slot = cp_local_slot(
ctx_pos, ctx_block_id, block_size, cp_rank, CP_SIZE, CP_INTERLEAVE, PAD_SLOT_ID
)
ctx_slot = tl.where(
ctx_resident,
ctx_block_id * block_size + (ctx_pos % block_size),
local_ctx_slot,
PAD_SLOT_ID,
)
# Stored over the full [0, num_ctx) span while the loads above are masked to
@@ -573,7 +595,7 @@ def _prepare_dflash_inputs_kernel(
is_bonus = is_query & (query_off == 0)
input_id = tl.where(is_bonus, bonus_token, parallel_drafting_token_id)
q_block_num = query_pos // block_size
q_block_num = query_pos // (block_size * CP_SIZE)
q_block_num = tl.minimum(q_block_num, block_table_stride - 1)
q_block_id = tl.load(
block_table_ptr + req_idx * block_table_stride + q_block_num,
@@ -583,9 +605,18 @@ def _prepare_dflash_inputs_kernel(
# A null block is never a writable cache slot. This can occur when a
# sliding-window block table contains evicted/global padding entries.
q_resident = is_query & (q_block_id != 0)
local_q_slot = cp_local_slot(
query_pos,
q_block_id,
block_size,
cp_rank,
CP_SIZE,
CP_INTERLEAVE,
PAD_SLOT_ID,
)
q_slot = tl.where(
q_resident,
q_block_id * block_size + (query_pos % block_size),
local_q_slot,
PAD_SLOT_ID,
)
@@ -681,6 +712,9 @@ def prepare_dflash_inputs(
# [max_num_reqs, max_num_blocks]
block_table: torch.Tensor,
block_size: int,
cp_rank: int,
cp_size: int,
cp_interleave: int,
parallel_drafting_token_id: int,
num_query_per_req: int,
num_speculative_steps: int,
@@ -728,7 +762,10 @@ def prepare_dflash_inputs(
max_num_reqs,
max_num_tokens,
max_model_len,
cp_rank,
SAMPLE_FROM_ANCHOR=sample_from_anchor,
PAD_SLOT_ID=PAD_SLOT_ID,
CP_SIZE=cp_size,
CP_INTERLEAVE=cp_interleave,
BLOCK_SIZE=BLOCK_SIZE,
)
@@ -234,6 +234,7 @@ class DraftModelSpeculator(BaseSpeculator):
num_query_per_req: int = 1,
causal: bool | Mapping[int, bool] = True,
query_start_loc_np: np.ndarray | None = None,
dcp_local_seq_lens: torch.Tensor | None = None,
) -> dict[str, Any] | None:
if query_start_loc_np is not None:
# Non-uniform query layout (e.g. multi-module MTP's mixed
@@ -278,6 +279,11 @@ class DraftModelSpeculator(BaseSpeculator):
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=max_query_len,
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
dcp_local_seq_lens=(
None
if dcp_local_seq_lens is None
else dcp_local_seq_lens[:num_reqs_padded]
),
max_seq_len=self.draft_max_seq_len,
block_tables=block_tables,
slot_mappings=slot_mappings,