[Spec decode] Support Kimi-K3 DCP with DSpark (#52188)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
@@ -140,26 +140,3 @@ def test_dspark_mla_speculative_config_preserves_architecture(tmp_path):
|
||||
assert speculative_config.draft_model_config.architectures == ["K3DSparkModel"]
|
||||
assert speculative_config.draft_model_config.hf_config.model_type == "k3_dspark"
|
||||
assert speculative_config.draft_model_config.use_mla
|
||||
|
||||
|
||||
def test_dspark_mla_rejects_decode_context_parallelism(tmp_path):
|
||||
target_path = tmp_path / "target"
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_target_config(target_path)
|
||||
_write_dspark_config(draft_path)
|
||||
target_config = ModelConfig(
|
||||
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not currently support decode context"):
|
||||
SpeculativeConfig(
|
||||
model=str(draft_path),
|
||||
method="dspark",
|
||||
num_speculative_tokens=8,
|
||||
target_model_config=target_config,
|
||||
target_parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=2,
|
||||
distributed_executor_backend="external_launcher",
|
||||
),
|
||||
)
|
||||
|
||||
@@ -45,6 +45,7 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
|
||||
impl.bmm1_scale = 1.0
|
||||
impl.bmm2_scale = 1.0
|
||||
impl.need_to_return_lse_for_decode = True
|
||||
impl.dcp_world_size = 2
|
||||
impl.num_heads = 6
|
||||
impl.qk_nope_head_dim = 128
|
||||
impl.kv_lora_rank = 512
|
||||
|
||||
@@ -837,9 +837,86 @@ def test_mock_mla_dcp_fp8_decode_gathers_quantized_query(
|
||||
def test_tokenspeed_mla_noncausal_capability():
|
||||
builder = tokenspeed_mla_module.TokenspeedMLAMetadataBuilder
|
||||
assert builder.supports_non_causal_multi_token_decode
|
||||
assert builder.supports_non_causal_multi_token_dcp
|
||||
assert tokenspeed_mla_module.TokenspeedMLABackend.supports_non_causal()
|
||||
|
||||
|
||||
def test_flashinfer_mla_dcp_multi_token_decode_uses_per_query_bounds(monkeypatch):
|
||||
flashinfer_mla_module = pytest.importorskip(
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla"
|
||||
)
|
||||
|
||||
decode_call = None
|
||||
|
||||
def fake_decode(**kwargs):
|
||||
nonlocal decode_call
|
||||
decode_call = kwargs
|
||||
query = kwargs["query"]
|
||||
output = torch.empty(*query.shape[:-1], 512, dtype=torch.bfloat16)
|
||||
lse = torch.empty(query.shape[0], query.shape[-2], dtype=torch.float32)
|
||||
return output, lse
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mla_module,
|
||||
"trtllm_batch_decode_with_kv_cache_mla",
|
||||
fake_decode,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mla_module,
|
||||
"_get_workspace_buffer",
|
||||
lambda return_lse: torch.empty(1, dtype=torch.int8),
|
||||
)
|
||||
|
||||
impl = object.__new__(flashinfer_mla_module.FlashInferMLAImpl)
|
||||
impl.dcp_world_size = 2
|
||||
impl.dcp_rank = 1
|
||||
impl.cp_kv_cache_interleave_size = 1
|
||||
impl.need_to_return_lse_for_decode = True
|
||||
impl.kv_lora_rank = 512
|
||||
impl.qk_nope_head_dim = 128
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.bmm1_scale = 1.0
|
||||
impl.bmm2_scale = 1.0
|
||||
|
||||
block_table = torch.tensor([[1], [2]], dtype=torch.int32)
|
||||
metadata = SimpleNamespace(
|
||||
num_decodes=2,
|
||||
num_decode_tokens=6,
|
||||
max_seq_len=7,
|
||||
causal=True,
|
||||
decode=SimpleNamespace(
|
||||
block_table=block_table,
|
||||
seq_lens=torch.tensor([5, 6], dtype=torch.int32),
|
||||
dcp_tot_seq_lens=torch.tensor([10, 13], dtype=torch.int32),
|
||||
flattened_block_table=None,
|
||||
flattened_seq_lens=None,
|
||||
query_len=0,
|
||||
),
|
||||
)
|
||||
query = torch.empty(6, 2, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.empty(3, 16, 576, dtype=torch.bfloat16)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
kv_cache,
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert output.shape == (6, 2, 512)
|
||||
assert lse is not None
|
||||
assert lse.shape == (6, 2)
|
||||
assert decode_call is not None
|
||||
assert decode_call["query"].shape == (6, 1, 2, 576)
|
||||
torch.testing.assert_close(
|
||||
decode_call["seq_lens"],
|
||||
torch.tensor([4, 4, 5, 5, 6, 6], dtype=torch.int32),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
decode_call["block_tables"], block_table.repeat_interleave(3, dim=0)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("causal", "tokens_per_decode", "dcp_world_size", "dcp_rank"),
|
||||
[
|
||||
|
||||
@@ -17,7 +17,14 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
def _run_prepare(*, target_positions: list[int], block_table_values: list[int]):
|
||||
def _run_prepare(
|
||||
*,
|
||||
target_positions: list[int],
|
||||
block_table_values: list[int],
|
||||
cp_rank: int = 0,
|
||||
cp_size: int = 1,
|
||||
cp_interleave: int = 1,
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
max_num_reqs = 4
|
||||
max_num_tokens = 16
|
||||
@@ -86,6 +93,9 @@ def _run_prepare(*, target_positions: list[int], block_table_values: list[int]):
|
||||
input_seeds,
|
||||
block_table,
|
||||
4,
|
||||
cp_rank,
|
||||
cp_size,
|
||||
cp_interleave,
|
||||
123,
|
||||
num_speculative_steps,
|
||||
num_speculative_steps,
|
||||
@@ -131,6 +141,20 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
|
||||
assert out.seeds[2].item() == 17
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp():
|
||||
out = _run_prepare(
|
||||
target_positions=[10, 11, 12, 13],
|
||||
block_table_values=[0, 7, 8, 9],
|
||||
cp_rank=1,
|
||||
cp_size=2,
|
||||
cp_interleave=2,
|
||||
)
|
||||
|
||||
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
|
||||
assert out.context_slot_mapping[:4].tolist() == [28, 29, PAD_SLOT_ID, PAD_SLOT_ID]
|
||||
assert out.query_slot_mapping[:3].tolist() == [PAD_SLOT_ID, PAD_SLOT_ID, 30]
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_never_writes_the_null_block():
|
||||
# The valid context uses logical block 0 and the replacement query uses
|
||||
# logical block 1. Both map to the null block and must remain unwritable.
|
||||
|
||||
@@ -1059,16 +1059,6 @@ class SpeculativeConfig:
|
||||
if self.method in ("dflash", "dspark"):
|
||||
self.parallel_drafting = True
|
||||
|
||||
if (
|
||||
self.method == "dspark"
|
||||
and "K3DSparkModel" in self.draft_model_config.architectures
|
||||
and self.target_parallel_config.decode_context_parallel_size > 1
|
||||
):
|
||||
raise ValueError(
|
||||
"MLA DSpark does not currently support decode context "
|
||||
"parallelism; set decode_context_parallel_size=1."
|
||||
)
|
||||
|
||||
if self.num_speculative_tokens is not None and hasattr(
|
||||
self.draft_model_config.hf_config, "num_lookahead_tokens"
|
||||
):
|
||||
|
||||
@@ -1984,6 +1984,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
# Whether this builder can flatten a non-causal query block into decode rows.
|
||||
supports_non_causal_multi_token_decode: ClassVar[bool] = False
|
||||
|
||||
# Whether can support non-causal multi-token decode with DCP KV cache.
|
||||
supports_non_causal_multi_token_dcp: ClassVar[bool] = False
|
||||
|
||||
# The threshold for reordering the batch into decode and prefill requests.
|
||||
# If > 1, the batch will be reordered such that requests with
|
||||
# query length <= threshold are classified as decode requests.
|
||||
@@ -1991,6 +1994,31 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
# when speculative decoding is enabled.
|
||||
reorder_batch_threshold: int = 1
|
||||
|
||||
def _validate_dspark_dcp_support(self, supports_dcp_with_varlen: bool) -> None:
|
||||
speculative_config = getattr(self.vllm_config, "speculative_config", None)
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
if (
|
||||
speculative_config is None
|
||||
or getattr(speculative_config, "method", None) != "dspark"
|
||||
or parallel_config.decode_context_parallel_size <= 1
|
||||
):
|
||||
return
|
||||
|
||||
if self.non_causal_multi_token_decode:
|
||||
supported = self.supports_non_causal_multi_token_dcp
|
||||
query_mode = "non-causal draft"
|
||||
else:
|
||||
supported = supports_dcp_with_varlen
|
||||
query_mode = "causal multi-token"
|
||||
|
||||
if not supported:
|
||||
raise ValueError(
|
||||
f"{type(self).__name__} does not support {query_mode} MLA "
|
||||
"attention for DSpark with decode context parallelism. Select "
|
||||
"a backend with explicit DSpark DCP support or set "
|
||||
"decode_context_parallel_size=1."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def determine_chunked_prefill_workspace_size(vllm_config: VllmConfig) -> int:
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
@@ -2084,6 +2112,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.non_causal_multi_token_decode = getattr(
|
||||
kv_cache_spec, "non_causal_multi_token_decode", False
|
||||
)
|
||||
self._validate_dspark_dcp_support(supports_dcp_with_varlen)
|
||||
|
||||
# A draft cache group can have a different head count from the target.
|
||||
self.num_heads = get_num_attention_heads_from_layers(
|
||||
|
||||
@@ -341,11 +341,6 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
"parallelism."
|
||||
)
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
assert self.dcp_world_size <= 1 or self.rotary_emb is None, (
|
||||
"Kimi-K3 MultiHeadLatentAttention does not support RoPE with decode "
|
||||
"context parallelism because gathered queries require gathered "
|
||||
"positions."
|
||||
)
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.dcp_world_size > 1:
|
||||
query_dtype = (
|
||||
|
||||
@@ -330,6 +330,11 @@ class AttentionBackend(ABC):
|
||||
except NotImplementedError:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_non_causal_dcp(cls) -> bool:
|
||||
builder_cls = cls.get_builder_cls()
|
||||
return bool(getattr(builder_cls, "supports_non_causal_multi_token_dcp", False))
|
||||
|
||||
@classmethod
|
||||
def supports_attn_type(cls, attn_type: str) -> bool:
|
||||
"""Check if backend supports a given attention type.
|
||||
@@ -378,6 +383,7 @@ class AttentionBackend(ABC):
|
||||
use_kv_connector: bool = False,
|
||||
use_pcp: bool = False,
|
||||
use_adaptive_verification: bool = False,
|
||||
use_dcp: bool = False,
|
||||
) -> list[str]:
|
||||
invalid_reasons = []
|
||||
if not cls.supports_head_size(head_size):
|
||||
@@ -414,6 +420,8 @@ class AttentionBackend(ABC):
|
||||
invalid_reasons.append("sliding window not supported")
|
||||
if use_non_causal and not cls.supports_non_causal():
|
||||
invalid_reasons.append("non-causal attention not supported")
|
||||
if use_mla and use_non_causal and use_dcp and not cls.supports_non_causal_dcp():
|
||||
invalid_reasons.append("non-causal MLA attention with DCP not supported")
|
||||
if use_batch_invariant and not cls.supports_batch_invariance():
|
||||
invalid_reasons.append("batch invariance not supported")
|
||||
if use_kv_connector and not cls.supports_kv_connector():
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import ClassVar
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
import torch
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
@@ -15,6 +16,7 @@ from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBackend,
|
||||
MLACommonDecodeMetadata,
|
||||
MLACommonImpl,
|
||||
MLACommonMetadata,
|
||||
MLACommonMetadataBuilder,
|
||||
@@ -30,6 +32,10 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import KVCacheLayoutType
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -112,12 +118,56 @@ def _get_multi_ctas_kv_counter_buffer(
|
||||
return _fi_multi_ctas_kv_counter
|
||||
|
||||
|
||||
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
@dataclass
|
||||
class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata):
|
||||
flattened_block_table: torch.Tensor | None = None
|
||||
flattened_seq_lens: torch.Tensor | None = None
|
||||
query_len: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]):
|
||||
pass
|
||||
|
||||
|
||||
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
|
||||
# Non-causal DSpark blocks are flattened to single-token rows in forward_mqa.
|
||||
supports_non_causal_multi_token_decode: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: "AttentionSpec",
|
||||
layer_names: list[str],
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
kv_cache_spec,
|
||||
layer_names,
|
||||
vllm_config,
|
||||
device,
|
||||
FlashInferMLAMetadata,
|
||||
supports_dcp_with_varlen=True,
|
||||
)
|
||||
|
||||
def _build_decode(
|
||||
self,
|
||||
block_table_tensor: torch.Tensor,
|
||||
seq_lens_device: torch.Tensor,
|
||||
max_seq_len: int,
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> FlashInferMLADecodeMetadata:
|
||||
return FlashInferMLADecodeMetadata(
|
||||
block_table=block_table_tensor,
|
||||
seq_lens=seq_lens_device,
|
||||
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
|
||||
)
|
||||
|
||||
|
||||
class FlashInferMLABackend(MLACommonBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
@@ -193,7 +243,7 @@ class FlashInferMLABackend(MLACommonBackend):
|
||||
return "HND"
|
||||
|
||||
|
||||
class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
# trtllm-gen MLA decode emits LSE in log2 (per flashinfer's own
|
||||
# reference at flashinfer/trace/templates/attention.py:81:
|
||||
@@ -264,7 +314,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: MLACommonMetadata,
|
||||
attn_metadata: FlashInferMLAMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert kv_c_and_k_pe_cache.numel() > 0
|
||||
@@ -276,16 +326,28 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
|
||||
block_table = attn_metadata.decode.block_table
|
||||
seq_lens = attn_metadata.decode.seq_lens
|
||||
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
|
||||
|
||||
if not attn_metadata.causal:
|
||||
# Non-causal DSpark block: flatten to single-token decode rows with
|
||||
# per-row context seq_lens (trtllm-gen has no causal flag and would
|
||||
# otherwise mask the block causally).
|
||||
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
|
||||
q = q.unsqueeze(1)
|
||||
if query_len > 1:
|
||||
block_table = block_table.repeat_interleave(query_len, dim=0)
|
||||
seq_lens = seq_lens.repeat_interleave(query_len)
|
||||
block_table, seq_lens = self._prepare_flattened_decode_metadata(
|
||||
attn_metadata,
|
||||
query_len,
|
||||
causal=False,
|
||||
)
|
||||
elif self.dcp_world_size > 1 and query_len > 1:
|
||||
# Causal DCP block: flatten to single-token decode rows with
|
||||
# per-row rank-local seq_lens for each query's visible prefix.
|
||||
block_table, seq_lens = self._prepare_flattened_decode_metadata(
|
||||
attn_metadata,
|
||||
query_len,
|
||||
causal=True,
|
||||
)
|
||||
q = q.unsqueeze(1)
|
||||
# trtllm API requires extra dimension q_len_per_request for MTP
|
||||
elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0:
|
||||
logger.warning_once(
|
||||
@@ -358,3 +420,49 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
o = o.view(-1, o.shape[-2], o.shape[-1])
|
||||
|
||||
return o, lse
|
||||
|
||||
def _prepare_flattened_decode_metadata(
|
||||
self,
|
||||
attn_metadata: FlashInferMLAMetadata,
|
||||
query_len: int,
|
||||
*,
|
||||
causal: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Prepare flattened decode metadata once for all layers in the group."""
|
||||
decode = attn_metadata.decode
|
||||
assert decode is not None
|
||||
if decode.query_len:
|
||||
assert decode.query_len == query_len
|
||||
assert decode.flattened_block_table is not None
|
||||
assert decode.flattened_seq_lens is not None
|
||||
return decode.flattened_block_table, decode.flattened_seq_lens
|
||||
|
||||
block_table = decode.block_table.repeat_interleave(query_len, dim=0)
|
||||
if causal:
|
||||
global_seq_lens = decode.dcp_tot_seq_lens
|
||||
assert global_seq_lens is not None
|
||||
offsets = torch.arange(
|
||||
query_len - 1,
|
||||
-1,
|
||||
-1,
|
||||
device=global_seq_lens.device,
|
||||
dtype=global_seq_lens.dtype,
|
||||
)
|
||||
per_query_global_lens = torch.clamp(
|
||||
(global_seq_lens.unsqueeze(1) - offsets).reshape(-1), min=0
|
||||
)
|
||||
interleave = self.cp_kv_cache_interleave_size
|
||||
dcp_span = self.dcp_world_size * interleave
|
||||
remainder = torch.clamp(
|
||||
per_query_global_lens % dcp_span - self.dcp_rank * interleave,
|
||||
min=0,
|
||||
max=interleave,
|
||||
)
|
||||
seq_lens = per_query_global_lens // dcp_span * interleave + remainder
|
||||
else:
|
||||
seq_lens = decode.seq_lens.repeat_interleave(query_len)
|
||||
|
||||
decode.flattened_block_table = block_table
|
||||
decode.flattened_seq_lens = seq_lens
|
||||
decode.query_len = query_len
|
||||
return block_table, seq_lens
|
||||
|
||||
@@ -61,6 +61,7 @@ class TokenspeedMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
# The kernel accepts an explicit causal mask, so a non-causal DSpark
|
||||
# block can remain fused instead of being flattened to single tokens.
|
||||
supports_non_causal_multi_token_decode: ClassVar[bool] = True
|
||||
supports_non_causal_multi_token_dcp: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -38,6 +38,7 @@ class AttentionSelectorConfig(NamedTuple):
|
||||
use_kv_connector: bool = False
|
||||
use_pcp: bool = False
|
||||
use_adaptive_verification: bool = False
|
||||
use_dcp: bool = False
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
@@ -56,7 +57,8 @@ class AttentionSelectorConfig(NamedTuple):
|
||||
f"use_batch_invariant={self.use_batch_invariant}, "
|
||||
f"use_kv_connector={self.use_kv_connector}, "
|
||||
f"use_adaptive_verification={self.use_adaptive_verification}, "
|
||||
f"use_pcp={self.use_pcp})"
|
||||
f"use_pcp={self.use_pcp}, "
|
||||
f"use_dcp={self.use_dcp})"
|
||||
)
|
||||
|
||||
|
||||
@@ -168,6 +170,7 @@ def get_attn_backend(
|
||||
use_kv_connector=use_kv_connector,
|
||||
use_pcp=vllm_config.parallel_config.prefill_context_parallel_size > 1,
|
||||
use_adaptive_verification=use_adaptive_verification,
|
||||
use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1,
|
||||
)
|
||||
|
||||
# A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over
|
||||
|
||||
@@ -59,3 +59,24 @@ def _dcp_local_seq_lens_kernel(
|
||||
# For [num_reqs, max_num_reqs), pad with 0
|
||||
local_seq_lens = tl.where(block < num_reqs, local_seq_lens, 0)
|
||||
tl.store(out_ptr + block, local_seq_lens, mask=block < max_num_reqs)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def cp_local_slot(
|
||||
positions,
|
||||
block_numbers,
|
||||
block_size,
|
||||
cp_rank,
|
||||
CP_SIZE: tl.constexpr,
|
||||
CP_INTERLEAVE: tl.constexpr,
|
||||
PAD_ID: tl.constexpr,
|
||||
):
|
||||
"""Return rank-local KV slots, or PAD_ID for positions not owned by this rank."""
|
||||
block_offsets = positions % (block_size * CP_SIZE)
|
||||
if CP_SIZE == 1:
|
||||
return block_numbers * block_size + block_offsets
|
||||
is_local = block_offsets // CP_INTERLEAVE % CP_SIZE == cp_rank
|
||||
rounds = block_offsets // (CP_INTERLEAVE * CP_SIZE)
|
||||
remainder = block_offsets % CP_INTERLEAVE
|
||||
local_offsets = rounds * CP_INTERLEAVE + remainder
|
||||
return tl.where(is_local, block_numbers * block_size + local_offsets, PAD_ID)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_slot_mappings_by_layer,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionState,
|
||||
BatchExecutionDescriptor,
|
||||
@@ -41,6 +42,17 @@ def _prepare_dflash_inputs_to_capture(
|
||||
attn_metadata = None
|
||||
if not skip_attn:
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
dcp_local_seq_lens = None
|
||||
if block_tables.cp_size > 1:
|
||||
prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
input_buffers.seq_lens,
|
||||
num_reqs,
|
||||
block_tables.cp_size,
|
||||
block_tables.cp_rank,
|
||||
block_tables.cp_interleave,
|
||||
)
|
||||
dcp_local_seq_lens = input_buffers.dcp_local_seq_lens
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
@@ -49,6 +61,7 @@ def _prepare_dflash_inputs_to_capture(
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=num_tokens // num_reqs,
|
||||
seq_lens=input_batch.seq_lens,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
max_seq_len=max_model_len,
|
||||
block_tables=input_block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import cp_local_slot, prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
@@ -285,10 +286,21 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_query_per_req: int | None = None,
|
||||
causal: bool | Mapping[int, bool] = False,
|
||||
query_start_loc_np: np.ndarray | None = None,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self.draft_attn_layer_names:
|
||||
return None
|
||||
assert num_query_per_req is None # Omitted for DFlash, read from self instead
|
||||
if dcp_local_seq_lens is None and self.block_tables.cp_size > 1:
|
||||
prepare_dcp_local_seq_lens(
|
||||
self.input_buffers.dcp_local_seq_lens,
|
||||
self.input_buffers.seq_lens,
|
||||
num_reqs,
|
||||
self.block_tables.cp_size,
|
||||
self.block_tables.cp_rank,
|
||||
self.block_tables.cp_interleave,
|
||||
)
|
||||
dcp_local_seq_lens = self.input_buffers.dcp_local_seq_lens
|
||||
return super()._build_draft_attn_metadata(
|
||||
num_reqs,
|
||||
num_reqs_padded,
|
||||
@@ -298,6 +310,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
causal=causal,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -393,6 +406,9 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
seeds,
|
||||
self.block_tables.input_block_tables[gid],
|
||||
self.block_tables.kernel_block_sizes[gid],
|
||||
self.block_tables.cp_rank,
|
||||
self.block_tables.cp_size,
|
||||
self.block_tables.cp_interleave,
|
||||
self.parallel_drafting_token_id,
|
||||
self.num_query_per_req,
|
||||
self.num_speculative_steps,
|
||||
@@ -507,8 +523,11 @@ def _prepare_dflash_inputs_kernel(
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
max_model_len,
|
||||
cp_rank,
|
||||
SAMPLE_FROM_ANCHOR: tl.constexpr,
|
||||
PAD_SLOT_ID: tl.constexpr,
|
||||
CP_SIZE: tl.constexpr,
|
||||
CP_INTERLEAVE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
@@ -543,7 +562,7 @@ def _prepare_dflash_inputs_kernel(
|
||||
# --- Context positions / slots ---
|
||||
ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0)
|
||||
ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_valid_ctx, other=0)
|
||||
ctx_block_num = ctx_pos // block_size
|
||||
ctx_block_num = ctx_pos // (block_size * CP_SIZE)
|
||||
ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1)
|
||||
ctx_block_id = tl.load(
|
||||
block_table_ptr + req_idx * block_table_stride + ctx_block_num,
|
||||
@@ -554,9 +573,12 @@ def _prepare_dflash_inputs_kernel(
|
||||
# to it after eviction; rejected suffix rows are invalid context as well.
|
||||
# Neither kind of row may write draft KV into physical block 0.
|
||||
ctx_resident = is_valid_ctx & (ctx_block_id != 0)
|
||||
local_ctx_slot = cp_local_slot(
|
||||
ctx_pos, ctx_block_id, block_size, cp_rank, CP_SIZE, CP_INTERLEAVE, PAD_SLOT_ID
|
||||
)
|
||||
ctx_slot = tl.where(
|
||||
ctx_resident,
|
||||
ctx_block_id * block_size + (ctx_pos % block_size),
|
||||
local_ctx_slot,
|
||||
PAD_SLOT_ID,
|
||||
)
|
||||
# Stored over the full [0, num_ctx) span while the loads above are masked to
|
||||
@@ -573,7 +595,7 @@ def _prepare_dflash_inputs_kernel(
|
||||
is_bonus = is_query & (query_off == 0)
|
||||
input_id = tl.where(is_bonus, bonus_token, parallel_drafting_token_id)
|
||||
|
||||
q_block_num = query_pos // block_size
|
||||
q_block_num = query_pos // (block_size * CP_SIZE)
|
||||
q_block_num = tl.minimum(q_block_num, block_table_stride - 1)
|
||||
q_block_id = tl.load(
|
||||
block_table_ptr + req_idx * block_table_stride + q_block_num,
|
||||
@@ -583,9 +605,18 @@ def _prepare_dflash_inputs_kernel(
|
||||
# A null block is never a writable cache slot. This can occur when a
|
||||
# sliding-window block table contains evicted/global padding entries.
|
||||
q_resident = is_query & (q_block_id != 0)
|
||||
local_q_slot = cp_local_slot(
|
||||
query_pos,
|
||||
q_block_id,
|
||||
block_size,
|
||||
cp_rank,
|
||||
CP_SIZE,
|
||||
CP_INTERLEAVE,
|
||||
PAD_SLOT_ID,
|
||||
)
|
||||
q_slot = tl.where(
|
||||
q_resident,
|
||||
q_block_id * block_size + (query_pos % block_size),
|
||||
local_q_slot,
|
||||
PAD_SLOT_ID,
|
||||
)
|
||||
|
||||
@@ -681,6 +712,9 @@ def prepare_dflash_inputs(
|
||||
# [max_num_reqs, max_num_blocks]
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
cp_rank: int,
|
||||
cp_size: int,
|
||||
cp_interleave: int,
|
||||
parallel_drafting_token_id: int,
|
||||
num_query_per_req: int,
|
||||
num_speculative_steps: int,
|
||||
@@ -728,7 +762,10 @@ def prepare_dflash_inputs(
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
max_model_len,
|
||||
cp_rank,
|
||||
SAMPLE_FROM_ANCHOR=sample_from_anchor,
|
||||
PAD_SLOT_ID=PAD_SLOT_ID,
|
||||
CP_SIZE=cp_size,
|
||||
CP_INTERLEAVE=cp_interleave,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
@@ -234,6 +234,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
num_query_per_req: int = 1,
|
||||
causal: bool | Mapping[int, bool] = True,
|
||||
query_start_loc_np: np.ndarray | None = None,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if query_start_loc_np is not None:
|
||||
# Non-uniform query layout (e.g. multi-module MTP's mixed
|
||||
@@ -278,6 +279,11 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=max_query_len,
|
||||
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
|
||||
dcp_local_seq_lens=(
|
||||
None
|
||||
if dcp_local_seq_lens is None
|
||||
else dcp_local_seq_lens[:num_reqs_padded]
|
||||
),
|
||||
max_seq_len=self.draft_max_seq_len,
|
||||
block_tables=block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
|
||||
Reference in New Issue
Block a user