From d1e3eee6fb8ed3623241ef5c8e3ac533f775bff9 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:08:45 -0400 Subject: [PATCH] [Spec decode] Support Kimi-K3 DCP with DSpark (#52188) Signed-off-by: wzhao18 --- .../test_dspark_mla_config.py | 23 ---- tests/v1/attention/test_flashinfer_mla_dcp.py | 1 + tests/v1/attention/test_mla_backends.py | 77 +++++++++++ .../spec_decode/test_dflash_prepare_inputs.py | 26 +++- vllm/config/speculative.py | 10 -- .../layers/attention/mla_attention.py | 29 +++++ vllm/models/kimi_k3/nvidia/mla.py | 5 - vllm/v1/attention/backend.py | 8 ++ .../attention/backends/mla/flashinfer_mla.py | 122 +++++++++++++++++- .../attention/backends/mla/tokenspeed_mla.py | 1 + vllm/v1/attention/selector.py | 5 +- vllm/v1/worker/gpu/cp_utils.py | 21 +++ .../gpu/spec_decode/dflash/cudagraph.py | 13 ++ .../gpu/spec_decode/dflash/speculator.py | 45 ++++++- vllm/v1/worker/gpu/spec_decode/speculator.py | 6 + 15 files changed, 341 insertions(+), 51 deletions(-) diff --git a/tests/transformers_utils/test_dspark_mla_config.py b/tests/transformers_utils/test_dspark_mla_config.py index f43d668b73..b81f06e143 100644 --- a/tests/transformers_utils/test_dspark_mla_config.py +++ b/tests/transformers_utils/test_dspark_mla_config.py @@ -140,26 +140,3 @@ def test_dspark_mla_speculative_config_preserves_architecture(tmp_path): assert speculative_config.draft_model_config.architectures == ["K3DSparkModel"] assert speculative_config.draft_model_config.hf_config.model_type == "k3_dspark" assert speculative_config.draft_model_config.use_mla - - -def test_dspark_mla_rejects_decode_context_parallelism(tmp_path): - target_path = tmp_path / "target" - draft_path = tmp_path / "draft" - _write_target_config(target_path) - _write_dspark_config(draft_path) - target_config = ModelConfig( - model=str(target_path), tokenizer_mode="skip", max_model_len=32768 - ) - - with pytest.raises(ValueError, match="does not currently support decode context"): - SpeculativeConfig( - model=str(draft_path), - method="dspark", - num_speculative_tokens=8, - target_model_config=target_config, - target_parallel_config=ParallelConfig( - tensor_parallel_size=2, - decode_context_parallel_size=2, - distributed_executor_backend="external_launcher", - ), - ) diff --git a/tests/v1/attention/test_flashinfer_mla_dcp.py b/tests/v1/attention/test_flashinfer_mla_dcp.py index acc38919b8..f3a46bbb19 100644 --- a/tests/v1/attention/test_flashinfer_mla_dcp.py +++ b/tests/v1/attention/test_flashinfer_mla_dcp.py @@ -45,6 +45,7 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch): impl.bmm1_scale = 1.0 impl.bmm2_scale = 1.0 impl.need_to_return_lse_for_decode = True + impl.dcp_world_size = 2 impl.num_heads = 6 impl.qk_nope_head_dim = 128 impl.kv_lora_rank = 512 diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index cd29193724..72773adcb6 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -837,9 +837,86 @@ def test_mock_mla_dcp_fp8_decode_gathers_quantized_query( def test_tokenspeed_mla_noncausal_capability(): builder = tokenspeed_mla_module.TokenspeedMLAMetadataBuilder assert builder.supports_non_causal_multi_token_decode + assert builder.supports_non_causal_multi_token_dcp assert tokenspeed_mla_module.TokenspeedMLABackend.supports_non_causal() +def test_flashinfer_mla_dcp_multi_token_decode_uses_per_query_bounds(monkeypatch): + flashinfer_mla_module = pytest.importorskip( + "vllm.v1.attention.backends.mla.flashinfer_mla" + ) + + decode_call = None + + def fake_decode(**kwargs): + nonlocal decode_call + decode_call = kwargs + query = kwargs["query"] + output = torch.empty(*query.shape[:-1], 512, dtype=torch.bfloat16) + lse = torch.empty(query.shape[0], query.shape[-2], dtype=torch.float32) + return output, lse + + monkeypatch.setattr( + flashinfer_mla_module, + "trtllm_batch_decode_with_kv_cache_mla", + fake_decode, + ) + monkeypatch.setattr( + flashinfer_mla_module, + "_get_workspace_buffer", + lambda return_lse: torch.empty(1, dtype=torch.int8), + ) + + impl = object.__new__(flashinfer_mla_module.FlashInferMLAImpl) + impl.dcp_world_size = 2 + impl.dcp_rank = 1 + impl.cp_kv_cache_interleave_size = 1 + impl.need_to_return_lse_for_decode = True + impl.kv_lora_rank = 512 + impl.qk_nope_head_dim = 128 + impl.qk_rope_head_dim = 64 + impl.bmm1_scale = 1.0 + impl.bmm2_scale = 1.0 + + block_table = torch.tensor([[1], [2]], dtype=torch.int32) + metadata = SimpleNamespace( + num_decodes=2, + num_decode_tokens=6, + max_seq_len=7, + causal=True, + decode=SimpleNamespace( + block_table=block_table, + seq_lens=torch.tensor([5, 6], dtype=torch.int32), + dcp_tot_seq_lens=torch.tensor([10, 13], dtype=torch.int32), + flattened_block_table=None, + flattened_seq_lens=None, + query_len=0, + ), + ) + query = torch.empty(6, 2, 576, dtype=torch.bfloat16) + kv_cache = torch.empty(3, 16, 576, dtype=torch.bfloat16) + + output, lse = impl.forward_mqa( + query, + kv_cache, + metadata, + SimpleNamespace(), + ) + + assert output.shape == (6, 2, 512) + assert lse is not None + assert lse.shape == (6, 2) + assert decode_call is not None + assert decode_call["query"].shape == (6, 1, 2, 576) + torch.testing.assert_close( + decode_call["seq_lens"], + torch.tensor([4, 4, 5, 5, 6, 6], dtype=torch.int32), + ) + torch.testing.assert_close( + decode_call["block_tables"], block_table.repeat_interleave(3, dim=0) + ) + + @pytest.mark.parametrize( ("causal", "tokens_per_decode", "dcp_world_size", "dcp_rank"), [ diff --git a/tests/v1/spec_decode/test_dflash_prepare_inputs.py b/tests/v1/spec_decode/test_dflash_prepare_inputs.py index 50a414a837..16d6d8e516 100644 --- a/tests/v1/spec_decode/test_dflash_prepare_inputs.py +++ b/tests/v1/spec_decode/test_dflash_prepare_inputs.py @@ -17,7 +17,14 @@ pytestmark = pytest.mark.skipif( ) -def _run_prepare(*, target_positions: list[int], block_table_values: list[int]): +def _run_prepare( + *, + target_positions: list[int], + block_table_values: list[int], + cp_rank: int = 0, + cp_size: int = 1, + cp_interleave: int = 1, +): device = torch.device("cuda") max_num_reqs = 4 max_num_tokens = 16 @@ -86,6 +93,9 @@ def _run_prepare(*, target_positions: list[int], block_table_values: list[int]): input_seeds, block_table, 4, + cp_rank, + cp_size, + cp_interleave, 123, num_speculative_steps, num_speculative_steps, @@ -131,6 +141,20 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix(): assert out.seeds[2].item() == 17 +def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp(): + out = _run_prepare( + target_positions=[10, 11, 12, 13], + block_table_values=[0, 7, 8, 9], + cp_rank=1, + cp_size=2, + cp_interleave=2, + ) + + assert out.context_positions[:4].tolist() == [10, 11, 0, 0] + assert out.context_slot_mapping[:4].tolist() == [28, 29, PAD_SLOT_ID, PAD_SLOT_ID] + assert out.query_slot_mapping[:3].tolist() == [PAD_SLOT_ID, PAD_SLOT_ID, 30] + + def test_prepare_dflash_inputs_never_writes_the_null_block(): # The valid context uses logical block 0 and the replacement query uses # logical block 1. Both map to the null block and must remain unwritable. diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index a85b895d86..c3e0866b45 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -1059,16 +1059,6 @@ class SpeculativeConfig: if self.method in ("dflash", "dspark"): self.parallel_drafting = True - if ( - self.method == "dspark" - and "K3DSparkModel" in self.draft_model_config.architectures - and self.target_parallel_config.decode_context_parallel_size > 1 - ): - raise ValueError( - "MLA DSpark does not currently support decode context " - "parallelism; set decode_context_parallel_size=1." - ) - if self.num_speculative_tokens is not None and hasattr( self.draft_model_config.hf_config, "num_lookahead_tokens" ): diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 839f4266cd..72d65c1574 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -1984,6 +1984,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): # Whether this builder can flatten a non-causal query block into decode rows. supports_non_causal_multi_token_decode: ClassVar[bool] = False + # Whether can support non-causal multi-token decode with DCP KV cache. + supports_non_causal_multi_token_dcp: ClassVar[bool] = False + # The threshold for reordering the batch into decode and prefill requests. # If > 1, the batch will be reordered such that requests with # query length <= threshold are classified as decode requests. @@ -1991,6 +1994,31 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): # when speculative decoding is enabled. reorder_batch_threshold: int = 1 + def _validate_dspark_dcp_support(self, supports_dcp_with_varlen: bool) -> None: + speculative_config = getattr(self.vllm_config, "speculative_config", None) + parallel_config = self.vllm_config.parallel_config + if ( + speculative_config is None + or getattr(speculative_config, "method", None) != "dspark" + or parallel_config.decode_context_parallel_size <= 1 + ): + return + + if self.non_causal_multi_token_decode: + supported = self.supports_non_causal_multi_token_dcp + query_mode = "non-causal draft" + else: + supported = supports_dcp_with_varlen + query_mode = "causal multi-token" + + if not supported: + raise ValueError( + f"{type(self).__name__} does not support {query_mode} MLA " + "attention for DSpark with decode context parallelism. Select " + "a backend with explicit DSpark DCP support or set " + "decode_context_parallel_size=1." + ) + @staticmethod def determine_chunked_prefill_workspace_size(vllm_config: VllmConfig) -> int: scheduler_config = vllm_config.scheduler_config @@ -2084,6 +2112,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): self.non_causal_multi_token_decode = getattr( kv_cache_spec, "non_causal_multi_token_decode", False ) + self._validate_dspark_dcp_support(supports_dcp_with_varlen) # A draft cache group can have a different head count from the target. self.num_heads = get_num_attention_heads_from_layers( diff --git a/vllm/models/kimi_k3/nvidia/mla.py b/vllm/models/kimi_k3/nvidia/mla.py index 71e9fa79f0..2c67a8a4c4 100644 --- a/vllm/models/kimi_k3/nvidia/mla.py +++ b/vllm/models/kimi_k3/nvidia/mla.py @@ -341,11 +341,6 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): "parallelism." ) self.dcp_world_size = parallel_config.decode_context_parallel_size - assert self.dcp_world_size <= 1 or self.rotary_emb is None, ( - "Kimi-K3 MultiHeadLatentAttention does not support RoPE with decode " - "context parallelism because gathered queries require gathered " - "positions." - ) self.dcp_manager: MLADCPManager | None = None if self.dcp_world_size > 1: query_dtype = ( diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 672f9e6376..daa217f855 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -330,6 +330,11 @@ class AttentionBackend(ABC): except NotImplementedError: return False + @classmethod + def supports_non_causal_dcp(cls) -> bool: + builder_cls = cls.get_builder_cls() + return bool(getattr(builder_cls, "supports_non_causal_multi_token_dcp", False)) + @classmethod def supports_attn_type(cls, attn_type: str) -> bool: """Check if backend supports a given attention type. @@ -378,6 +383,7 @@ class AttentionBackend(ABC): use_kv_connector: bool = False, use_pcp: bool = False, use_adaptive_verification: bool = False, + use_dcp: bool = False, ) -> list[str]: invalid_reasons = [] if not cls.supports_head_size(head_size): @@ -414,6 +420,8 @@ class AttentionBackend(ABC): invalid_reasons.append("sliding window not supported") if use_non_causal and not cls.supports_non_causal(): invalid_reasons.append("non-causal attention not supported") + if use_mla and use_non_causal and use_dcp and not cls.supports_non_causal_dcp(): + invalid_reasons.append("non-causal MLA attention with DCP not supported") if use_batch_invariant and not cls.supports_batch_invariance(): invalid_reasons.append("batch invariance not supported") if use_kv_connector and not cls.supports_kv_connector(): diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla.py b/vllm/v1/attention/backends/mla/flashinfer_mla.py index bf92bf692b..79beb5332e 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import ClassVar +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar import torch from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla @@ -15,6 +16,7 @@ from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.model_executor.layers.attention.mla_attention import ( MLACommonBackend, + MLACommonDecodeMetadata, MLACommonImpl, MLACommonMetadata, MLACommonMetadataBuilder, @@ -30,6 +32,10 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.backends.utils import KVCacheLayoutType +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.v1.kv_cache_interface import AttentionSpec + logger = init_logger(__name__) @@ -112,12 +118,56 @@ def _get_multi_ctas_kv_counter_buffer( return _fi_multi_ctas_kv_counter -class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): +@dataclass +class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata): + flattened_block_table: torch.Tensor | None = None + flattened_seq_lens: torch.Tensor | None = None + query_len: int = 0 + + +@dataclass +class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]): + pass + + +class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]): _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM # Non-causal DSpark blocks are flattened to single-token rows in forward_mqa. supports_non_causal_multi_token_decode: ClassVar[bool] = True + def __init__( + self, + kv_cache_spec: "AttentionSpec", + layer_names: list[str], + vllm_config: "VllmConfig", + device: torch.device, + ) -> None: + super().__init__( + kv_cache_spec, + layer_names, + vllm_config, + device, + FlashInferMLAMetadata, + supports_dcp_with_varlen=True, + ) + + def _build_decode( + self, + block_table_tensor: torch.Tensor, + seq_lens_device: torch.Tensor, + max_seq_len: int, + query_start_loc_cpu: torch.Tensor, + query_start_loc_device: torch.Tensor, + num_decode_tokens: int, + dcp_tot_seq_lens_device: torch.Tensor | None, + ) -> FlashInferMLADecodeMetadata: + return FlashInferMLADecodeMetadata( + block_table=block_table_tensor, + seq_lens=seq_lens_device, + dcp_tot_seq_lens=dcp_tot_seq_lens_device, + ) + class FlashInferMLABackend(MLACommonBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] @@ -193,7 +243,7 @@ class FlashInferMLABackend(MLACommonBackend): return "HND" -class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): +class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]): can_return_lse_for_decode: bool = True # trtllm-gen MLA decode emits LSE in log2 (per flashinfer's own # reference at flashinfer/trace/templates/attention.py:81: @@ -264,7 +314,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): self, q: torch.Tensor | tuple[torch.Tensor, torch.Tensor], kv_c_and_k_pe_cache: torch.Tensor, - attn_metadata: MLACommonMetadata, + attn_metadata: FlashInferMLAMetadata, layer: AttentionLayer, ) -> tuple[torch.Tensor, torch.Tensor | None]: assert kv_c_and_k_pe_cache.numel() > 0 @@ -276,16 +326,28 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): block_table = attn_metadata.decode.block_table seq_lens = attn_metadata.decode.seq_lens + query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes if not attn_metadata.causal: # Non-causal DSpark block: flatten to single-token decode rows with # per-row context seq_lens (trtllm-gen has no causal flag and would # otherwise mask the block causally). - query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes q = q.unsqueeze(1) if query_len > 1: - block_table = block_table.repeat_interleave(query_len, dim=0) - seq_lens = seq_lens.repeat_interleave(query_len) + block_table, seq_lens = self._prepare_flattened_decode_metadata( + attn_metadata, + query_len, + causal=False, + ) + elif self.dcp_world_size > 1 and query_len > 1: + # Causal DCP block: flatten to single-token decode rows with + # per-row rank-local seq_lens for each query's visible prefix. + block_table, seq_lens = self._prepare_flattened_decode_metadata( + attn_metadata, + query_len, + causal=True, + ) + q = q.unsqueeze(1) # trtllm API requires extra dimension q_len_per_request for MTP elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0: logger.warning_once( @@ -358,3 +420,49 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): o = o.view(-1, o.shape[-2], o.shape[-1]) return o, lse + + def _prepare_flattened_decode_metadata( + self, + attn_metadata: FlashInferMLAMetadata, + query_len: int, + *, + causal: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Prepare flattened decode metadata once for all layers in the group.""" + decode = attn_metadata.decode + assert decode is not None + if decode.query_len: + assert decode.query_len == query_len + assert decode.flattened_block_table is not None + assert decode.flattened_seq_lens is not None + return decode.flattened_block_table, decode.flattened_seq_lens + + block_table = decode.block_table.repeat_interleave(query_len, dim=0) + if causal: + global_seq_lens = decode.dcp_tot_seq_lens + assert global_seq_lens is not None + offsets = torch.arange( + query_len - 1, + -1, + -1, + device=global_seq_lens.device, + dtype=global_seq_lens.dtype, + ) + per_query_global_lens = torch.clamp( + (global_seq_lens.unsqueeze(1) - offsets).reshape(-1), min=0 + ) + interleave = self.cp_kv_cache_interleave_size + dcp_span = self.dcp_world_size * interleave + remainder = torch.clamp( + per_query_global_lens % dcp_span - self.dcp_rank * interleave, + min=0, + max=interleave, + ) + seq_lens = per_query_global_lens // dcp_span * interleave + remainder + else: + seq_lens = decode.seq_lens.repeat_interleave(query_len) + + decode.flattened_block_table = block_table + decode.flattened_seq_lens = seq_lens + decode.query_len = query_len + return block_table, seq_lens diff --git a/vllm/v1/attention/backends/mla/tokenspeed_mla.py b/vllm/v1/attention/backends/mla/tokenspeed_mla.py index 8ab81c438f..57a628b82b 100644 --- a/vllm/v1/attention/backends/mla/tokenspeed_mla.py +++ b/vllm/v1/attention/backends/mla/tokenspeed_mla.py @@ -61,6 +61,7 @@ class TokenspeedMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): # The kernel accepts an explicit causal mask, so a non-causal DSpark # block can remain fused instead of being flattened to single tokens. supports_non_causal_multi_token_decode: ClassVar[bool] = True + supports_non_causal_multi_token_dcp: ClassVar[bool] = True def __init__( self, diff --git a/vllm/v1/attention/selector.py b/vllm/v1/attention/selector.py index 00f2aed5c5..b10cb34977 100644 --- a/vllm/v1/attention/selector.py +++ b/vllm/v1/attention/selector.py @@ -38,6 +38,7 @@ class AttentionSelectorConfig(NamedTuple): use_kv_connector: bool = False use_pcp: bool = False use_adaptive_verification: bool = False + use_dcp: bool = False def __repr__(self): return ( @@ -56,7 +57,8 @@ class AttentionSelectorConfig(NamedTuple): f"use_batch_invariant={self.use_batch_invariant}, " f"use_kv_connector={self.use_kv_connector}, " f"use_adaptive_verification={self.use_adaptive_verification}, " - f"use_pcp={self.use_pcp})" + f"use_pcp={self.use_pcp}, " + f"use_dcp={self.use_dcp})" ) @@ -168,6 +170,7 @@ def get_attn_backend( use_kv_connector=use_kv_connector, use_pcp=vllm_config.parallel_config.prefill_context_parallel_size > 1, use_adaptive_verification=use_adaptive_verification, + use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1, ) # A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over diff --git a/vllm/v1/worker/gpu/cp_utils.py b/vllm/v1/worker/gpu/cp_utils.py index 6dd8fd3474..77010990b8 100644 --- a/vllm/v1/worker/gpu/cp_utils.py +++ b/vllm/v1/worker/gpu/cp_utils.py @@ -59,3 +59,24 @@ def _dcp_local_seq_lens_kernel( # For [num_reqs, max_num_reqs), pad with 0 local_seq_lens = tl.where(block < num_reqs, local_seq_lens, 0) tl.store(out_ptr + block, local_seq_lens, mask=block < max_num_reqs) + + +@triton.jit +def cp_local_slot( + positions, + block_numbers, + block_size, + cp_rank, + CP_SIZE: tl.constexpr, + CP_INTERLEAVE: tl.constexpr, + PAD_ID: tl.constexpr, +): + """Return rank-local KV slots, or PAD_ID for positions not owned by this rank.""" + block_offsets = positions % (block_size * CP_SIZE) + if CP_SIZE == 1: + return block_numbers * block_size + block_offsets + is_local = block_offsets // CP_INTERLEAVE % CP_SIZE == cp_rank + rounds = block_offsets // (CP_INTERLEAVE * CP_SIZE) + remainder = block_offsets % CP_INTERLEAVE + local_offsets = rounds * CP_INTERLEAVE + remainder + return tl.where(is_local, block_numbers * block_size + local_offsets, PAD_ID) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py index e59202c879..c68b549745 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py @@ -11,6 +11,7 @@ from vllm.v1.worker.gpu.attn_utils import ( build_slot_mappings_by_layer, ) from vllm.v1.worker.gpu.block_table import BlockTables +from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.cudagraph_utils import ( AttentionState, BatchExecutionDescriptor, @@ -41,6 +42,17 @@ def _prepare_dflash_inputs_to_capture( attn_metadata = None if not skip_attn: query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np) + dcp_local_seq_lens = None + if block_tables.cp_size > 1: + prepare_dcp_local_seq_lens( + input_buffers.dcp_local_seq_lens, + input_buffers.seq_lens, + num_reqs, + block_tables.cp_size, + block_tables.cp_rank, + block_tables.cp_interleave, + ) + dcp_local_seq_lens = input_buffers.dcp_local_seq_lens attn_metadata = build_attn_metadata( attn_groups=attn_groups, num_reqs=num_reqs, @@ -49,6 +61,7 @@ def _prepare_dflash_inputs_to_capture( query_start_loc_cpu=query_start_loc_cpu, max_query_len=num_tokens // num_reqs, seq_lens=input_batch.seq_lens, + dcp_local_seq_lens=dcp_local_seq_lens, max_seq_len=max_model_len, block_tables=input_block_tables, slot_mappings=slot_mappings, diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index eb5dc470b2..92837afd68 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -17,6 +17,7 @@ from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.block_table import BlockTables +from vllm.v1.worker.gpu.cp_utils import cp_local_slot, prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState @@ -285,10 +286,21 @@ class DFlashSpeculator(DraftModelSpeculator): num_query_per_req: int | None = None, causal: bool | Mapping[int, bool] = False, query_start_loc_np: np.ndarray | None = None, + dcp_local_seq_lens: torch.Tensor | None = None, ) -> dict[str, Any] | None: if not self.draft_attn_layer_names: return None assert num_query_per_req is None # Omitted for DFlash, read from self instead + if dcp_local_seq_lens is None and self.block_tables.cp_size > 1: + prepare_dcp_local_seq_lens( + self.input_buffers.dcp_local_seq_lens, + self.input_buffers.seq_lens, + num_reqs, + self.block_tables.cp_size, + self.block_tables.cp_rank, + self.block_tables.cp_interleave, + ) + dcp_local_seq_lens = self.input_buffers.dcp_local_seq_lens return super()._build_draft_attn_metadata( num_reqs, num_reqs_padded, @@ -298,6 +310,7 @@ class DFlashSpeculator(DraftModelSpeculator): num_query_per_req=self.num_query_per_req, causal=causal, query_start_loc_np=query_start_loc_np, + dcp_local_seq_lens=dcp_local_seq_lens, ) @torch.inference_mode() @@ -393,6 +406,9 @@ class DFlashSpeculator(DraftModelSpeculator): seeds, self.block_tables.input_block_tables[gid], self.block_tables.kernel_block_sizes[gid], + self.block_tables.cp_rank, + self.block_tables.cp_size, + self.block_tables.cp_interleave, self.parallel_drafting_token_id, self.num_query_per_req, self.num_speculative_steps, @@ -507,8 +523,11 @@ def _prepare_dflash_inputs_kernel( max_num_reqs, max_num_tokens, max_model_len, + cp_rank, SAMPLE_FROM_ANCHOR: tl.constexpr, PAD_SLOT_ID: tl.constexpr, + CP_SIZE: tl.constexpr, + CP_INTERLEAVE: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): req_idx = tl.program_id(0) @@ -543,7 +562,7 @@ def _prepare_dflash_inputs_kernel( # --- Context positions / slots --- ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0) ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_valid_ctx, other=0) - ctx_block_num = ctx_pos // block_size + ctx_block_num = ctx_pos // (block_size * CP_SIZE) ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1) ctx_block_id = tl.load( block_table_ptr + req_idx * block_table_stride + ctx_block_num, @@ -554,9 +573,12 @@ def _prepare_dflash_inputs_kernel( # to it after eviction; rejected suffix rows are invalid context as well. # Neither kind of row may write draft KV into physical block 0. ctx_resident = is_valid_ctx & (ctx_block_id != 0) + local_ctx_slot = cp_local_slot( + ctx_pos, ctx_block_id, block_size, cp_rank, CP_SIZE, CP_INTERLEAVE, PAD_SLOT_ID + ) ctx_slot = tl.where( ctx_resident, - ctx_block_id * block_size + (ctx_pos % block_size), + local_ctx_slot, PAD_SLOT_ID, ) # Stored over the full [0, num_ctx) span while the loads above are masked to @@ -573,7 +595,7 @@ def _prepare_dflash_inputs_kernel( is_bonus = is_query & (query_off == 0) input_id = tl.where(is_bonus, bonus_token, parallel_drafting_token_id) - q_block_num = query_pos // block_size + q_block_num = query_pos // (block_size * CP_SIZE) q_block_num = tl.minimum(q_block_num, block_table_stride - 1) q_block_id = tl.load( block_table_ptr + req_idx * block_table_stride + q_block_num, @@ -583,9 +605,18 @@ def _prepare_dflash_inputs_kernel( # A null block is never a writable cache slot. This can occur when a # sliding-window block table contains evicted/global padding entries. q_resident = is_query & (q_block_id != 0) + local_q_slot = cp_local_slot( + query_pos, + q_block_id, + block_size, + cp_rank, + CP_SIZE, + CP_INTERLEAVE, + PAD_SLOT_ID, + ) q_slot = tl.where( q_resident, - q_block_id * block_size + (query_pos % block_size), + local_q_slot, PAD_SLOT_ID, ) @@ -681,6 +712,9 @@ def prepare_dflash_inputs( # [max_num_reqs, max_num_blocks] block_table: torch.Tensor, block_size: int, + cp_rank: int, + cp_size: int, + cp_interleave: int, parallel_drafting_token_id: int, num_query_per_req: int, num_speculative_steps: int, @@ -728,7 +762,10 @@ def prepare_dflash_inputs( max_num_reqs, max_num_tokens, max_model_len, + cp_rank, SAMPLE_FROM_ANCHOR=sample_from_anchor, PAD_SLOT_ID=PAD_SLOT_ID, + CP_SIZE=cp_size, + CP_INTERLEAVE=cp_interleave, BLOCK_SIZE=BLOCK_SIZE, ) diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 69e0cc1091..80de83109b 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -234,6 +234,7 @@ class DraftModelSpeculator(BaseSpeculator): num_query_per_req: int = 1, causal: bool | Mapping[int, bool] = True, query_start_loc_np: np.ndarray | None = None, + dcp_local_seq_lens: torch.Tensor | None = None, ) -> dict[str, Any] | None: if query_start_loc_np is not None: # Non-uniform query layout (e.g. multi-module MTP's mixed @@ -278,6 +279,11 @@ class DraftModelSpeculator(BaseSpeculator): query_start_loc_cpu=query_start_loc_cpu, max_query_len=max_query_len, seq_lens=self.input_buffers.seq_lens[:num_reqs_padded], + dcp_local_seq_lens=( + None + if dcp_local_seq_lens is None + else dcp_local_seq_lens[:num_reqs_padded] + ), max_seq_len=self.draft_max_seq_len, block_tables=block_tables, slot_mappings=slot_mappings,