DCP supports hybrid attention (#40996)

Signed-off-by: YanXu <yancey.yx@alibaba-inc.com>
Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
Yan Xu
2026-07-09 21:34:45 -07:00
committed by GitHub
co-authored by Jingyi Yang
parent 2d814a0082
commit 95ed0feaa5
26 changed files with 783 additions and 125 deletions
+60 -15
View File
@@ -153,10 +153,12 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
KVCacheSpecKind,
KVQuantMode,
MambaSpec,
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
get_kv_cache_spec_kind,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.outputs import (
@@ -201,9 +203,11 @@ from vllm.v1.spec_decode.utils import update_num_computed_tokens_for_batch_chang
from vllm.v1.structured_output.utils import apply_grammar_bitmask
from vllm.v1.utils import CpuGpuBuffer, record_function_or_nullcontext
from vllm.v1.worker import mamba_utils
from vllm.v1.worker.block_table import SlotMappingMode
from vllm.v1.worker.cp_utils import (
check_attention_cp_compatibility,
get_total_cp_world_size,
get_dcp_dummy_context_len,
prepare_dcp_dummy_context_metadata,
)
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
from vllm.v1.worker.ec_connector_model_runner_mixin import ECConnectorModelRunnerMixin
@@ -681,8 +685,13 @@ class GPUModelRunner(
placeholder_block_size = (
self.cache_config.block_size or CacheConfig.DEFAULT_BLOCK_SIZE
)
placeholder_max_num_blocks = cdiv(
max(self.max_model_len, self.max_encoder_len), placeholder_block_size
)
self._init_block_sizes = [placeholder_block_size]
self._init_kernel_block_sizes = [placeholder_block_size]
self._init_max_num_blocks = [placeholder_max_num_blocks]
self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT]
self.input_batch = InputBatch(
max_num_reqs=self.max_num_reqs,
# We need to use the encoder length for encoder-decoder
@@ -693,6 +702,7 @@ class GPUModelRunner(
vocab_size=self.model_config.get_vocab_size(),
block_sizes=[placeholder_block_size],
kernel_block_sizes=[placeholder_block_size],
max_num_blocks_per_req=[placeholder_max_num_blocks],
num_spec_tokens=self.num_spec_tokens,
logitsprocs=build_logitsprocs(
self.vllm_config,
@@ -5858,6 +5868,14 @@ class GPUModelRunner(
num_reqs_padded = (
batch_desc.num_reqs if batch_desc.num_reqs is not None else num_reqs
)
dcp_dummy_context_len = get_dcp_dummy_context_len(
self.dcp_world_size,
self.parallel_config.cp_kv_cache_interleave_size,
hasattr(self, "kv_cache_config"),
create_mixed_batch,
is_graph_capturing,
uniform_decode,
)
ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices(
should_ubatch,
num_scheduled_tokens,
@@ -5900,10 +5918,19 @@ class GPUModelRunner(
# In the mixed batch mode (used for FI warmup), we use
# shorter sequence lengths to run faster.
# TODO(luka) better system for describing dummy batches
seq_lens = torch.tensor( # type: ignore[assignment]
[1] * num_decode_tokens + [num_prefill_tokens + 1],
dtype=torch.int,
)
if dcp_dummy_context_len > 0:
seq_lens = torch.tensor( # type: ignore[assignment]
[1 + dcp_dummy_context_len] * num_decode_tokens
+ [num_prefill_tokens + dcp_dummy_context_len],
dtype=torch.int,
)
else:
seq_lens = torch.tensor( # type: ignore[assignment]
[1] * num_decode_tokens + [num_prefill_tokens + 1],
dtype=torch.int,
)
elif dcp_dummy_context_len > 0:
seq_lens = max_query_len + dcp_dummy_context_len # type: ignore[assignment]
else:
seq_lens = max_query_len # type: ignore[assignment]
self.optimistic_seq_lens_cpu[:num_reqs] = seq_lens
@@ -5919,6 +5946,17 @@ class GPUModelRunner(
)
self.query_start_loc.copy_to_gpu()
prepare_dcp_dummy_context_metadata(
input_batch=self.input_batch,
kv_cache_config=getattr(self, "kv_cache_config", None),
query_pos=self.query_pos,
positions=self.positions,
query_start_loc=self.query_start_loc,
num_reqs=num_reqs,
num_tokens_unpadded=num_tokens_unpadded,
dcp_dummy_context_len=dcp_dummy_context_len,
)
# Sync block table CPU->GPU so cleared rows from
# remove_request() are visible to the attention metadata
# builder. Without this, stale block IDs from finished
@@ -7029,29 +7067,34 @@ class GPUModelRunner(
"""
block_sizes = []
max_num_blocks = []
slot_mapping_modes = []
max_model_len = max(self.max_model_len, self.max_encoder_len)
for kv_cache_group in kv_cache_config.kv_cache_groups:
if isinstance(kv_cache_group.kv_cache_spec, EncoderOnlyAttentionSpec):
kv_cache_spec = kv_cache_group.kv_cache_spec
kv_cache_spec_kind = get_kv_cache_spec_kind(kv_cache_spec)
if kv_cache_spec_kind == KVCacheSpecKind.ENCODER_ONLY_ATTENTION:
continue
block_size = kv_cache_group.kv_cache_spec.block_size
block_size = kv_cache_spec.block_size
block_sizes.append(block_size)
max_num_blocks_per_req = cdiv(
max_model_len, block_size * get_total_cp_world_size()
if kv_cache_spec_kind == KVCacheSpecKind.MAMBA:
slot_mapping_modes.append(SlotMappingMode.NONE)
else:
slot_mapping_modes.append(SlotMappingMode.TOKEN_TO_KV_SLOT)
max_num_blocks_per_req = kv_cache_spec.max_num_blocks_per_req(
self.vllm_config, max_model_len
)
if isinstance(kv_cache_group.kv_cache_spec, MambaSpec):
max_num_blocks_per_req = (
max_num_blocks_per_req
if self.cache_config.enable_prefix_caching
else 1
) + kv_cache_group.kv_cache_spec.num_speculative_blocks
max_num_blocks.append(max_num_blocks_per_req)
if (
block_sizes != self._init_block_sizes
or kernel_block_sizes != self._init_kernel_block_sizes
or max_num_blocks != self._init_max_num_blocks
or slot_mapping_modes != self._init_slot_mapping_modes
):
self._init_block_sizes = block_sizes
self._init_kernel_block_sizes = kernel_block_sizes
self._init_max_num_blocks = max_num_blocks
self._init_slot_mapping_modes = slot_mapping_modes
self.input_batch = InputBatch(
max_num_reqs=self.max_num_reqs,
max_model_len=max_model_len,
@@ -7065,7 +7108,9 @@ class GPUModelRunner(
logitsprocs=self.input_batch.logitsprocs,
logitsprocs_need_output_token_ids=self.input_batch.logitsprocs_need_output_token_ids,
is_pooling_model=self.is_pooling_model,
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
reasoning_config=self.vllm_config.reasoning_config,
slot_mapping_modes=slot_mapping_modes,
)
assert self._init_block_sizes == block_sizes, (