DCP supports hybrid attention (#40996)

Signed-off-by: YanXu <yancey.yx@alibaba-inc.com>
Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
Yan Xu
2026-07-09 21:34:45 -07:00
committed by GitHub
co-authored by Jingyi Yang
parent 2d814a0082
commit 95ed0feaa5
26 changed files with 783 additions and 125 deletions
+34
View File
@@ -128,6 +128,18 @@ class KVCacheSpec:
"""
raise NotImplementedError
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
"""
The number of block table entries needed per request, i.e. the row
length of the worker-side block table for this cache group.
Args:
vllm_config: The vllm config.
max_len: The maximum sequence length to size for, including the
encoder length for encoder-decoder models.
"""
return cdiv(max_len, self.block_size)
def copy_with_new_block_size(self, block_size: int) -> Self:
"""
Create a new KVCacheSpec from self but replacing the block size.
@@ -201,6 +213,16 @@ class AttentionSpec(KVCacheSpec):
* get_dtype_size(self.dtype)
)
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
# Attention KV is token-interleaved across DCP/PCP ranks, so each rank
# only stores max_len // (dcp * pcp) tokens per request.
parallel_config = vllm_config.parallel_config
total_cp_size = (
parallel_config.decode_context_parallel_size
* parallel_config.prefill_context_parallel_size
)
return cdiv(max_len, self.block_size * total_cp_size)
@dataclass(frozen=True, kw_only=True)
class FullAttentionSpec(AttentionSpec):
@@ -699,6 +721,18 @@ class MambaSpec(KVCacheSpec):
else:
return self.page_size_bytes * (1 + self.num_speculative_blocks)
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
# Mamba state is replicated across DCP/PCP ranks, never sharded, so
# no CP scaling applies.
if vllm_config.cache_config.mamba_cache_mode == "align":
# Block table rows are position-indexed over the full sequence
# even though only 2 + num_speculative_blocks state blocks are
# resident at a time (earlier states are nulled out by
# remove_skipped_blocks), so the row length must cover max_len
# rather than max_memory_usage_bytes.
return cdiv(max_len, self.block_size) + self.num_speculative_blocks
return cdiv(self.max_memory_usage_bytes(vllm_config), self.page_size_bytes)
def is_uniform_with_collection(
self, kv_cache_specs: dict[str, KVCacheSpec]
) -> bool: