DCP supports hybrid attention (#40996)
Signed-off-by: YanXu <yancey.yx@alibaba-inc.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -128,6 +128,18 @@ class KVCacheSpec:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
"""
|
||||
The number of block table entries needed per request, i.e. the row
|
||||
length of the worker-side block table for this cache group.
|
||||
|
||||
Args:
|
||||
vllm_config: The vllm config.
|
||||
max_len: The maximum sequence length to size for, including the
|
||||
encoder length for encoder-decoder models.
|
||||
"""
|
||||
return cdiv(max_len, self.block_size)
|
||||
|
||||
def copy_with_new_block_size(self, block_size: int) -> Self:
|
||||
"""
|
||||
Create a new KVCacheSpec from self but replacing the block size.
|
||||
@@ -201,6 +213,16 @@ class AttentionSpec(KVCacheSpec):
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
# Attention KV is token-interleaved across DCP/PCP ranks, so each rank
|
||||
# only stores max_len // (dcp * pcp) tokens per request.
|
||||
parallel_config = vllm_config.parallel_config
|
||||
total_cp_size = (
|
||||
parallel_config.decode_context_parallel_size
|
||||
* parallel_config.prefill_context_parallel_size
|
||||
)
|
||||
return cdiv(max_len, self.block_size * total_cp_size)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class FullAttentionSpec(AttentionSpec):
|
||||
@@ -699,6 +721,18 @@ class MambaSpec(KVCacheSpec):
|
||||
else:
|
||||
return self.page_size_bytes * (1 + self.num_speculative_blocks)
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
# Mamba state is replicated across DCP/PCP ranks, never sharded, so
|
||||
# no CP scaling applies.
|
||||
if vllm_config.cache_config.mamba_cache_mode == "align":
|
||||
# Block table rows are position-indexed over the full sequence
|
||||
# even though only 2 + num_speculative_blocks state blocks are
|
||||
# resident at a time (earlier states are nulled out by
|
||||
# remove_skipped_blocks), so the row length must cover max_len
|
||||
# rather than max_memory_usage_bytes.
|
||||
return cdiv(max_len, self.block_size) + self.num_speculative_blocks
|
||||
return cdiv(self.max_memory_usage_bytes(vllm_config), self.page_size_bytes)
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user