[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -218,14 +218,9 @@ class AttentionSpec(KVCacheSpec):
|
||||
)
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
# Attention KV is token-interleaved across DCP/PCP ranks, so each rank
|
||||
# only stores max_len // (dcp * pcp) tokens per request.
|
||||
parallel_config = vllm_config.parallel_config
|
||||
total_cp_size = (
|
||||
parallel_config.decode_context_parallel_size
|
||||
* parallel_config.prefill_context_parallel_size
|
||||
)
|
||||
return cdiv(max_len, self.block_size * total_cp_size)
|
||||
kv_shard_count = parallel_config.decode_context_parallel_size
|
||||
return cdiv(max_len, self.block_size * kv_shard_count)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
@@ -263,11 +258,8 @@ class FullAttentionSpec(AttentionSpec):
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
pcp_world_size = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
# Note(hc): each dcp rank only need save
|
||||
# (max_model_len//dcp_world_size) tokens locally.
|
||||
if dcp_world_size * pcp_world_size > 1:
|
||||
max_model_len = cdiv(max_model_len, dcp_world_size * pcp_world_size)
|
||||
if dcp_world_size > 1:
|
||||
max_model_len = cdiv(max_model_len, dcp_world_size)
|
||||
return cdiv(max_model_len, self.block_size) * self.page_size_bytes
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user