[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -163,8 +163,6 @@ class BlockTable:
|
||||
return
|
||||
assert self.slot_mapping_mode == SlotMappingMode.TOKEN_TO_KV_SLOT
|
||||
|
||||
total_cp_world_size = self.pcp_world_size * self.dcp_world_size
|
||||
total_cp_rank = self.pcp_rank * self.dcp_world_size + self.dcp_rank
|
||||
_compute_slot_mapping_kernel[(num_reqs + 1,)](
|
||||
num_tokens,
|
||||
self.max_num_batched_tokens,
|
||||
@@ -176,8 +174,8 @@ class BlockTable:
|
||||
self.slot_mapping.gpu,
|
||||
KV_CACHE_BLOCK_SIZE=self.kv_cache_block_size,
|
||||
BLOCKS_PER_KV_BLOCK=self.blocks_per_kv_block,
|
||||
TOTAL_CP_WORLD_SIZE=total_cp_world_size,
|
||||
TOTAL_CP_RANK=total_cp_rank,
|
||||
TOTAL_CP_WORLD_SIZE=self.dcp_world_size,
|
||||
TOTAL_CP_RANK=self.dcp_rank,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE=self.cp_kv_cache_interleave_size,
|
||||
PAD_ID=PAD_SLOT_ID,
|
||||
BLOCK_SIZE=1024,
|
||||
|
||||
Reference in New Issue
Block a user