[Bugfix][MRV2] Fix KVCache tensor explicit kernel_block_size dim (#42766)

Signed-off-by: NickLucche <nlucches@redhat.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nicolò Lucchesi
2026-05-18 20:25:41 -07:00
committed by GitHub
co-authored by Nick Hill
parent da03e549b3
commit fba010dd74
6 changed files with 68 additions and 30 deletions
+12 -2
View File
@@ -17,11 +17,13 @@ class BlockTables:
max_num_batched_tokens: int,
max_num_blocks_per_group: list[int],
device: torch.device,
kernel_block_sizes: list[int],
cp_size: int = 1,
cp_rank: int = 0,
cp_interleave: int = 1,
):
self.block_sizes = block_sizes
self.kernel_block_sizes = kernel_block_sizes
self.max_num_reqs = max_num_reqs
self.max_num_batched_tokens = max_num_batched_tokens
self.device = device
@@ -32,10 +34,15 @@ class BlockTables:
self.num_kv_cache_groups = len(self.block_sizes)
assert len(max_num_blocks_per_group) == self.num_kv_cache_groups
self.blocks_per_kv_block = [
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
]
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
self.block_tables: list[StagedWriteTensor] = []
for i in range(self.num_kv_cache_groups):
max_num_blocks = max_num_blocks_per_group[i]
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
block_table = StagedWriteTensor(
(self.max_num_reqs, max_num_blocks),
dtype=torch.int32,
@@ -84,7 +91,7 @@ class BlockTables:
device=self.device,
)
self.block_sizes_tensor = torch.tensor(
self.block_sizes, dtype=torch.int32, device=self.device
self.kernel_block_sizes, dtype=torch.int32, device=self.device
)
self.input_block_table_ptrs = self._make_ptr_tensor(self.input_block_tables)
@@ -97,6 +104,9 @@ class BlockTables:
for i in range(self.num_kv_cache_groups):
start = self.num_blocks.np[i, req_index] if not overwrite else 0
block_ids = new_block_ids[i]
bpk = self.blocks_per_kv_block[i]
if bpk > 1:
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
self.block_tables[i].stage_write(req_index, start, block_ids)
self.num_blocks.np[i, req_index] = start + len(block_ids)