[Bugfix][Kernel] Handle kernel block sizes in V2 DCP slot mapping (#51031)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-24 07:28:38 +00:00
committed by GitHub
parent a047e2543d
commit 0ecc284790
2 changed files with 73 additions and 15 deletions
+44
View File
@@ -132,6 +132,50 @@ def test_block_tables_apply_staged_writes_single_group():
) )
@pytest.mark.parametrize("cp_rank", range(4))
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
"""DCP interleave is expressed in logical-block token coordinates."""
device = torch.device("cuda")
block_tables = BlockTables(
block_sizes=[128],
max_num_reqs=1,
max_num_batched_tokens=1024,
max_num_blocks_per_group=[2],
device=device,
kernel_block_sizes=[64],
cp_size=4,
cp_rank=cp_rank,
cp_interleave=128,
)
block_tables.append_block_ids(
req_index=0,
new_block_ids=([5, 9],),
overwrite=True,
)
block_tables.apply_staged_writes()
idx_mapping = torch.zeros(1, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 1024], dtype=torch.int32, device=device)
positions = torch.arange(1024, dtype=torch.int64, device=device)
actual = block_tables.compute_slot_mappings(
idx_mapping,
query_start_loc,
positions,
num_tokens_padded=1024,
)[0]
expected = torch.full((1024,), -1, dtype=torch.int64, device=device)
first_start = cp_rank * 128
second_start = 512 + first_start
expected[first_start : first_start + 128] = torch.arange(
5 * 128, 6 * 128, dtype=torch.int64, device=device
)
expected[second_start : second_start + 128] = torch.arange(
9 * 128, 10 * 128, dtype=torch.int64, device=device
)
assert torch.equal(actual, expected)
def test_v1_block_table_move_row_clears_vacated_row(): def test_v1_block_table_move_row_clears_vacated_row():
"""condense() moves the last row into a freed slot; the vacated row must """condense() moves the last row into a freed slot; the vacated row must
not keep stale block ids. Padded dummy-run batches dereference stale rows not keep stale block ids. Padded dummy-run batches dereference stale rows
+29 -15
View File
@@ -88,9 +88,9 @@ class BlockTables:
def init_block_table_layout_tensors(self) -> None: def init_block_table_layout_tensors(self) -> None:
# Called at init and after a CuMem kv_cache wake-up. The ptr tensors # Called at init and after a CuMem kv_cache wake-up. The ptr tensors
# cache raw data_ptr() values that go stale once the underlying tensors # cache raw data_ptr() values that go stale once the underlying tensors
# are reallocated on wake; block_sizes_tensor needs re-populating # are reallocated on wake; the size tensors need re-populating because
# because its storage lives under the kv_cache pool tag and comes back # their storage lives under the kv_cache pool tag and comes back with
# with undefined contents. # undefined contents.
self.block_table_ptrs = self._make_ptr_tensor( self.block_table_ptrs = self._make_ptr_tensor(
[b.gpu for b in self.block_tables] [b.gpu for b in self.block_tables]
) )
@@ -100,6 +100,9 @@ class BlockTables:
device=self.device, device=self.device,
) )
self.block_sizes_tensor = torch.tensor( self.block_sizes_tensor = torch.tensor(
self.block_sizes, dtype=torch.int32, device=self.device
)
self.kernel_block_sizes_tensor = torch.tensor(
self.kernel_block_sizes, dtype=torch.int32, device=self.device self.kernel_block_sizes, dtype=torch.int32, device=self.device
) )
self.input_block_table_ptrs = self._make_ptr_tensor(self.input_block_tables) self.input_block_table_ptrs = self._make_ptr_tensor(self.input_block_tables)
@@ -203,6 +206,7 @@ class BlockTables:
self.block_table_ptrs, self.block_table_ptrs,
self.block_table_strides, self.block_table_strides,
self.block_sizes_tensor, self.block_sizes_tensor,
self.kernel_block_sizes_tensor,
slot_mappings, slot_mappings,
slot_mappings.stride(0), slot_mappings.stride(0),
self.cp_rank, self.cp_rank,
@@ -274,6 +278,7 @@ def _compute_slot_mappings_kernel(
block_table_ptrs, # [num_kv_cache_groups] block_table_ptrs, # [num_kv_cache_groups]
block_table_strides, # [num_kv_cache_groups] block_table_strides, # [num_kv_cache_groups]
block_sizes, # [num_kv_cache_groups] block_sizes, # [num_kv_cache_groups]
kernel_block_sizes, # [num_kv_cache_groups]
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens] slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
slot_mappings_stride, slot_mappings_stride,
cp_rank, cp_rank,
@@ -300,7 +305,8 @@ def _compute_slot_mappings_kernel(
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32) block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
block_table_stride = tl.load(block_table_strides + group_id) block_table_stride = tl.load(block_table_strides + group_id)
block_size = tl.load(block_sizes + group_id) kv_block_size = tl.load(block_sizes + group_id)
kernel_block_size = tl.load(kernel_block_sizes + group_id)
req_state_idx = tl.load(idx_mapping + batch_idx) req_state_idx = tl.load(idx_mapping + batch_idx)
start_idx = tl.load(query_start_loc + batch_idx) start_idx = tl.load(query_start_loc + batch_idx)
@@ -309,22 +315,30 @@ def _compute_slot_mappings_kernel(
offset = i + tl.arange(0, TRITON_BLOCK_SIZE) offset = i + tl.arange(0, TRITON_BLOCK_SIZE)
positions = tl.load(pos + offset, mask=offset < end_idx, other=0) positions = tl.load(pos + offset, mask=offset < end_idx, other=0)
block_indices = positions // (block_size * CP_SIZE)
block_offsets = positions % (block_size * CP_SIZE)
block_numbers = tl.load(
block_table_ptr + req_state_idx * block_table_stride + block_indices
)
if CP_SIZE == 1: if CP_SIZE == 1:
# Common case: Context parallelism is not used. # Common case: Context parallelism is not used.
slot_ids = block_numbers * block_size + block_offsets local_positions = positions
is_local = True
else: else:
# Context parallelism is used. # Context parallelism is used.
is_local = block_offsets // CP_INTERLEAVE % CP_SIZE == cp_rank virtual_block_size = kv_block_size * CP_SIZE
rounds = block_offsets // (CP_INTERLEAVE * CP_SIZE) virtual_block_indices = positions // virtual_block_size
remainder = block_offsets % CP_INTERLEAVE virtual_block_offsets = positions % virtual_block_size
is_local = virtual_block_offsets // CP_INTERLEAVE % CP_SIZE == cp_rank
rounds = virtual_block_offsets // (CP_INTERLEAVE * CP_SIZE)
remainder = virtual_block_offsets % CP_INTERLEAVE
local_offsets = rounds * CP_INTERLEAVE + remainder local_offsets = rounds * CP_INTERLEAVE + remainder
slot_ids = block_numbers * block_size + local_offsets local_positions = virtual_block_indices * kv_block_size + local_offsets
block_indices = local_positions // kernel_block_size
block_offsets = local_positions % kernel_block_size
block_numbers = tl.load(
block_table_ptr + req_state_idx * block_table_stride + block_indices,
mask=is_local,
other=0,
)
slot_ids = block_numbers * kernel_block_size + block_offsets
if CP_SIZE != 1:
slot_ids = tl.where(is_local, slot_ids, PAD_ID) slot_ids = tl.where(is_local, slot_ids, PAD_ID)
tl.store(slot_mapping_ptr + offset, slot_ids, mask=offset < end_idx) tl.store(slot_mapping_ptr + offset, slot_ids, mask=offset < end_idx)