[Bugfix][Kernel] Handle kernel block sizes in V2 DCP slot mapping (#51031)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -132,6 +132,50 @@ def test_block_tables_apply_staged_writes_single_group():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("cp_rank", range(4))
|
||||||
|
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||||
|
"""DCP interleave is expressed in logical-block token coordinates."""
|
||||||
|
device = torch.device("cuda")
|
||||||
|
block_tables = BlockTables(
|
||||||
|
block_sizes=[128],
|
||||||
|
max_num_reqs=1,
|
||||||
|
max_num_batched_tokens=1024,
|
||||||
|
max_num_blocks_per_group=[2],
|
||||||
|
device=device,
|
||||||
|
kernel_block_sizes=[64],
|
||||||
|
cp_size=4,
|
||||||
|
cp_rank=cp_rank,
|
||||||
|
cp_interleave=128,
|
||||||
|
)
|
||||||
|
block_tables.append_block_ids(
|
||||||
|
req_index=0,
|
||||||
|
new_block_ids=([5, 9],),
|
||||||
|
overwrite=True,
|
||||||
|
)
|
||||||
|
block_tables.apply_staged_writes()
|
||||||
|
|
||||||
|
idx_mapping = torch.zeros(1, dtype=torch.int32, device=device)
|
||||||
|
query_start_loc = torch.tensor([0, 1024], dtype=torch.int32, device=device)
|
||||||
|
positions = torch.arange(1024, dtype=torch.int64, device=device)
|
||||||
|
actual = block_tables.compute_slot_mappings(
|
||||||
|
idx_mapping,
|
||||||
|
query_start_loc,
|
||||||
|
positions,
|
||||||
|
num_tokens_padded=1024,
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
expected = torch.full((1024,), -1, dtype=torch.int64, device=device)
|
||||||
|
first_start = cp_rank * 128
|
||||||
|
second_start = 512 + first_start
|
||||||
|
expected[first_start : first_start + 128] = torch.arange(
|
||||||
|
5 * 128, 6 * 128, dtype=torch.int64, device=device
|
||||||
|
)
|
||||||
|
expected[second_start : second_start + 128] = torch.arange(
|
||||||
|
9 * 128, 10 * 128, dtype=torch.int64, device=device
|
||||||
|
)
|
||||||
|
assert torch.equal(actual, expected)
|
||||||
|
|
||||||
|
|
||||||
def test_v1_block_table_move_row_clears_vacated_row():
|
def test_v1_block_table_move_row_clears_vacated_row():
|
||||||
"""condense() moves the last row into a freed slot; the vacated row must
|
"""condense() moves the last row into a freed slot; the vacated row must
|
||||||
not keep stale block ids. Padded dummy-run batches dereference stale rows
|
not keep stale block ids. Padded dummy-run batches dereference stale rows
|
||||||
|
|||||||
@@ -88,9 +88,9 @@ class BlockTables:
|
|||||||
def init_block_table_layout_tensors(self) -> None:
|
def init_block_table_layout_tensors(self) -> None:
|
||||||
# Called at init and after a CuMem kv_cache wake-up. The ptr tensors
|
# Called at init and after a CuMem kv_cache wake-up. The ptr tensors
|
||||||
# cache raw data_ptr() values that go stale once the underlying tensors
|
# cache raw data_ptr() values that go stale once the underlying tensors
|
||||||
# are reallocated on wake; block_sizes_tensor needs re-populating
|
# are reallocated on wake; the size tensors need re-populating because
|
||||||
# because its storage lives under the kv_cache pool tag and comes back
|
# their storage lives under the kv_cache pool tag and comes back with
|
||||||
# with undefined contents.
|
# undefined contents.
|
||||||
self.block_table_ptrs = self._make_ptr_tensor(
|
self.block_table_ptrs = self._make_ptr_tensor(
|
||||||
[b.gpu for b in self.block_tables]
|
[b.gpu for b in self.block_tables]
|
||||||
)
|
)
|
||||||
@@ -100,6 +100,9 @@ class BlockTables:
|
|||||||
device=self.device,
|
device=self.device,
|
||||||
)
|
)
|
||||||
self.block_sizes_tensor = torch.tensor(
|
self.block_sizes_tensor = torch.tensor(
|
||||||
|
self.block_sizes, dtype=torch.int32, device=self.device
|
||||||
|
)
|
||||||
|
self.kernel_block_sizes_tensor = torch.tensor(
|
||||||
self.kernel_block_sizes, dtype=torch.int32, device=self.device
|
self.kernel_block_sizes, dtype=torch.int32, device=self.device
|
||||||
)
|
)
|
||||||
self.input_block_table_ptrs = self._make_ptr_tensor(self.input_block_tables)
|
self.input_block_table_ptrs = self._make_ptr_tensor(self.input_block_tables)
|
||||||
@@ -203,6 +206,7 @@ class BlockTables:
|
|||||||
self.block_table_ptrs,
|
self.block_table_ptrs,
|
||||||
self.block_table_strides,
|
self.block_table_strides,
|
||||||
self.block_sizes_tensor,
|
self.block_sizes_tensor,
|
||||||
|
self.kernel_block_sizes_tensor,
|
||||||
slot_mappings,
|
slot_mappings,
|
||||||
slot_mappings.stride(0),
|
slot_mappings.stride(0),
|
||||||
self.cp_rank,
|
self.cp_rank,
|
||||||
@@ -274,6 +278,7 @@ def _compute_slot_mappings_kernel(
|
|||||||
block_table_ptrs, # [num_kv_cache_groups]
|
block_table_ptrs, # [num_kv_cache_groups]
|
||||||
block_table_strides, # [num_kv_cache_groups]
|
block_table_strides, # [num_kv_cache_groups]
|
||||||
block_sizes, # [num_kv_cache_groups]
|
block_sizes, # [num_kv_cache_groups]
|
||||||
|
kernel_block_sizes, # [num_kv_cache_groups]
|
||||||
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
|
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
|
||||||
slot_mappings_stride,
|
slot_mappings_stride,
|
||||||
cp_rank,
|
cp_rank,
|
||||||
@@ -300,7 +305,8 @@ def _compute_slot_mappings_kernel(
|
|||||||
|
|
||||||
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
|
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
|
||||||
block_table_stride = tl.load(block_table_strides + group_id)
|
block_table_stride = tl.load(block_table_strides + group_id)
|
||||||
block_size = tl.load(block_sizes + group_id)
|
kv_block_size = tl.load(block_sizes + group_id)
|
||||||
|
kernel_block_size = tl.load(kernel_block_sizes + group_id)
|
||||||
|
|
||||||
req_state_idx = tl.load(idx_mapping + batch_idx)
|
req_state_idx = tl.load(idx_mapping + batch_idx)
|
||||||
start_idx = tl.load(query_start_loc + batch_idx)
|
start_idx = tl.load(query_start_loc + batch_idx)
|
||||||
@@ -309,22 +315,30 @@ def _compute_slot_mappings_kernel(
|
|||||||
offset = i + tl.arange(0, TRITON_BLOCK_SIZE)
|
offset = i + tl.arange(0, TRITON_BLOCK_SIZE)
|
||||||
positions = tl.load(pos + offset, mask=offset < end_idx, other=0)
|
positions = tl.load(pos + offset, mask=offset < end_idx, other=0)
|
||||||
|
|
||||||
block_indices = positions // (block_size * CP_SIZE)
|
|
||||||
block_offsets = positions % (block_size * CP_SIZE)
|
|
||||||
block_numbers = tl.load(
|
|
||||||
block_table_ptr + req_state_idx * block_table_stride + block_indices
|
|
||||||
)
|
|
||||||
|
|
||||||
if CP_SIZE == 1:
|
if CP_SIZE == 1:
|
||||||
# Common case: Context parallelism is not used.
|
# Common case: Context parallelism is not used.
|
||||||
slot_ids = block_numbers * block_size + block_offsets
|
local_positions = positions
|
||||||
|
is_local = True
|
||||||
else:
|
else:
|
||||||
# Context parallelism is used.
|
# Context parallelism is used.
|
||||||
is_local = block_offsets // CP_INTERLEAVE % CP_SIZE == cp_rank
|
virtual_block_size = kv_block_size * CP_SIZE
|
||||||
rounds = block_offsets // (CP_INTERLEAVE * CP_SIZE)
|
virtual_block_indices = positions // virtual_block_size
|
||||||
remainder = block_offsets % CP_INTERLEAVE
|
virtual_block_offsets = positions % virtual_block_size
|
||||||
|
is_local = virtual_block_offsets // CP_INTERLEAVE % CP_SIZE == cp_rank
|
||||||
|
rounds = virtual_block_offsets // (CP_INTERLEAVE * CP_SIZE)
|
||||||
|
remainder = virtual_block_offsets % CP_INTERLEAVE
|
||||||
local_offsets = rounds * CP_INTERLEAVE + remainder
|
local_offsets = rounds * CP_INTERLEAVE + remainder
|
||||||
slot_ids = block_numbers * block_size + local_offsets
|
local_positions = virtual_block_indices * kv_block_size + local_offsets
|
||||||
|
|
||||||
|
block_indices = local_positions // kernel_block_size
|
||||||
|
block_offsets = local_positions % kernel_block_size
|
||||||
|
block_numbers = tl.load(
|
||||||
|
block_table_ptr + req_state_idx * block_table_stride + block_indices,
|
||||||
|
mask=is_local,
|
||||||
|
other=0,
|
||||||
|
)
|
||||||
|
slot_ids = block_numbers * kernel_block_size + block_offsets
|
||||||
|
if CP_SIZE != 1:
|
||||||
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
|
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
|
||||||
|
|
||||||
tl.store(slot_mapping_ptr + offset, slot_ids, mask=offset < end_idx)
|
tl.store(slot_mapping_ptr + offset, slot_ids, mask=offset < end_idx)
|
||||||
|
|||||||
Reference in New Issue
Block a user