[Bugfix][KV Offload] Do not let a recurrent group's unhashed block truncate the load boundary (#52807)
Signed-off-by: Yifan Jiang <19356972+yifjiang@users.noreply.github.com>
This commit is contained in:
@@ -1025,18 +1025,23 @@ class OffloadingConnectorScheduler:
|
||||
num_gpu_blocks = cdiv(num_cached_tokens, tokens_per_block)
|
||||
|
||||
assert len(group_blocks) >= num_gpu_blocks
|
||||
num_locally_computed_gpu_blocks = num_gpu_blocks
|
||||
# Skip null placeholder blocks (used for sliding window or mamba padding).
|
||||
for i, block in enumerate(group_blocks[:num_gpu_blocks]):
|
||||
# ``load_start_gpu_block_idx``: the index in ``group_blocks`` where the
|
||||
# load region begins -- a slice bound, not a count of computed blocks.
|
||||
# Scan from the computed boundary, not 0: sparse groups (Mamba,
|
||||
# SWA) legitimately hold non-null unhashed blocks below it.
|
||||
# Skip nulls (sentinel / out-of-retention padding).
|
||||
first_fresh_gpu_block_idx = cdiv(
|
||||
num_locally_computed_tokens, tokens_per_block
|
||||
)
|
||||
load_start_gpu_block_idx = num_gpu_blocks
|
||||
for i in range(first_fresh_gpu_block_idx, num_gpu_blocks):
|
||||
block = group_blocks[i]
|
||||
if not block.is_null and block.block_hash is None:
|
||||
num_locally_computed_gpu_blocks = i
|
||||
load_start_gpu_block_idx = i
|
||||
break
|
||||
|
||||
assert (
|
||||
num_locally_computed_tokens
|
||||
<= num_locally_computed_gpu_blocks * tokens_per_block
|
||||
)
|
||||
num_pending_gpu_blocks = num_gpu_blocks - num_locally_computed_gpu_blocks
|
||||
assert num_locally_computed_tokens % tokens_per_block == 0
|
||||
num_pending_gpu_blocks = num_gpu_blocks - load_start_gpu_block_idx
|
||||
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
assert (
|
||||
@@ -1049,7 +1054,7 @@ class OffloadingConnectorScheduler:
|
||||
num_chunks = cdiv(num_cached_tokens, tokens_per_chunk)
|
||||
if num_pending_gpu_blocks:
|
||||
start_chunk_idx = (
|
||||
num_locally_computed_gpu_blocks // self.config.blocks_per_chunk
|
||||
load_start_gpu_block_idx // self.config.blocks_per_chunk
|
||||
)
|
||||
end_chunk_idx = num_chunks - (partial_tail_boundary is not None)
|
||||
assert len(offload_keys) >= end_chunk_idx
|
||||
@@ -1063,12 +1068,10 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
dst_block_ids.extend(
|
||||
block.block_id
|
||||
for block in group_blocks[
|
||||
num_locally_computed_gpu_blocks:num_gpu_blocks
|
||||
]
|
||||
for block in group_blocks[load_start_gpu_block_idx:num_gpu_blocks]
|
||||
)
|
||||
group_sizes.append(num_pending_gpu_blocks)
|
||||
block_indices.append(num_locally_computed_gpu_blocks)
|
||||
block_indices.append(load_start_gpu_block_idx)
|
||||
|
||||
# Skip prefix-hit chunks for block-level policy; for
|
||||
# request-level, next_stored_chunk_idx stays at 0 so all
|
||||
|
||||
Reference in New Issue
Block a user