[Bugfix][KV Offload] Stop offloading the final sampled token's KV slot (#54288)
Signed-off-by: almogtavor <almogtavor@gmail.com> Signed-off-by: Almog Tavor <70065337+almogtavor@users.noreply.github.com> Co-authored-by: Or Ozeri <or@ozery.com>
This commit is contained in:
@@ -371,18 +371,20 @@ def test_scheduler_reports_allocation_failure(request_runner):
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
@pytest.mark.parametrize("prompt_offset", [-1, -2])
|
||||
def test_last_block_offloaded_at_request_finish(
|
||||
def test_final_sampled_token_does_not_complete_an_offloaded_block(
|
||||
request_runner, async_scheduling: bool, prompt_offset: int
|
||||
):
|
||||
"""EOS fills the last block at request finish - verify the final block is stored.
|
||||
"""A block whose last slot holds the final sampled token is not stored.
|
||||
|
||||
prompt = block_size + prompt_offset tokens → not a full block at schedule time,
|
||||
so _build_store_jobs creates no store job. After EOS, request_finished
|
||||
keeps req_status alive so _build_store_jobs can process it on the next step.
|
||||
That token came out of the previous position's forward pass; its own KV
|
||||
slot is never written, and under spec decode it holds the first rejected
|
||||
draft's KV. Storing the block would publish it under a content hash that
|
||||
the GPU prefix cache itself declines to commit, so a later request with
|
||||
the same tokens would load the unwritten slot.
|
||||
|
||||
prompt_offset=-1: EOS fills the block → store job created on next step.
|
||||
prompt_offset=-2: block remains partial → no store job, cleanup in
|
||||
_build_store_jobs deletes req_status.
|
||||
prompt = 2 * block_size + prompt_offset tokens. Block 0 is filled by
|
||||
prefill and stored either way. At prompt_offset=-1 the EOS fills block 1,
|
||||
which the finishing step must still decline; at -2 block 1 stays partial.
|
||||
"""
|
||||
block_size = 4
|
||||
runner = request_runner(
|
||||
@@ -390,18 +392,11 @@ def test_last_block_offloaded_at_request_finish(
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=async_scheduling,
|
||||
)
|
||||
# prompt = block_size + prompt_offset tokens
|
||||
runner.new_request(token_ids=[0] * (block_size + prompt_offset))
|
||||
runner.new_request(token_ids=[0] * (2 * block_size + prompt_offset))
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(list(keys))
|
||||
)
|
||||
|
||||
if prompt_offset == -1:
|
||||
# EOS fills the block, so a store job is created for block 0.
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0,))
|
||||
else:
|
||||
# Block remains partial, so no store job is created.
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0,))
|
||||
|
||||
cs = runner.connector_scheduler
|
||||
# After the full run completes, req_status is cleaned up.
|
||||
@@ -933,10 +928,13 @@ def test_on_request_finished_not_deferred_until_store_completion(
|
||||
def test_on_request_finished_fires_after_final_block_store(
|
||||
request_runner, async_scheduling: bool
|
||||
):
|
||||
"""on_request_finished fires after the final-block prepare_store at EOS.
|
||||
"""on_request_finished fires after the final prepare_store of the request.
|
||||
|
||||
When EOS fills a partial block, request_finished() keeps req_status alive
|
||||
so _build_store_jobs can create a store job for it on the next step.
|
||||
request_finished() keeps req_status alive so _build_store_jobs can retry a
|
||||
block on the next step; the retry must be issued before the manager is told
|
||||
the request is over. Every attempt made before the request finishes is
|
||||
declined here, so the store that succeeds is the one from the finishing
|
||||
step and the ordering has something to order.
|
||||
"""
|
||||
block_size = 4
|
||||
runner = request_runner(
|
||||
@@ -950,22 +948,31 @@ def test_on_request_finished_fires_after_final_block_store(
|
||||
("on_request_finished", req_context.req_id)
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * block_size)
|
||||
request = runner.scheduler.requests[str(runner.req_id)]
|
||||
|
||||
def prepare_store(keys, req_context):
|
||||
calls.append(("prepare_store", req_context.req_id))
|
||||
return generate_store_output(keys)
|
||||
finished = request.is_finished()
|
||||
calls.append(
|
||||
("stored_after_finish" if finished else "prepare_store", req_context.req_id)
|
||||
)
|
||||
return generate_store_output(keys) if finished else None
|
||||
|
||||
runner.manager.prepare_store.side_effect = prepare_store
|
||||
|
||||
runner.new_request(token_ids=[0] * (block_size - 1))
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0,))
|
||||
|
||||
req_id = str(runner.req_id)
|
||||
assert calls.count(("on_request_finished", req_id)) == 1, calls
|
||||
assert calls.count(("stored_after_finish", req_id)) == 1, calls
|
||||
|
||||
finished_idx = calls.index(("on_request_finished", req_id))
|
||||
prepare_indices = [i for i, c in enumerate(calls) if c == ("prepare_store", req_id)]
|
||||
assert prepare_indices, calls
|
||||
assert finished_idx > max(prepare_indices), calls
|
||||
store_indices = [
|
||||
i
|
||||
for i, c in enumerate(calls)
|
||||
if c in (("prepare_store", req_id), ("stored_after_finish", req_id))
|
||||
]
|
||||
assert finished_idx > max(store_indices), calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
@@ -1156,10 +1163,9 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 6
|
||||
|
||||
# EOS fills the 7th block (offset 6). The extra schedule step processes
|
||||
# finished_req_ids and stores block 6 for both groups before the request's
|
||||
# GPU blocks are freed.
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=(6,))
|
||||
# EOS lands in the last slot of the 7th block (offset 6). No forward pass
|
||||
# writes that slot, so the finishing step declines the block.
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
|
||||
|
||||
@@ -1384,7 +1384,10 @@ class OffloadingConnectorScheduler:
|
||||
if req.status is RequestStatus.FINISHED_ABORTED:
|
||||
num_tokens_after_batch = req.num_computed_tokens
|
||||
elif req.is_finished():
|
||||
num_tokens_after_batch = req.num_tokens
|
||||
# Clamp to the GPU prefix cache's commit point. The final sampled
|
||||
# token's slot is never committed (under spec decode it holds a
|
||||
# rejected draft's KV), so a block ending there must not be stored.
|
||||
num_tokens_after_batch = max(req.num_prompt_tokens, req.num_tokens - 1)
|
||||
else:
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
|
||||
|
||||
Reference in New Issue
Block a user