[BugFix] Fix num_output_placeholders preemption underflow (#48245)

Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Chris Eastwood <chris.eastwood@pwn4g3.dev>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Nick Hill
2026-07-29 06:41:54 -07:00
committed by GitHub
co-authored by Chris Eastwood Yifan Qiao
parent 43eaefba5a
commit a0c092ee72
5 changed files with 385 additions and 77 deletions
+16 -1
View File
@@ -1107,7 +1107,7 @@ def test_preemption_re_records_prefix_cache_query():
request = create_requests(num_requests=1)[0]
scheduler.add_request(request)
scheduler.schedule()
scheduler_output = scheduler.schedule()
stats = scheduler.kv_cache_manager.prefix_cache_stats
assert stats is not None
assert (stats.requests, stats.preempted_requests) == (1, 0)
@@ -1116,6 +1116,21 @@ def test_preemption_re_records_prefix_cache_query():
scheduler._preempt_request(request, 0.0)
assert request.status == RequestStatus.PREEMPTED
scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[1000]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
assert request.num_stale_output_tokens == 0
stats = scheduler.kv_cache_manager.prefix_cache_stats
assert stats is not None
scheduler.schedule()
assert request.status == RequestStatus.RUNNING
assert stats.preempted_requests == 1