[BugFix] Handle unscheduled requests properly when async scheduling (#27756)

Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Nick Hill
2025-10-29 21:04:25 -07:00
committed by GitHub
parent b5bae42f91
commit 2ce5c5d3d6
9 changed files with 63 additions and 43 deletions
@@ -494,5 +494,5 @@ def yield_req_data(
yield from zip(
cached_reqs.req_ids,
cached_reqs.new_block_ids,
cached_reqs.resumed_from_preemption,
(req_id in cached_reqs.resumed_req_ids for req_id in cached_reqs.req_ids),
)
@@ -415,10 +415,10 @@ class P2pNcclConnector(KVConnectorBase_V1):
for i, req_id in enumerate(cached_reqs.req_ids):
num_computed_tokens = cached_reqs.num_computed_tokens[i]
new_block_ids = cached_reqs.new_block_ids[i]
resumed_from_preemption = cached_reqs.resumed_from_preemption[i]
resumed_from_preemption = req_id in cached_reqs.resumed_req_ids
if self.is_producer:
num_scheduled_tokens = (scheduler_output.num_scheduled_tokens)[req_id]
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
num_tokens = num_scheduled_tokens + num_computed_tokens
assert req_id in self.chunked_prefill
assert new_block_ids is not None
@@ -336,7 +336,7 @@ class SharedStorageConnector(KVConnectorBase_V1):
cached_reqs = scheduler_output.scheduled_cached_reqs
for i, req_id in enumerate(cached_reqs.req_ids):
resumed_from_preemption = cached_reqs.resumed_from_preemption[i]
resumed_from_preemption = req_id in cached_reqs.resumed_req_ids
if not resumed_from_preemption or req_id not in self._requests_need_load:
continue