[BugFix] Handle unscheduled requests properly when async scheduling (#27756)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -706,7 +706,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
req_state = self.requests[req_id]
|
||||
num_computed_tokens = req_data.num_computed_tokens[i]
|
||||
new_block_ids = req_data.new_block_ids[i]
|
||||
resumed_from_preemption = req_data.resumed_from_preemption[i]
|
||||
resumed_from_preemption = req_id in req_data.resumed_req_ids
|
||||
num_output_tokens = req_data.num_output_tokens[i]
|
||||
|
||||
# Update the cached states.
|
||||
@@ -754,16 +754,17 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
# Replace the existing block IDs with the new ones.
|
||||
req_state.block_ids = new_block_ids
|
||||
|
||||
if self.use_async_scheduling and num_output_tokens > 0:
|
||||
# We must recover the output token ids for resumed requests in the
|
||||
# async scheduling case, so that correct input_ids are obtained.
|
||||
resumed_token_ids = req_data.resumed_req_token_ids[i]
|
||||
assert resumed_token_ids is not None
|
||||
req_state.output_token_ids = resumed_token_ids[-num_output_tokens:]
|
||||
if req_index is None:
|
||||
# The request is not in the persistent batch.
|
||||
# The request was either preempted and resumed later, or was not
|
||||
# scheduled in the previous step and needs to be added again.
|
||||
|
||||
if self.use_async_scheduling and num_output_tokens > 0:
|
||||
# We must recover the output token ids for resumed requests in the
|
||||
# async scheduling case, so that correct input_ids are obtained.
|
||||
resumed_token_ids = req_data.all_token_ids[req_id]
|
||||
req_state.output_token_ids = resumed_token_ids[-num_output_tokens:]
|
||||
|
||||
reqs_to_add.append(req_state)
|
||||
continue
|
||||
|
||||
|
||||
Reference in New Issue
Block a user