[KVConnector] OffloadingConnector: Fix bug in handling of preemptions (#29870)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -3112,6 +3112,11 @@ class GPUModelRunner(
|
||||
"after execute_model() returns None."
|
||||
)
|
||||
|
||||
if scheduler_output.preempted_req_ids and has_kv_transfer_group():
|
||||
get_kv_transfer_group().handle_preemptions(
|
||||
scheduler_output.preempted_req_ids
|
||||
)
|
||||
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
with (
|
||||
record_function_or_nullcontext("gpu_model_runner: preprocess"),
|
||||
|
||||
Reference in New Issue
Block a user