[Bugfix][Core] Keep resumable sessions that stop while preempted (#58778)

Signed-off-by: David Cheung <d7cheung@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
David Cheung
2026-10-17 09:08:14 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Yifan Zong
parent 2067297b2f
commit f98ae6a9fd
4 changed files with 102 additions and 9 deletions
+14 -9
View File
@@ -1732,14 +1732,14 @@ class Scheduler(SchedulerInterface):
Discards the last sampled output token from the prior input chunk.
"""
# Current streaming input behaviour: Keep only computed output tokens
# (discard final sampled output token).
num_computed_tokens = session.num_computed_tokens
# Keep all output tokens but the final sampled one.
keep_end = session.num_tokens - 1
assert keep_end >= session.num_computed_tokens
kept_output_tokens = session._all_token_ids[
session.num_prompt_tokens : num_computed_tokens
session.num_prompt_tokens : keep_end
]
del session._all_token_ids[num_computed_tokens:]
del session.block_hashes[num_computed_tokens // self.hash_block_size :]
del session._all_token_ids[keep_end:]
del session.block_hashes[keep_end // self.hash_block_size :]
session._output_token_ids.clear()
assert session.prompt_token_ids is not None
# Extend prompt with kept output tokens.
@@ -2155,6 +2155,8 @@ class Scheduler(SchedulerInterface):
# to avoid expensive operations inside the loop.
stopped_running_reqs: set[Request] = set()
stopped_preempted_reqs: set[Request] = set()
# Stopped resumable requests to re-enqueue, in output order.
continuing_reqs: list[Request] = []
for req_id, num_tokens_scheduled in num_scheduled_tokens.items():
assert num_tokens_scheduled > 0
request = self.requests.get(req_id)
@@ -2320,6 +2322,8 @@ class Scheduler(SchedulerInterface):
kv_transfer_params["remote_prefill_cached_tokens"] = (
prefill_stats.num_cached_tokens
)
else:
continuing_reqs.append(request)
if status_before_stop == RequestStatus.RUNNING:
stopped_running_reqs.add(request)
@@ -2371,14 +2375,16 @@ class Scheduler(SchedulerInterface):
assert not prompt_logprobs_tensors
assert prompt_token_id_logprobs is None
# Remove the stopped requests from the running and waiting queues.
# Remove stopped requests from their queues, then enqueue the ones
# that continue, so each ends up in exactly one queue.
if stopped_running_reqs:
self.running = remove_all(self.running, stopped_running_reqs)
if stopped_preempted_reqs:
# This is a rare case and unlikely to impact performance.
self.waiting.remove_requests(stopped_preempted_reqs)
self.kv_holding_waiting.remove_requests(stopped_preempted_reqs)
self.deferred_waiting.difference_update(stopped_preempted_reqs)
for request in continuing_reqs:
self._enqueue_waiting_request(request)
error_req_ids = set(self.grammar_compile_error_reqs)
self.grammar_compile_error_reqs.clear()
@@ -2568,7 +2574,6 @@ class Scheduler(SchedulerInterface):
request.status = RequestStatus.WAITING_FOR_STREAMING_REQ
self.num_waiting_for_streaming_input += 1
self._enqueue_waiting_request(request)
return False
def _update_request_with_output(
+8
View File
@@ -599,6 +599,7 @@ class AsyncLLM(EngineClient):
session_id: str | None = None,
kv_hints: KvHintsEnvelope | None = None,
) -> RequestOutputCollector:
self._validate_resumable_request()
self._validate_streaming_input_sampling_params(sampling_params)
inputs = dict(
@@ -693,6 +694,13 @@ class AsyncLLM(EngineClient):
queue._input_stream_task = asyncio.create_task(handle_inputs())
return queue
def _validate_resumable_request(self) -> None:
if self.vllm_config.is_mm_encoder_only or self.model_config.is_diffusion:
raise VLLMValidationError(
"Resumable requests are not supported for encoder-only or "
"diffusion models."
)
@staticmethod
def _validate_streaming_input_sampling_params(
params: SamplingParams | PoolingParams,