[Bugfix][Core] Keep resumable sessions that stop while preempted (#58778)
Signed-off-by: David Cheung <d7cheung@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Yifan Zong
parent
2067297b2f
commit
f98ae6a9fd
@@ -1732,14 +1732,14 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
Discards the last sampled output token from the prior input chunk.
|
||||
"""
|
||||
# Current streaming input behaviour: Keep only computed output tokens
|
||||
# (discard final sampled output token).
|
||||
num_computed_tokens = session.num_computed_tokens
|
||||
# Keep all output tokens but the final sampled one.
|
||||
keep_end = session.num_tokens - 1
|
||||
assert keep_end >= session.num_computed_tokens
|
||||
kept_output_tokens = session._all_token_ids[
|
||||
session.num_prompt_tokens : num_computed_tokens
|
||||
session.num_prompt_tokens : keep_end
|
||||
]
|
||||
del session._all_token_ids[num_computed_tokens:]
|
||||
del session.block_hashes[num_computed_tokens // self.hash_block_size :]
|
||||
del session._all_token_ids[keep_end:]
|
||||
del session.block_hashes[keep_end // self.hash_block_size :]
|
||||
session._output_token_ids.clear()
|
||||
assert session.prompt_token_ids is not None
|
||||
# Extend prompt with kept output tokens.
|
||||
@@ -2155,6 +2155,8 @@ class Scheduler(SchedulerInterface):
|
||||
# to avoid expensive operations inside the loop.
|
||||
stopped_running_reqs: set[Request] = set()
|
||||
stopped_preempted_reqs: set[Request] = set()
|
||||
# Stopped resumable requests to re-enqueue, in output order.
|
||||
continuing_reqs: list[Request] = []
|
||||
for req_id, num_tokens_scheduled in num_scheduled_tokens.items():
|
||||
assert num_tokens_scheduled > 0
|
||||
request = self.requests.get(req_id)
|
||||
@@ -2320,6 +2322,8 @@ class Scheduler(SchedulerInterface):
|
||||
kv_transfer_params["remote_prefill_cached_tokens"] = (
|
||||
prefill_stats.num_cached_tokens
|
||||
)
|
||||
else:
|
||||
continuing_reqs.append(request)
|
||||
|
||||
if status_before_stop == RequestStatus.RUNNING:
|
||||
stopped_running_reqs.add(request)
|
||||
@@ -2371,14 +2375,16 @@ class Scheduler(SchedulerInterface):
|
||||
assert not prompt_logprobs_tensors
|
||||
assert prompt_token_id_logprobs is None
|
||||
|
||||
# Remove the stopped requests from the running and waiting queues.
|
||||
# Remove stopped requests from their queues, then enqueue the ones
|
||||
# that continue, so each ends up in exactly one queue.
|
||||
if stopped_running_reqs:
|
||||
self.running = remove_all(self.running, stopped_running_reqs)
|
||||
if stopped_preempted_reqs:
|
||||
# This is a rare case and unlikely to impact performance.
|
||||
self.waiting.remove_requests(stopped_preempted_reqs)
|
||||
self.kv_holding_waiting.remove_requests(stopped_preempted_reqs)
|
||||
self.deferred_waiting.difference_update(stopped_preempted_reqs)
|
||||
for request in continuing_reqs:
|
||||
self._enqueue_waiting_request(request)
|
||||
|
||||
error_req_ids = set(self.grammar_compile_error_reqs)
|
||||
self.grammar_compile_error_reqs.clear()
|
||||
@@ -2568,7 +2574,6 @@ class Scheduler(SchedulerInterface):
|
||||
request.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
self.num_waiting_for_streaming_input += 1
|
||||
|
||||
self._enqueue_waiting_request(request)
|
||||
return False
|
||||
|
||||
def _update_request_with_output(
|
||||
|
||||
@@ -599,6 +599,7 @@ class AsyncLLM(EngineClient):
|
||||
session_id: str | None = None,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
) -> RequestOutputCollector:
|
||||
self._validate_resumable_request()
|
||||
self._validate_streaming_input_sampling_params(sampling_params)
|
||||
|
||||
inputs = dict(
|
||||
@@ -693,6 +694,13 @@ class AsyncLLM(EngineClient):
|
||||
queue._input_stream_task = asyncio.create_task(handle_inputs())
|
||||
return queue
|
||||
|
||||
def _validate_resumable_request(self) -> None:
|
||||
if self.vllm_config.is_mm_encoder_only or self.model_config.is_diffusion:
|
||||
raise VLLMValidationError(
|
||||
"Resumable requests are not supported for encoder-only or "
|
||||
"diffusion models."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_streaming_input_sampling_params(
|
||||
params: SamplingParams | PoolingParams,
|
||||
|
||||
Reference in New Issue
Block a user