diff --git a/tests/v1/core/test_worker_slot_overflow.py b/tests/v1/core/test_worker_slot_overflow.py new file mode 100644 index 0000000000..79366b0a23 --- /dev/null +++ b/tests/v1/core/test_worker_slot_overflow.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for the V2 model runner "No free indices" assertion. + +The V2 model runner stores per-request state in a fixed-size slab sized to +``max_num_seqs`` (``RequestState.free_indices`` in +``vllm/v1/worker/gpu/states.py``). A slot is occupied from the moment a request +appears in ``scheduled_new_reqs`` (worker ``add_requests``) until it appears in +``finished_req_ids``/``preempted_req_ids`` (worker ``finish_requests``), with a +defensive same-id ``_remove_request`` on re-add. If the scheduler ever lets the +number of slot-holding requests exceed ``max_num_seqs``, ``add_request`` trips:: + + assert len(self.free_indices) > 0, "No free indices" + +The invariant the scheduler must preserve is: *every request that still holds a +worker slot is counted against the admission limit*. These tests exercise two +ways that invariant can break, by replaying real scheduler outputs through a +faithful model of the worker's slot accounting. +""" + +import pytest + +from vllm.v1.outputs import ModelRunnerOutput +from vllm.v1.request import RequestStatus + +from .test_scheduler import create_scheduler_with_priority +from .utils import create_requests, create_scheduler + +pytestmark = pytest.mark.cpu_test + + +class WorkerSlots: + """Faithful model of the V2 model runner's request-slot accounting. + + Mirrors ``GPUModelRunner.finish_requests`` + ``add_requests`` and + ``RequestState.add_request``/``remove_request`` (the ``free_indices`` pool). + ``apply`` raises ``AssertionError("No free indices")`` exactly where the real + worker would, so a scheduler over-admission surfaces as the same failure. + """ + + def __init__(self, max_num_reqs: int): + self.max_num_reqs = max_num_reqs + self.occupied: set[str] = set() + + def apply(self, scheduler_output) -> None: + # finish_requests: free finished and preempted slots first. + freed = set(scheduler_output.finished_req_ids) + freed |= set(scheduler_output.preempted_req_ids) + self.occupied -= freed + + # update_requests: cached reqs must already own a slot, else the + # worker's req_id_to_index lookup would KeyError. + for req_id in scheduler_output.scheduled_cached_reqs.req_ids: + assert req_id in self.occupied, f"cached req {req_id!r} has no worker slot" + + # add_requests: new (and, under V2, resumed) reqs claim a slot. The + # same-id _remove_request guard frees a stale slot for the same id. + for new_req in scheduler_output.scheduled_new_reqs: + self.occupied.discard(new_req.req_id) + assert len(self.occupied) < self.max_num_reqs, "No free indices" + self.occupied.add(new_req.req_id) + + +def _model_runner_output(req_ids: list[str], sampled: list[list[int]]): + return ModelRunnerOutput( + req_ids=req_ids, + req_id_to_index={req_id: i for i, req_id in enumerate(req_ids)}, + sampled_token_ids=sampled, + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + ) + + +def test_streaming_pause_does_not_over_admit_worker_slot(): + """A paused resumable session keeps its worker slot; with max_num_seqs=1 a + new request must not be admitted on top of it. + + Repro of the "No free indices" crash: the session pauses into + WAITING_FOR_STREAMING_REQ (removed from ``running`` but never reported as + finished/preempted, so the worker keeps its slot), then a different request + is admitted into the single slot. + """ + STOP_TOKEN = 7 + scheduler = create_scheduler(max_num_seqs=1, use_v2_model_runner=True) + slots = WorkerSlots(max_num_reqs=1) + + # A resumable streaming session. + (session,) = create_requests( + num_requests=1, + num_tokens=4, + req_ids=["session"], + stop_token_ids=[STOP_TOKEN], + max_tokens=16, + ) + session.resumable = True + scheduler.add_request(session) + + out = scheduler.schedule() + slots.apply(out) + assert [r.req_id for r in out.scheduled_new_reqs] == ["session"] + + # The session emits its stop token and pauses, waiting for the next input + # chunk. It leaves `running` but is NOT finished/preempted, so the worker + # still holds its slot. + scheduler.update_from_output(out, _model_runner_output(["session"], [[STOP_TOKEN]])) + assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ + # A blocked waiting status lands in skipped_waiting, not the main queue. + assert session in scheduler.skipped_waiting + assert scheduler.num_waiting_for_streaming_input == 1 + + # A different request arrives while the session is paused. + (other,) = create_requests( + num_requests=1, num_tokens=4, req_ids=["other"], max_tokens=16 + ) + scheduler.add_request(other) + + out = scheduler.schedule() + # Before the fix this admits `other` as a new request while the session's + # slot is still held -> WorkerSlots.apply raises "No free indices". + slots.apply(out) + + # After the fix: `other` is deferred until the session resumes or finishes. + assert "other" not in [r.req_id for r in out.scheduled_new_reqs] + assert other in scheduler.waiting + + +def test_reset_prefix_cache_priority_does_not_over_admit_worker_slot(): + """reset_prefix_cache(reset_running_requests=True) force-preempts the running + request out-of-band; under priority scheduling a higher-priority newcomer + jumps ahead of the preempted request and would over-admit the single worker + slot unless the preemption is reported to the worker. + + The fix buffers the force-preemption and reports it in the next scheduler + output's preempted_req_ids, so the worker frees the slot before the newcomer + is admitted. + """ + scheduler = create_scheduler_with_priority( + max_num_seqs=1, enable_prefix_caching=True, use_v2_model_runner=True + ) + slots = WorkerSlots(max_num_reqs=1) + + # Low-priority request A (larger priority value == lower priority). + (req_a,) = create_requests( + num_requests=1, num_tokens=4, req_ids=["A"], ignore_eos=True, max_tokens=100 + ) + req_a.priority = 10 + scheduler.add_request(req_a) + + out = scheduler.schedule() + slots.apply(out) + assert [r.req_id for r in out.scheduled_new_reqs] == ["A"] + + scheduler.update_from_output(out, _model_runner_output(["A"], [[42]])) + assert req_a.status == RequestStatus.RUNNING + assert req_a in scheduler.running + + # Force a prefix-cache reset that preempts A out-of-band. A goes back to the + # waiting queue (PREEMPTED); the preemption is buffered for the next output. + assert scheduler.reset_prefix_cache(reset_running_requests=True) + assert req_a.status == RequestStatus.PREEMPTED + + # A higher-priority request B arrives and outranks the preempted A. + (req_b,) = create_requests( + num_requests=1, num_tokens=4, req_ids=["B"], ignore_eos=True, max_tokens=100 + ) + req_b.priority = 0 + scheduler.add_request(req_b) + + out = scheduler.schedule() + # The fix reports A's force-preemption here, so the worker frees A's slot + # before B is admitted -> no overflow. + assert "A" in out.preempted_req_ids + assert "B" in [r.req_id for r in out.scheduled_new_reqs] + slots.apply(out) + assert slots.occupied == {"B"} + + +def test_reset_prefix_cache_same_step_resume_purges_then_re_adds(): + """When a force-preempted request resumes in the SAME step (FCFS, no + competing request), it appears in both preempted_req_ids and (under V2) + scheduled_new_reqs. The worker runs finish_requests before add_requests, so + the request's slot is purged and then cleanly re-added. + """ + scheduler = create_scheduler(max_num_seqs=1, use_v2_model_runner=True) + slots = WorkerSlots(max_num_reqs=1) + + (req_a,) = create_requests( + num_requests=1, num_tokens=4, req_ids=["A"], ignore_eos=True, max_tokens=100 + ) + scheduler.add_request(req_a) + + out = scheduler.schedule() + slots.apply(out) + scheduler.update_from_output(out, _model_runner_output(["A"], [[42]])) + assert req_a in scheduler.running + + assert scheduler.reset_prefix_cache(reset_running_requests=True) + assert req_a.status == RequestStatus.PREEMPTED + # Reset invalidates A's computed tokens; it will re-prefill from scratch. + assert req_a.num_computed_tokens == 0 + + out = scheduler.schedule() + # A is force-preempted AND resumed in this one step. + assert "A" in out.preempted_req_ids + assert "A" in [r.req_id for r in out.scheduled_new_reqs] + # WorkerSlots mirrors finish-before-add: purge frees the slot, re-add fills + # it. No overflow, A still occupies its single slot afterward. + slots.apply(out) + assert slots.occupied == {"A"} diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index fe59a29e12..ea657d7e94 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -189,6 +189,9 @@ class Scheduler(SchedulerInterface): # This is flushed at the end of each scheduling step. self.finished_req_ids: set[str] = set() + # IDs of requests preempted since the last call to schedule(). + self.reset_preempted_req_ids: set[str] = set() + # Counter for requests waiting for streaming input. Used to calculate # number of unfinished requests self.num_waiting_for_streaming_input: int = 0 @@ -635,7 +638,10 @@ class Scheduler(SchedulerInterface): step_skipped_waiting = create_request_queue(self.policy) while (self.waiting or self.skipped_waiting) and token_budget > 0: - if len(self.running) == self.max_num_running_reqs: + # Paused streaming sessions (WAITING_FOR_STREAMING_REQ) are not + # in `running` but still hold a model-runner request slot. + num_running = len(self.running) + self.num_waiting_for_streaming_input + if num_running >= self.max_num_running_reqs: break request_queue = self._select_waiting_queue_for_scheduling() @@ -1093,7 +1099,7 @@ class Scheduler(SchedulerInterface): scheduled_spec_decode_tokens=scheduled_spec_decode_tokens, scheduled_encoder_inputs=scheduled_encoder_inputs, num_common_prefix_blocks=num_common_prefix_blocks, - preempted_req_ids={req.request_id for req in preempted_reqs}, + preempted_req_ids=self.reset_preempted_req_ids, # finished_req_ids is an existing state in the scheduler, # instead of being newly scheduled in this step. # It contains the request IDs that are finished in between @@ -1155,6 +1161,7 @@ class Scheduler(SchedulerInterface): # Put the request back to the waiting queue. self.waiting.prepend_request(request) + self.reset_preempted_req_ids.add(request.request_id) def _update_after_schedule(self, scheduler_output: SchedulerOutput) -> None: # Advance the number of computed tokens for the request AFTER @@ -1199,10 +1206,11 @@ class Scheduler(SchedulerInterface): } ) - # Clear the finished request IDs. - # NOTE: We shouldn't do self.finished_req_ids.clear() here because - # it will also affect the scheduler output. + # Clear the finished and preempted request IDs. + # NOTE: We shouldn't just clear() here because it will also affect + # the scheduler output. self.finished_req_ids = set() + self.reset_preempted_req_ids = set() def _update_request_as_session( self, session: Request, update: StreamingUpdate