Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-09-29 12:54:18 +07:00
264 changed files with 13095 additions and 1541 deletions
+46 -1
View File
@@ -13,7 +13,7 @@ from vllm.v1.request import RequestStatus
from vllm.v1.structured_output import StructuredOutputGrammar
from vllm.v1.utils import ConstantList
from .utils import create_requests, create_scheduler, mock_kv
from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv
pytestmark = pytest.mark.cpu_test
@@ -766,3 +766,48 @@ def test_kv_pressure_preempt_mid_handoff(kv_role: str, defer_free: bool):
else:
assert handoff.is_finished()
assert handoff.num_output_tokens == 1
def test_resumable_request_handoff():
"""New streaming input resumes with the correct context and stale results
are ignored.
"""
scheduler = create_scheduler(
async_scheduling=True,
use_v2_model_runner=True,
)
engine = PipelinedEngine(scheduler, queue_size=3)
engine._next_token = EOS_TOKEN_ID
old_prompt_len = 10
continuation_len = 12
request_id = "resumable-race"
request = create_requests(
1, num_tokens=old_prompt_len, max_tokens=4, req_ids=[request_id]
)[0]
request.resumable = True
continuation = create_requests(
1, num_tokens=continuation_len, max_tokens=4, req_ids=[request_id]
)[0]
continuation.resumable = True
scheduler.add_request(request)
for _ in range(3):
assert engine._schedule()
engine._process_oldest_step()
scheduler.add_request(continuation)
assert engine._schedule()
new_step, _ = engine.queue[0]
assert new_step.scheduled_new_reqs[0].num_computed_tokens == old_prompt_len
all_token_ids_before_stale = list(request.all_token_ids)
for _ in range(2):
engine._process_oldest_step()
assert list(request.output_token_ids) == []
assert list(request.all_token_ids) == all_token_ids_before_stale
engine._process_oldest_step()
assert len(request.output_token_ids) == 1
+28 -32
View File
@@ -2273,7 +2273,8 @@ def test_spec_decode_padding_skipped_with_prefill_in_batch():
def test_scheduler_stats_waiting_queues():
"""Test that scheduler stats correctly report waiting and skipped_waiting queues."""
"""Test that scheduler stats correctly report capacity-bound and blocked
waiting requests."""
# Create scheduler with limited capacity so we can have waiting requests
scheduler = create_scheduler(max_num_batched_tokens=100)
@@ -2286,18 +2287,18 @@ def test_scheduler_stats_waiting_queues():
for request in all_requests[:3]:
scheduler.add_request(request)
# Manually add 2 more to skipped_waiting to simulate constraint-blocked
# Add 2 more that are blocked on grammar compilation
for request in all_requests[3:]:
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.skipped_waiting.add_request(request)
request.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR
scheduler.add_request(request)
# Schedule - this will schedule 2 requests, leaving 1 in waiting
output = scheduler.schedule()
# Verify: 2 scheduled, 1 still waiting on capacity, 2 blocked by constraints
assert len(output.scheduled_new_reqs) == 2
assert len(scheduler.waiting) == 1
assert len(scheduler.skipped_waiting) == 2
assert len(scheduler.waiting) == 3
assert len(scheduler.deferred_waiting) == 2
# Call update_from_output() to get frontend-facing stat
scheduled_req_ids = list(output.num_scheduled_tokens.keys())
@@ -2395,7 +2396,7 @@ def _step_until_done(
def _num_waiting_requests(scheduler: Scheduler) -> int:
return len(scheduler.waiting) + len(scheduler.skipped_waiting)
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
@@ -2944,7 +2945,7 @@ def test_kv_connector_handles_preemption(
if is_async:
waiting_req_ids = [
req.request_id
for req in scheduler.skipped_waiting
for req in scheduler.kv_holding_waiting
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
]
assert len(waiting_req_ids) == 1
@@ -3807,8 +3808,8 @@ def test_schedule_skip_tokenizer_init_structured_output_request():
output = scheduler.schedule()
assert len(output.scheduled_new_reqs) == 0
assert len(scheduler.running) == 0
assert len(scheduler.waiting) == 0
assert len(scheduler.skipped_waiting) == 1
assert len(scheduler.waiting) == 1
assert len(scheduler.deferred_waiting) == 1
@pytest.mark.parametrize("async_grammar", [True, False])
@@ -5227,7 +5228,7 @@ def test_prepend_skipped_requests_order():
req.status = RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.waiting.remove_requests(expected_waiting_reqs[:2])
for req in expected_waiting_reqs[:2]:
scheduler.skipped_waiting.add_request(req)
scheduler.kv_holding_waiting.add_request(req)
# schedule step
# expect the first 2 waiting to be skipped, the third running,
@@ -5237,21 +5238,18 @@ def test_prepend_skipped_requests_order():
# pop the third request which is expected to be running
expected_waiting_reqs.pop(2)
# verify waiting order is preserved
waiting_reqs = list(scheduler.skipped_waiting) + list(scheduler.waiting)
# verify waiting order is preserved, with the KV-holding requests first
waiting_reqs = list(scheduler.kv_holding_waiting) + list(scheduler.waiting)
assert waiting_reqs == expected_waiting_reqs
def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix():
def test_remote_kv_promotion_drains_before_grammar_prefix():
scheduler = create_scheduler(max_num_seqs=1)
scheduler.connector = Mock()
scheduler.connector.get_num_new_matched_tokens.return_value = (0, False)
requests = create_requests(num_requests=4)
for request in requests:
scheduler.add_request(request)
req_grammar_1, req_grammar_2, req_remote, req_tail = list(scheduler.waiting)
req_grammar_1, req_grammar_2, req_remote, req_tail = requests
# simulate two structured-output grammar requests at the waiting head
# that become ready now.
@@ -5260,31 +5258,28 @@ def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix():
req_grammar_2.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR
req_grammar_2.structured_output_request = Mock(grammar=object())
# simulate a remote-KV request that is ready to be promoted now.
# simulate a remote-KV request that is ready to be promoted now. Parked
# loads hold KV blocks, so they drain ahead of the grammar requests.
req_remote.status = RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.waiting.remove_requests([req_grammar_1, req_grammar_2, req_remote])
scheduler.skipped_waiting.add_request(req_grammar_1)
scheduler.skipped_waiting.add_request(req_grammar_2)
scheduler.skipped_waiting.add_request(req_remote)
for request in requests:
scheduler.add_request(request)
assert list(scheduler.kv_holding_waiting) == [req_remote]
scheduler.finished_recving_kv_req_ids.add(req_remote.request_id)
scheduler._update_waiting_for_remote_kv = Mock()
output = scheduler.schedule()
assert output.scheduled_new_reqs
assert output.scheduled_new_reqs[0].req_id == req_grammar_1.request_id
waiting_req_ids = [
req.request_id
for req in list(scheduler.skipped_waiting) + list(scheduler.waiting)
]
assert output.scheduled_new_reqs[0].req_id == req_remote.request_id
waiting_req_ids = [req.request_id for req in scheduler.waiting]
assert waiting_req_ids == [
req_grammar_1.request_id,
req_grammar_2.request_id,
req_remote.request_id,
req_tail.request_id,
]
def test_fcfs_mixed_skipped_waiting_types_keep_order():
def test_fcfs_mixed_blocked_types_remote_load_drains_first():
scheduler = create_scheduler(max_num_batched_tokens=20)
scheduler._update_waiting_for_remote_kv = Mock()
@@ -5305,7 +5300,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order():
for req in (req_grammar, req_remote, req_stream, req_regular, req_tail):
scheduler.add_request(req)
scheduler.schedule()
assert list(scheduler.skipped_waiting) == [req_grammar, req_remote, req_stream]
assert list(scheduler.waiting) == [req_grammar, req_stream, req_tail]
assert list(scheduler.kv_holding_waiting) == [req_remote]
scheduler.finish_requests(req_regular.request_id, RequestStatus.FINISHED_ABORTED)
assert not scheduler.running
@@ -5316,8 +5312,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order():
second_output = scheduler.schedule()
expected_order = [
req_grammar.request_id,
req_remote.request_id,
req_grammar.request_id,
req_stream.request_id,
req_tail.request_id,
]
+2 -2
View File
@@ -105,8 +105,8 @@ def test_streaming_pause_does_not_over_admit_worker_slot():
# still holds its slot.
scheduler.update_from_output(out, _model_runner_output(["session"], [[STOP_TOKEN]]))
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
# A blocked waiting status lands in skipped_waiting, not the main queue.
assert session in scheduler.skipped_waiting
# The paused session keeps its KV blocks, so it lands in kv_holding_waiting.
assert session in scheduler.kv_holding_waiting
assert scheduler.num_waiting_for_streaming_input == 1
# A different request arrives while the session is paused.