Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -13,7 +13,7 @@ from vllm.v1.request import RequestStatus
|
||||
from vllm.v1.structured_output import StructuredOutputGrammar
|
||||
from vllm.v1.utils import ConstantList
|
||||
|
||||
from .utils import create_requests, create_scheduler, mock_kv
|
||||
from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -766,3 +766,48 @@ def test_kv_pressure_preempt_mid_handoff(kv_role: str, defer_free: bool):
|
||||
else:
|
||||
assert handoff.is_finished()
|
||||
assert handoff.num_output_tokens == 1
|
||||
|
||||
|
||||
def test_resumable_request_handoff():
|
||||
"""New streaming input resumes with the correct context and stale results
|
||||
are ignored.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
async_scheduling=True,
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
engine = PipelinedEngine(scheduler, queue_size=3)
|
||||
engine._next_token = EOS_TOKEN_ID
|
||||
|
||||
old_prompt_len = 10
|
||||
continuation_len = 12
|
||||
request_id = "resumable-race"
|
||||
|
||||
request = create_requests(
|
||||
1, num_tokens=old_prompt_len, max_tokens=4, req_ids=[request_id]
|
||||
)[0]
|
||||
request.resumable = True
|
||||
continuation = create_requests(
|
||||
1, num_tokens=continuation_len, max_tokens=4, req_ids=[request_id]
|
||||
)[0]
|
||||
continuation.resumable = True
|
||||
|
||||
scheduler.add_request(request)
|
||||
|
||||
for _ in range(3):
|
||||
assert engine._schedule()
|
||||
|
||||
engine._process_oldest_step()
|
||||
scheduler.add_request(continuation)
|
||||
assert engine._schedule()
|
||||
new_step, _ = engine.queue[0]
|
||||
assert new_step.scheduled_new_reqs[0].num_computed_tokens == old_prompt_len
|
||||
|
||||
all_token_ids_before_stale = list(request.all_token_ids)
|
||||
for _ in range(2):
|
||||
engine._process_oldest_step()
|
||||
assert list(request.output_token_ids) == []
|
||||
assert list(request.all_token_ids) == all_token_ids_before_stale
|
||||
|
||||
engine._process_oldest_step()
|
||||
assert len(request.output_token_ids) == 1
|
||||
|
||||
@@ -2273,7 +2273,8 @@ def test_spec_decode_padding_skipped_with_prefill_in_batch():
|
||||
|
||||
|
||||
def test_scheduler_stats_waiting_queues():
|
||||
"""Test that scheduler stats correctly report waiting and skipped_waiting queues."""
|
||||
"""Test that scheduler stats correctly report capacity-bound and blocked
|
||||
waiting requests."""
|
||||
# Create scheduler with limited capacity so we can have waiting requests
|
||||
scheduler = create_scheduler(max_num_batched_tokens=100)
|
||||
|
||||
@@ -2286,18 +2287,18 @@ def test_scheduler_stats_waiting_queues():
|
||||
for request in all_requests[:3]:
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Manually add 2 more to skipped_waiting to simulate constraint-blocked
|
||||
# Add 2 more that are blocked on grammar compilation
|
||||
for request in all_requests[3:]:
|
||||
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.skipped_waiting.add_request(request)
|
||||
request.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Schedule - this will schedule 2 requests, leaving 1 in waiting
|
||||
output = scheduler.schedule()
|
||||
|
||||
# Verify: 2 scheduled, 1 still waiting on capacity, 2 blocked by constraints
|
||||
assert len(output.scheduled_new_reqs) == 2
|
||||
assert len(scheduler.waiting) == 1
|
||||
assert len(scheduler.skipped_waiting) == 2
|
||||
assert len(scheduler.waiting) == 3
|
||||
assert len(scheduler.deferred_waiting) == 2
|
||||
|
||||
# Call update_from_output() to get frontend-facing stat
|
||||
scheduled_req_ids = list(output.num_scheduled_tokens.keys())
|
||||
@@ -2395,7 +2396,7 @@ def _step_until_done(
|
||||
|
||||
|
||||
def _num_waiting_requests(scheduler: Scheduler) -> int:
|
||||
return len(scheduler.waiting) + len(scheduler.skipped_waiting)
|
||||
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
|
||||
|
||||
|
||||
def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
|
||||
@@ -2944,7 +2945,7 @@ def test_kv_connector_handles_preemption(
|
||||
if is_async:
|
||||
waiting_req_ids = [
|
||||
req.request_id
|
||||
for req in scheduler.skipped_waiting
|
||||
for req in scheduler.kv_holding_waiting
|
||||
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
]
|
||||
assert len(waiting_req_ids) == 1
|
||||
@@ -3807,8 +3808,8 @@ def test_schedule_skip_tokenizer_init_structured_output_request():
|
||||
output = scheduler.schedule()
|
||||
assert len(output.scheduled_new_reqs) == 0
|
||||
assert len(scheduler.running) == 0
|
||||
assert len(scheduler.waiting) == 0
|
||||
assert len(scheduler.skipped_waiting) == 1
|
||||
assert len(scheduler.waiting) == 1
|
||||
assert len(scheduler.deferred_waiting) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_grammar", [True, False])
|
||||
@@ -5227,7 +5228,7 @@ def test_prepend_skipped_requests_order():
|
||||
req.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.waiting.remove_requests(expected_waiting_reqs[:2])
|
||||
for req in expected_waiting_reqs[:2]:
|
||||
scheduler.skipped_waiting.add_request(req)
|
||||
scheduler.kv_holding_waiting.add_request(req)
|
||||
|
||||
# schedule step
|
||||
# expect the first 2 waiting to be skipped, the third running,
|
||||
@@ -5237,21 +5238,18 @@ def test_prepend_skipped_requests_order():
|
||||
# pop the third request which is expected to be running
|
||||
expected_waiting_reqs.pop(2)
|
||||
|
||||
# verify waiting order is preserved
|
||||
waiting_reqs = list(scheduler.skipped_waiting) + list(scheduler.waiting)
|
||||
# verify waiting order is preserved, with the KV-holding requests first
|
||||
waiting_reqs = list(scheduler.kv_holding_waiting) + list(scheduler.waiting)
|
||||
assert waiting_reqs == expected_waiting_reqs
|
||||
|
||||
|
||||
def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix():
|
||||
def test_remote_kv_promotion_drains_before_grammar_prefix():
|
||||
scheduler = create_scheduler(max_num_seqs=1)
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_num_new_matched_tokens.return_value = (0, False)
|
||||
|
||||
requests = create_requests(num_requests=4)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
|
||||
req_grammar_1, req_grammar_2, req_remote, req_tail = list(scheduler.waiting)
|
||||
req_grammar_1, req_grammar_2, req_remote, req_tail = requests
|
||||
|
||||
# simulate two structured-output grammar requests at the waiting head
|
||||
# that become ready now.
|
||||
@@ -5260,31 +5258,28 @@ def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix():
|
||||
req_grammar_2.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR
|
||||
req_grammar_2.structured_output_request = Mock(grammar=object())
|
||||
|
||||
# simulate a remote-KV request that is ready to be promoted now.
|
||||
# simulate a remote-KV request that is ready to be promoted now. Parked
|
||||
# loads hold KV blocks, so they drain ahead of the grammar requests.
|
||||
req_remote.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.waiting.remove_requests([req_grammar_1, req_grammar_2, req_remote])
|
||||
scheduler.skipped_waiting.add_request(req_grammar_1)
|
||||
scheduler.skipped_waiting.add_request(req_grammar_2)
|
||||
scheduler.skipped_waiting.add_request(req_remote)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
assert list(scheduler.kv_holding_waiting) == [req_remote]
|
||||
scheduler.finished_recving_kv_req_ids.add(req_remote.request_id)
|
||||
scheduler._update_waiting_for_remote_kv = Mock()
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert output.scheduled_new_reqs
|
||||
assert output.scheduled_new_reqs[0].req_id == req_grammar_1.request_id
|
||||
waiting_req_ids = [
|
||||
req.request_id
|
||||
for req in list(scheduler.skipped_waiting) + list(scheduler.waiting)
|
||||
]
|
||||
assert output.scheduled_new_reqs[0].req_id == req_remote.request_id
|
||||
waiting_req_ids = [req.request_id for req in scheduler.waiting]
|
||||
assert waiting_req_ids == [
|
||||
req_grammar_1.request_id,
|
||||
req_grammar_2.request_id,
|
||||
req_remote.request_id,
|
||||
req_tail.request_id,
|
||||
]
|
||||
|
||||
|
||||
def test_fcfs_mixed_skipped_waiting_types_keep_order():
|
||||
def test_fcfs_mixed_blocked_types_remote_load_drains_first():
|
||||
scheduler = create_scheduler(max_num_batched_tokens=20)
|
||||
scheduler._update_waiting_for_remote_kv = Mock()
|
||||
|
||||
@@ -5305,7 +5300,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order():
|
||||
for req in (req_grammar, req_remote, req_stream, req_regular, req_tail):
|
||||
scheduler.add_request(req)
|
||||
scheduler.schedule()
|
||||
assert list(scheduler.skipped_waiting) == [req_grammar, req_remote, req_stream]
|
||||
assert list(scheduler.waiting) == [req_grammar, req_stream, req_tail]
|
||||
assert list(scheduler.kv_holding_waiting) == [req_remote]
|
||||
|
||||
scheduler.finish_requests(req_regular.request_id, RequestStatus.FINISHED_ABORTED)
|
||||
assert not scheduler.running
|
||||
@@ -5316,8 +5312,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order():
|
||||
|
||||
second_output = scheduler.schedule()
|
||||
expected_order = [
|
||||
req_grammar.request_id,
|
||||
req_remote.request_id,
|
||||
req_grammar.request_id,
|
||||
req_stream.request_id,
|
||||
req_tail.request_id,
|
||||
]
|
||||
|
||||
@@ -105,8 +105,8 @@ def test_streaming_pause_does_not_over_admit_worker_slot():
|
||||
# still holds its slot.
|
||||
scheduler.update_from_output(out, _model_runner_output(["session"], [[STOP_TOKEN]]))
|
||||
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
# A blocked waiting status lands in skipped_waiting, not the main queue.
|
||||
assert session in scheduler.skipped_waiting
|
||||
# The paused session keeps its KV blocks, so it lands in kv_holding_waiting.
|
||||
assert session in scheduler.kv_holding_waiting
|
||||
assert scheduler.num_waiting_for_streaming_input == 1
|
||||
|
||||
# A different request arrives while the session is paused.
|
||||
|
||||
Reference in New Issue
Block a user