[Feature] Fully support for async scheduling + PP, 30.8% E2E throughput improvement, 31.8% TPOT improvement (#32618)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Wentao Ye
2026-01-28 20:30:32 +00:00
committed by GitHub
co-authored by Nick Hill
parent 8bdd3979d8
commit 3e440786af
4 changed files with 103 additions and 23 deletions
+16
View File
@@ -42,6 +42,22 @@ def test_compile_config_repr_succeeds():
assert "inductor_passes" in val
def test_async_scheduling_with_pipeline_parallelism_is_allowed():
cfg = VllmConfig(
scheduler_config=SchedulerConfig(
max_model_len=8192,
is_encoder_decoder=False,
async_scheduling=True,
),
parallel_config=ParallelConfig(
pipeline_parallel_size=2,
distributed_executor_backend="mp",
nnodes=2,
),
)
assert cfg.scheduler_config.async_scheduling is True
@dataclass
class _TestConfigFields:
a: int
+15
View File
@@ -127,6 +127,21 @@ def test_schedule_multimodal_requests():
assert len(encoder_input) == 1
def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
"""Async scheduling + PP: allow multi-step in-flight scheduling per request"""
scheduler = create_scheduler(async_scheduling=True, pipeline_parallel_size=2)
(req,) = create_requests(num_requests=1, num_tokens=8)
scheduler.add_request(req)
_ = scheduler.schedule()
assert req.num_output_placeholders > 0
# before any update_from_output, we still expect the request can be
# scheduled again (multi-step in-flight).
output = scheduler.schedule()
assert req.request_id in output.num_scheduled_tokens
def test_schedule_partial_requests():
"""Test scheduling behavior with partial requests.