[Feature] Fully support for async scheduling + PP, 30.8% E2E throughput improvement, 31.8% TPOT improvement (#32618)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -42,6 +42,22 @@ def test_compile_config_repr_succeeds():
|
||||
assert "inductor_passes" in val
|
||||
|
||||
|
||||
def test_async_scheduling_with_pipeline_parallelism_is_allowed():
|
||||
cfg = VllmConfig(
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=8192,
|
||||
is_encoder_decoder=False,
|
||||
async_scheduling=True,
|
||||
),
|
||||
parallel_config=ParallelConfig(
|
||||
pipeline_parallel_size=2,
|
||||
distributed_executor_backend="mp",
|
||||
nnodes=2,
|
||||
),
|
||||
)
|
||||
assert cfg.scheduler_config.async_scheduling is True
|
||||
|
||||
|
||||
@dataclass
|
||||
class _TestConfigFields:
|
||||
a: int
|
||||
|
||||
@@ -127,6 +127,21 @@ def test_schedule_multimodal_requests():
|
||||
assert len(encoder_input) == 1
|
||||
|
||||
|
||||
def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
|
||||
"""Async scheduling + PP: allow multi-step in-flight scheduling per request"""
|
||||
scheduler = create_scheduler(async_scheduling=True, pipeline_parallel_size=2)
|
||||
(req,) = create_requests(num_requests=1, num_tokens=8)
|
||||
scheduler.add_request(req)
|
||||
|
||||
_ = scheduler.schedule()
|
||||
assert req.num_output_placeholders > 0
|
||||
|
||||
# before any update_from_output, we still expect the request can be
|
||||
# scheduled again (multi-step in-flight).
|
||||
output = scheduler.schedule()
|
||||
assert req.request_id in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_schedule_partial_requests():
|
||||
"""Test scheduling behavior with partial requests.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user