diff --git a/tests/v1/distributed/test_async_llm_dp.py b/tests/v1/distributed/test_async_llm_dp.py index cb23910566..ecdef1f38b 100644 --- a/tests/v1/distributed/test_async_llm_dp.py +++ b/tests/v1/distributed/test_async_llm_dp.py @@ -132,6 +132,26 @@ async def test_load( ) after.callback(engine.shutdown) + # The first request an engine serves pays a few hundred milliseconds of + # one-off setup, and while it does the engine holds the requests already + # routed to it while reporting itself idle, which is enough to skew the + # split asserted on below. Pay it on every engine first, then measure. + await asyncio.gather( + *( + generate( + engine, + f"warmup-{rank}", + prompt, + output_kind, + max_tokens=1, + data_parallel_rank=rank, + ) + for rank in range(DP_SIZE) + ) + ) + for sl in stats_loggers.values(): + sl.finished_req_count = 0 + NUM_REQUESTS = 100 NUM_EXPECTED_TOKENS = 10