[CI/Build] Warm the DP engines before measuring load balance in test_load (#60241)
Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
@@ -132,6 +132,26 @@ async def test_load(
|
||||
)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
# The first request an engine serves pays a few hundred milliseconds of
|
||||
# one-off setup, and while it does the engine holds the requests already
|
||||
# routed to it while reporting itself idle, which is enough to skew the
|
||||
# split asserted on below. Pay it on every engine first, then measure.
|
||||
await asyncio.gather(
|
||||
*(
|
||||
generate(
|
||||
engine,
|
||||
f"warmup-{rank}",
|
||||
prompt,
|
||||
output_kind,
|
||||
max_tokens=1,
|
||||
data_parallel_rank=rank,
|
||||
)
|
||||
for rank in range(DP_SIZE)
|
||||
)
|
||||
)
|
||||
for sl in stats_loggers.values():
|
||||
sl.finished_req_count = 0
|
||||
|
||||
NUM_REQUESTS = 100
|
||||
NUM_EXPECTED_TOKENS = 10
|
||||
|
||||
|
||||
Reference in New Issue
Block a user