[V1] Eagerly remove finished requests from the batch (#14388)

Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Nick Hill
2025-03-07 10:56:00 -08:00
committed by GitHub
parent c6359e8ca6
commit 8ed5421aaa
9 changed files with 58 additions and 16 deletions
+10
View File
@@ -102,14 +102,24 @@ def test_engine_core(monkeypatch):
engine_core.add_request(req)
assert len(engine_core.scheduler.waiting) == 1
assert len(engine_core.scheduler.running) == 0
assert engine_core.scheduler.has_unfinished_requests()
assert not engine_core.scheduler.has_finished_requests()
_ = engine_core.step()
assert len(engine_core.scheduler.waiting) == 0
assert len(engine_core.scheduler.running) == 1
assert engine_core.scheduler.has_unfinished_requests()
assert not engine_core.scheduler.has_finished_requests()
engine_core.abort_requests([request_id])
assert len(engine_core.scheduler.waiting) == 0
assert len(engine_core.scheduler.running) == 0
assert not engine_core.scheduler.has_unfinished_requests()
assert engine_core.scheduler.has_finished_requests()
_ = engine_core.step()
assert not engine_core.scheduler.has_unfinished_requests()
assert not engine_core.scheduler.has_finished_requests()
# Add, step, abort 1 of the 3.
req0 = make_request()
+2 -2
View File
@@ -50,7 +50,7 @@ def loop_until_done(client: EngineCoreClient, outputs: dict):
engine_core_outputs = client.get_output().outputs
if len(engine_core_outputs) == 0:
break
continue
all_finished = True
for out in engine_core_outputs:
@@ -68,7 +68,7 @@ async def loop_until_done_async(client: EngineCoreClient, outputs: dict):
engine_core_outputs = (await client.get_output_async()).outputs
if len(engine_core_outputs) == 0:
break
continue
all_finished = True
for out in engine_core_outputs: