[Core] Scheduling optimization 2 (#4280)

This commit is contained in:
SangBin Cho
2024-04-23 08:02:11 +00:00
committed by GitHub
parent 8f2ea22bde
commit 050f285ff6
3 changed files with 15 additions and 3 deletions
+2 -1
View File
@@ -563,7 +563,8 @@ def test_decode_schedule_preempted():
assert len(output.preempted) == 2
# Verify budgets are updated.
assert budget.num_batched_tokens == 1
assert budget.num_curr_seqs == 1
# NOTE: When enable_chunk is False, num_seqs budget is not updated.
# assert budget.num_curr_seqs == 1
# Both should be preempted, not swapped.
assert output.blocks_to_swap_out == {}
# Nothing is copied.