[KVConnector] Scheduler: Fix num_computed_tokens after async KV load (#34616)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -121,7 +121,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
|
||||
|
||||
assert len(fail_scheduler.waiting) == 1
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert request.num_computed_tokens == 0
|
||||
assert request.num_computed_tokens == num_external_computed_tokens
|
||||
|
||||
(req_block_ids,) = fail_scheduler.kv_cache_manager.get_block_ids(request.request_id)
|
||||
invalid_block_ids = {req_block_ids[invalid_block_idx]}
|
||||
|
||||
@@ -339,7 +339,7 @@ def test_async_recompute_blocks_not_cached_when_invalid(
|
||||
# request should be waiting for remote KVs
|
||||
assert len(recompute_scheduler.waiting) == 1
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert request.num_computed_tokens == 0
|
||||
assert request.num_computed_tokens == num_external_computed_tokens
|
||||
|
||||
# get the allocated block IDs
|
||||
(req_block_ids,) = recompute_scheduler.kv_cache_manager.get_block_ids(
|
||||
|
||||
@@ -78,7 +78,7 @@ def test_async_load_failure(
|
||||
|
||||
assert len(scheduler.waiting) == 3
|
||||
for request in scheduler.waiting:
|
||||
assert request.num_computed_tokens == 0
|
||||
assert request.num_computed_tokens == num_external_computed_tokens
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert scheduler.connector.get_num_new_matched_tokens.call_count == 3
|
||||
|
||||
@@ -103,7 +103,7 @@ def test_async_load_failure(
|
||||
min_invalid_block_idx * scheduler.block_size
|
||||
)
|
||||
else:
|
||||
assert request.num_computed_tokens == 0
|
||||
assert request.num_computed_tokens == num_external_computed_tokens
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert scheduler.failed_recving_kv_req_ids == {request2.request_id}
|
||||
assert scheduler.connector.get_num_new_matched_tokens.call_count == 3
|
||||
@@ -305,7 +305,7 @@ def test_async_progressive_load_failure(
|
||||
|
||||
assert len(scheduler.waiting) == 1
|
||||
assert scheduler.waiting.peek_request().request_id == request.request_id
|
||||
assert request.num_computed_tokens == 0
|
||||
assert request.num_computed_tokens == num_external_computed_tokens
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert scheduler.connector.get_num_new_matched_tokens.call_count == 1
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ def test_basic_lifecycle():
|
||||
assert len(scheduler.waiting) == 1
|
||||
assert request in scheduler.waiting
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert request.num_computed_tokens == 0
|
||||
assert request.num_computed_tokens == NUM_TOKENS
|
||||
|
||||
# ... but should have (uncached) blocks allocated to it.
|
||||
block_pool = scheduler.kv_cache_manager.block_pool
|
||||
|
||||
Reference in New Issue
Block a user