From 9ce74042d368d6f985741d048ad53d6d66cace6d Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 13 May 2026 01:53:32 -0700 Subject: [PATCH] [Bugfix][SimpleCPUOffloadBackend] Dedup in-flight CPU offload stores across scheduler steps (#41289) Signed-off-by: Yifan Qiao Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .buildkite/test_areas/misc.yaml | 2 + tests/v1/simple_kv_offload/test_scheduler.py | 132 ++++++++++++++++--- vllm/v1/simple_kv_offload/manager.py | 13 +- 3 files changed, 126 insertions(+), 21 deletions(-) diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index f769c16804..ca9d3fcef8 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -79,6 +79,7 @@ steps: - tests/v1/core - tests/v1/executor - tests/v1/kv_offload + - tests/v1/simple_kv_offload - tests/v1/worker - tests/v1/kv_connector/unit - tests/v1/metrics @@ -90,6 +91,7 @@ steps: - pytest -v -s -m 'not cpu_test' v1/core - pytest -v -s v1/executor - pytest -v -s v1/kv_offload + - pytest -v -s v1/simple_kv_offload - pytest -v -s v1/worker - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit - pytest -v -s -m 'not cpu_test' v1/metrics diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index 4d685103df..d1bdaf3231 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -179,6 +179,7 @@ _req_counter = 0 def make_request( num_blocks: int = 2, request_id: str | None = None, + extra_tokens: int = 1, ) -> Request: """Create a Request with deterministic block hashes.""" global _req_counter @@ -186,13 +187,7 @@ def make_request( if request_id is None: request_id = f"req-{_req_counter}" - # Add one extra token beyond the last full block so that - # ``max_cache_hit_length = num_tokens - 1`` (see - # KVCacheManager.get_computed_blocks) does not truncate the final - # full block: ``find_longest_cache_hit`` uses - # ``max_length // block_size`` and would otherwise drop one block - # when the prompt is an exact multiple of block_size. - num_tokens = num_blocks * BLOCK_SIZE + 1 + num_tokens = num_blocks * BLOCK_SIZE + extra_tokens start = _req_counter * 10000 prompt_token_ids = list(range(start, start + num_tokens)) sampling_params = SamplingParams(max_tokens=1) @@ -386,6 +381,8 @@ def test_eager_store_and_load_roundtrip() -> None: block_hasher=req._block_hasher, ) hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) + # make_request pads num_tokens by +1 beyond the last full block, so the + # manager's max_hit_len = num_tokens - 1 cap leaves all full blocks intact. assert hit_tokens == num_blocks * BLOCK_SIZE assert is_async is True @@ -405,7 +402,44 @@ def test_eager_store_and_load_roundtrip() -> None: # --------------------------------------------------------------------------- -# Test 1b: Lazy store-and-load roundtrip +# Test 1b: Boundary — max_hit_len cap drops the last full block when the +# prompt is an exact multiple of BLOCK_SIZE. +# --------------------------------------------------------------------------- +def test_max_hit_len_cap_drops_last_full_block() -> None: + """When num_tokens is an exact multiple of BLOCK_SIZE, the manager's + ``max_hit_len = num_tokens - 1`` cap forces ``find_longest_cache_hit`` to + drop the final block (since ``max_length // block_size`` rounds down). + """ + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + + num_blocks = 2 + req = make_request(num_blocks=num_blocks, extra_tokens=0) + assert req.num_tokens == num_blocks * BLOCK_SIZE + + kv_blocks = _alloc_and_register(fix, req, num_blocks) + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + sched_out = make_scheduler_output( + {req.request_id: num_blocks * BLOCK_SIZE}, + new_reqs={req.request_id: kv_blocks.get_block_ids()}, + ) + meta = sched.build_connector_meta(sched_out) + simulate_store_completion(sched, meta.store_event) + + req2 = Request( + request_id="req-cap-boundary", + prompt_token_ids=req.prompt_token_ids, + sampling_params=req.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=req._block_hasher, + ) + hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) + assert hit_tokens == (num_blocks - 1) * BLOCK_SIZE + + +# --------------------------------------------------------------------------- +# Test 1c: Lazy store-and-load roundtrip # --------------------------------------------------------------------------- def _flush_old_blocks_to_lru_head( gpu_pool: BlockPool, @@ -469,8 +503,11 @@ def test_lazy_store_and_load_roundtrip() -> None: hit_tokens, is_async = sched.get_num_new_matched_tokens( req_old2, num_computed_tokens=0 ) - assert hit_tokens == num_blocks * BLOCK_SIZE, ( - f"Expected {num_blocks * BLOCK_SIZE} hit tokens, got {hit_tokens}" + # make_request pads num_tokens by +1 beyond the last full block, so the + # manager's max_hit_len = num_tokens - 1 cap leaves all full blocks intact. + expected_hit = num_blocks * BLOCK_SIZE + assert hit_tokens == expected_hit, ( + f"Expected {expected_hit} hit tokens, got {hit_tokens}" ) assert is_async is True @@ -537,7 +574,72 @@ def test_eager_duplicate_store_skipped() -> None: # --------------------------------------------------------------------------- -# Test 2b: Lazy duplicate store is skipped +# Test 2b: Eager dedup of in-flight stores across consecutive steps +# --------------------------------------------------------------------------- +def test_eager_in_flight_store_dedup_across_steps() -> None: + """Eager: a second request sharing a prefix with an in-flight store + must not re-offload the same GPU blocks before completion lands. + + Simulates a GPU prefix-cache hit by reusing the first request's + GPU block IDs in the second scheduler step, which is the path the + real scheduler takes when two requests share a prefix. + """ + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + + num_blocks = 2 + req = make_request(num_blocks=num_blocks) + + kv_blocks = _alloc_and_register(fix, req, num_blocks) + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + block_ids = kv_blocks.get_block_ids() + sched_out = make_scheduler_output( + {req.request_id: num_blocks * BLOCK_SIZE}, + new_reqs={req.request_id: block_ids}, + ) + + meta1 = sched.build_connector_meta(sched_out) + assert meta1.store_event >= 0 + assert len(meta1.store_cpu_blocks) == num_blocks + # In-flight set tracks the scheduled GPU blocks until completion. + assert sched._in_flight_store_gpu_blocks == set(meta1.store_gpu_blocks) + cpu_free_after_first = get_cpu_free_blocks(sched) + + # Second request shares the prefix and reuses the same GPU block IDs + # (the real scheduler path: GPU prefix cache returns the same blocks). + # Do NOT simulate completion — the first store is still in-flight. + req2 = Request( + request_id="req-dup-eager-inflight", + prompt_token_ids=req.prompt_token_ids, + sampling_params=req.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=req._block_hasher, + ) + req2.num_computed_tokens = num_blocks * BLOCK_SIZE + sched.update_state_after_alloc(req2, kv_blocks, num_external_tokens=0) + sched_out2 = make_scheduler_output( + {req2.request_id: num_blocks * BLOCK_SIZE}, + new_reqs={req2.request_id: block_ids}, + ) + + meta2 = sched.build_connector_meta(sched_out2) + if meta2.store_event >= 0: + assert len(meta2.store_cpu_blocks) == 0, ( + "Expected no new CPU blocks for in-flight duplicate hashes" + ) + assert get_cpu_free_blocks(sched) == cpu_free_after_first, ( + "Second request should not consume CPU blocks while the first " + "store is still in-flight" + ) + + # After completion, the in-flight set is cleared. + simulate_store_completion(sched, meta1.store_event) + assert sched._in_flight_store_gpu_blocks == set() + + +# --------------------------------------------------------------------------- +# Test 2c: Lazy duplicate store is skipped # --------------------------------------------------------------------------- def test_lazy_duplicate_store_skipped() -> None: """Lazy: blocks already offloaded to CPU should not be offloaded again. @@ -1092,11 +1194,9 @@ def test_partial_gpu_prefix_plus_cpu_load() -> None: hit_tokens, is_async = sched.get_num_new_matched_tokens( req2, num_computed_tokens=gpu_local_computed ) - # CPU should hit blocks 2,3 (not 4,5 — those are beyond the CPU range). - num_cpu_hit_blocks = 2 - # Actually CPU has all 6 stored; it returns hits starting from position 2. - # The number of CPU hit blocks = min(remaining request blocks, CPU cached). - # Here remaining = 6 - 2 = 4 blocks are in CPU, so hit = 4 * BLOCK_SIZE. + # CPU has all 6 blocks stored. make_request pads num_tokens by +1, so + # the manager's num_tokens - 1 cap leaves all full blocks intact: + # remaining hashable range = 6 - 2 = 4 blocks, all hit. num_cpu_hit_blocks = 4 assert hit_tokens == num_cpu_hit_blocks * BLOCK_SIZE, ( f"Expected {num_cpu_hit_blocks * BLOCK_SIZE} CPU hit tokens, got {hit_tokens}" diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 846526e5be..0d06460b30 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -146,6 +146,7 @@ class SimpleCPUOffloadScheduler: # Eager mode only self._reqs_to_store: dict[str, StoreRequestState] = {} self._store_event_to_reqs: dict[int, list[str]] = {} + self._in_flight_store_gpu_blocks: set[int] = set() # Event counters self._load_event_counter: int = 0 @@ -468,7 +469,8 @@ class SimpleCPUOffloadScheduler: num_free = cpu_block_pool.get_num_free_blocks() kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups num_groups = len(kv_cache_groups) - gpu_blocks_this_step: set[int] = set() + # Dedup against blocks already scheduled. + in_flight = self._in_flight_store_gpu_blocks for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output): state = self._reqs_to_store.get(req_id) @@ -523,10 +525,9 @@ class SimpleCPUOffloadScheduler: if bhash_with_group is None: break - # Check if this group's data is already scheduled for store - # in this step or already cached in CPU. + # Skip if already scheduled for store or already cached in CPU. if ( - gpu_block_id in gpu_blocks_this_step + gpu_block_id in in_flight or cpu_block_pool.cached_block_hash_to_block.get_one_block( bhash_with_group ) @@ -561,7 +562,7 @@ class SimpleCPUOffloadScheduler: req_ids.append(req_id) merged_gpu_block_ids.extend(gpu_block_ids) merged_cpu_block_ids.extend(cpu_block_ids) - gpu_blocks_this_step.update(gpu_block_ids) + in_flight.update(gpu_block_ids) # Touch GPU blocks to prevent freeing during async copy gpu_block_pool.touch( @@ -608,6 +609,8 @@ class SimpleCPUOffloadScheduler: def _process_store_event(self, event_idx: int) -> None: """Process a fully-completed store event.""" transfer = self._store_event_to_blocks.pop(event_idx) + if not self._lazy_mode: + self._in_flight_store_gpu_blocks.difference_update(transfer.gpu_block_ids) self._process_store_completion(transfer.gpu_block_ids, transfer.cpu_block_ids) logger.debug( "Store event %d completed: cached %d blocks to CPU",