From c88d3d4775c41793543e97f1609d3161a5689905 Mon Sep 17 00:00:00 2001 From: Jonathan Chen Date: Sat, 20 Jun 2026 18:01:06 -0400 Subject: [PATCH] [SimpleCPUOffloadConnector] PCP + DCP support (#39831) Signed-off-by: Jonathan Chen --- tests/v1/simple_kv_offload/test_scheduler.py | 230 +++++++++++++++++++ vllm/v1/simple_kv_offload/manager.py | 20 +- 2 files changed, 243 insertions(+), 7 deletions(-) diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index cff60ea01d..1ec986eada 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -6,6 +6,7 @@ from __future__ import annotations from dataclasses import dataclass +import pytest import torch from vllm import SamplingParams @@ -1528,3 +1529,232 @@ def test_reset_pending_loads() -> None: # All GPU blocks free num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks() assert num_used == 1, f"Expected only null block in use, got {num_used}" + + +def _make_cp_vllm_config( + dcp_world_size: int = 1, + pcp_world_size: int = 1, +) -> VllmConfig: + """VllmConfig with context-parallel sizes set for scheduler-only tests.""" + cfg = _make_vllm_config() + + cfg.parallel_config.decode_context_parallel_size = dcp_world_size + cfg.parallel_config.prefill_context_parallel_size = pcp_world_size + return cfg + + +def _make_cp_scheduler( + *, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + num_cpu_blocks: int = 8, + num_gpu_blocks: int = 16, + lazy: bool = False, +) -> SchedulerFixture: + """Build a SimpleCPUOffloadScheduler with CP-scaled virtual block size.""" + cp_world_size = dcp_world_size * pcp_world_size + virtual_block_size = BLOCK_SIZE * cp_world_size + + kv_cache_config = _make_kv_cache_config(num_gpu_blocks) + vllm_config = _make_cp_vllm_config(dcp_world_size, pcp_world_size) + cpu_capacity_bytes = _BYTES_PER_BLOCK * num_cpu_blocks + + sched = SimpleCPUOffloadScheduler( + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + cpu_capacity_bytes=cpu_capacity_bytes, + scheduler_block_size=virtual_block_size, + hash_block_size=virtual_block_size, + lazy_offload=lazy, + ) + + gpu_block_pool = BlockPool( + num_gpu_blocks=num_gpu_blocks, + enable_caching=True, + hash_block_size=virtual_block_size, + ) + sched.bind_gpu_block_pool(gpu_block_pool) + + return SchedulerFixture( + scheduler=sched, + gpu_block_pool=gpu_block_pool, + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + ) + + +def _make_cp_request( + num_blocks: int, + virtual_block_size: int, + request_id: str | None = None, +) -> Request: + """Create a request whose block hashes are computed at the virtual + (CP-scaled) block size, matching what the real scheduler does. + """ + global _req_counter + _req_counter += 1 + if request_id is None: + request_id = f"req-cp-{_req_counter}" + + num_tokens = num_blocks * virtual_block_size + 1 + start = _req_counter * 10000 + prompt_token_ids = list(range(start, start + num_tokens)) + sampling_params = SamplingParams(max_tokens=1) + + return Request( + request_id=request_id, + prompt_token_ids=prompt_token_ids, + sampling_params=sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=get_request_block_hasher(virtual_block_size, sha256), + ) + + +def _allocate_cp_gpu_blocks( + gpu_block_pool: BlockPool, + request: Request, + num_blocks: int, + virtual_block_size: int, + group_id: int = 0, +) -> list: + """Allocate GPU blocks and cache them using the CP-scaled block size.""" + blocks = gpu_block_pool.get_new_blocks(num_blocks) + num_full = min(num_blocks, len(request.block_hashes)) + if num_full > 0: + gpu_block_pool.cache_full_blocks( + request=request, + blocks=blocks, + num_cached_blocks=0, + num_full_blocks=num_full, + block_size=virtual_block_size, + kv_cache_group_id=group_id, + ) + return blocks + + +# --------------------------------------------------------------------------- +# Test 15: CP block size scaling is correct +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "dcp_world_size, pcp_world_size", + [ + (2, 1), # DCP only + (1, 2), # PCP only + (2, 2), # DCP + PCP + ], +) +def test_cp_block_size_scaling(dcp_world_size: int, pcp_world_size: int) -> None: + """Verify that the scheduler's block_size and cp_world_size are correctly + scaled when context parallelism is enabled.""" + fix = _make_cp_scheduler( + dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size + ) + sched = fix.scheduler + + expected_cp = dcp_world_size * pcp_world_size + assert sched.cp_world_size == expected_cp + assert sched.block_size == BLOCK_SIZE * expected_cp + + +# --------------------------------------------------------------------------- +# Test 16: CP eager store-and-load roundtrip +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "dcp_world_size, pcp_world_size", + [ + (2, 1), + (1, 2), + ], +) +def test_cp_eager_store_and_load_roundtrip( + dcp_world_size: int, pcp_world_size: int +) -> None: + """With CP enabled, store blocks to CPU and reload them for a new request + with matching tokens. Verifies that hash matching and transfer-pair + construction work with the virtual block size.""" + fix = _make_cp_scheduler( + dcp_world_size=dcp_world_size, + pcp_world_size=pcp_world_size, + num_cpu_blocks=8, + num_gpu_blocks=16, + lazy=False, + ) + sched = fix.scheduler + cp = dcp_world_size * pcp_world_size + vbs = BLOCK_SIZE * cp + + num_blocks = 2 + req = _make_cp_request(num_blocks, vbs) + + # Allocate GPU blocks and register hashes + gpu_blocks = _allocate_cp_gpu_blocks(fix.gpu_block_pool, req, num_blocks, vbs) + kv_blocks = KVCacheBlocks(blocks=(gpu_blocks,)) + req.num_computed_tokens = num_blocks * vbs + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + + block_ids = kv_blocks.get_block_ids() + sched_out = make_scheduler_output( + {req.request_id: num_blocks * vbs}, + new_reqs={req.request_id: block_ids}, + ) + + meta = sched.build_connector_meta(sched_out) + assert meta.store_event >= 0, "Expected a store event" + assert len(meta.store_gpu_blocks) == num_blocks + assert len(meta.store_cpu_blocks) == num_blocks + simulate_store_completion(sched, meta.store_event) + + # New request with same tokens — should get a full CPU cache hit. + req2 = Request( + request_id="req-cp-load", + prompt_token_ids=req.prompt_token_ids, + sampling_params=req.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=req._block_hasher, + ) + + hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) + assert hit_tokens == num_blocks * vbs + assert is_async is True + + # Allocate fresh GPU blocks for the load. + gpu_blocks2 = fix.gpu_block_pool.get_new_blocks(num_blocks) + kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) + sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) + + sched_out2 = make_scheduler_output( + {req2.request_id: 1}, + new_reqs={req2.request_id: kv_blocks2.get_block_ids()}, + ) + meta2 = sched.build_connector_meta(sched_out2) + assert meta2.load_event >= 0, "Expected a load event" + assert len(meta2.load_gpu_blocks) == num_blocks + assert len(meta2.load_cpu_blocks) == num_blocks + + +# --------------------------------------------------------------------------- +# Test 17: CP lazy target blocks are scaled correctly +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("cp_world_size", [1, 2, 4]) +def test_cp_lazy_target_blocks_scaling(cp_world_size: int) -> None: + """_estimate_lazy_target_blocks returns fewer blocks when cp_world_size > 1 + because each virtual block covers more tokens.""" + kv_cache_config = _make_kv_cache_config(num_blocks=16) + max_batched = 64 + + target_base = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks( + kv_cache_config, max_batched, cp_world_size=1 + ) + target_cp = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks( + kv_cache_config, max_batched, cp_world_size=cp_world_size + ) + + if cp_world_size == 1: + assert target_cp == target_base + else: + assert target_cp < target_base, ( + f"cp_world_size={cp_world_size}: target_cp={target_cp} should be " + f"less than target_base={target_base}" + ) diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 5e431e6238..07978a9dd6 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -82,6 +82,9 @@ class SimpleCPUOffloadScheduler: vllm_config.kv_events_config is not None and vllm_config.kv_events_config.enable_kv_cache_events ) + dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size + pcp_world_size = vllm_config.parallel_config.prefill_context_parallel_size + self.cp_world_size = dcp_world_size * pcp_world_size self.block_size = scheduler_block_size self.hash_block_size = hash_block_size assert self.block_size % self.hash_block_size == 0 @@ -113,9 +116,6 @@ class SimpleCPUOffloadScheduler: ) # TODO (yifan): maybe need to enable kv_cache_events and metrics_collector here. - dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size - pcp_world_size = vllm_config.parallel_config.prefill_context_parallel_size - assert dcp_world_size == 1 and pcp_world_size == 1 self.cpu_coordinator: KVCacheCoordinator = get_kv_cache_coordinator( kv_cache_config=self.cpu_kv_cache_config, max_model_len=vllm_config.model_config.max_model_len, @@ -155,6 +155,7 @@ class SimpleCPUOffloadScheduler: self._target_free = self._estimate_lazy_target_blocks( kv_cache_config, vllm_config.scheduler_config.max_num_batched_tokens, + self.cp_world_size, ) else: self._target_free = 0 @@ -215,19 +216,22 @@ class SimpleCPUOffloadScheduler: @staticmethod def _estimate_lazy_target_blocks( - kv_cache_config: "KVCacheConfig", max_num_batched_tokens: int + kv_cache_config: "KVCacheConfig", + max_num_batched_tokens: int, + cp_world_size: int = 1, ) -> int: """GPU blocks to keep available (free/offloaded) per step in lazy mode.""" WATERMARK_RATIO = 1.0 # Reserve larger space to avoid running out of GPU blocks target = 0 for g in kv_cache_config.kv_cache_groups: spec = g.kv_cache_spec + block_size = spec.block_size * cp_world_size if isinstance(spec, MambaSpec): target += 2 elif isinstance(spec, SlidingWindowSpec): - target += cdiv(spec.sliding_window, spec.block_size) + 1 + target += cdiv(spec.sliding_window, block_size) + 1 else: - target += cdiv(max_num_batched_tokens, spec.block_size) + target += cdiv(max_num_batched_tokens, block_size) return int(target * (1 + WATERMARK_RATIO)) def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None: @@ -363,7 +367,9 @@ class SimpleCPUOffloadScheduler: continue # Number of blocks in the computed range for this group. - g_block_size = kv_cache_groups[g].kv_cache_spec.block_size + g_block_size = ( + kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size + ) n_computed_g = cdiv(total_computed_tokens, g_block_size) # Back-trace: ext blocks sit at the tail of the computed range.