[SimpleCPUOffloadConnector] PCP + DCP support (#39831)

Signed-off-by: Jonathan Chen <chenleejonathan@gmail.com>
This commit is contained in:
Jonathan Chen
2026-06-20 15:01:06 -07:00
committed by GitHub
parent ab7fcbdd5d
commit c88d3d4775
2 changed files with 243 additions and 7 deletions
@@ -6,6 +6,7 @@ from __future__ import annotations
from dataclasses import dataclass
import pytest
import torch
from vllm import SamplingParams
@@ -1528,3 +1529,232 @@ def test_reset_pending_loads() -> None:
# All GPU blocks free
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
assert num_used == 1, f"Expected only null block in use, got {num_used}"
def _make_cp_vllm_config(
dcp_world_size: int = 1,
pcp_world_size: int = 1,
) -> VllmConfig:
"""VllmConfig with context-parallel sizes set for scheduler-only tests."""
cfg = _make_vllm_config()
cfg.parallel_config.decode_context_parallel_size = dcp_world_size
cfg.parallel_config.prefill_context_parallel_size = pcp_world_size
return cfg
def _make_cp_scheduler(
*,
dcp_world_size: int = 1,
pcp_world_size: int = 1,
num_cpu_blocks: int = 8,
num_gpu_blocks: int = 16,
lazy: bool = False,
) -> SchedulerFixture:
"""Build a SimpleCPUOffloadScheduler with CP-scaled virtual block size."""
cp_world_size = dcp_world_size * pcp_world_size
virtual_block_size = BLOCK_SIZE * cp_world_size
kv_cache_config = _make_kv_cache_config(num_gpu_blocks)
vllm_config = _make_cp_vllm_config(dcp_world_size, pcp_world_size)
cpu_capacity_bytes = _BYTES_PER_BLOCK * num_cpu_blocks
sched = SimpleCPUOffloadScheduler(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
cpu_capacity_bytes=cpu_capacity_bytes,
scheduler_block_size=virtual_block_size,
hash_block_size=virtual_block_size,
lazy_offload=lazy,
)
gpu_block_pool = BlockPool(
num_gpu_blocks=num_gpu_blocks,
enable_caching=True,
hash_block_size=virtual_block_size,
)
sched.bind_gpu_block_pool(gpu_block_pool)
return SchedulerFixture(
scheduler=sched,
gpu_block_pool=gpu_block_pool,
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
)
def _make_cp_request(
num_blocks: int,
virtual_block_size: int,
request_id: str | None = None,
) -> Request:
"""Create a request whose block hashes are computed at the virtual
(CP-scaled) block size, matching what the real scheduler does.
"""
global _req_counter
_req_counter += 1
if request_id is None:
request_id = f"req-cp-{_req_counter}"
num_tokens = num_blocks * virtual_block_size + 1
start = _req_counter * 10000
prompt_token_ids = list(range(start, start + num_tokens))
sampling_params = SamplingParams(max_tokens=1)
return Request(
request_id=request_id,
prompt_token_ids=prompt_token_ids,
sampling_params=sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=get_request_block_hasher(virtual_block_size, sha256),
)
def _allocate_cp_gpu_blocks(
gpu_block_pool: BlockPool,
request: Request,
num_blocks: int,
virtual_block_size: int,
group_id: int = 0,
) -> list:
"""Allocate GPU blocks and cache them using the CP-scaled block size."""
blocks = gpu_block_pool.get_new_blocks(num_blocks)
num_full = min(num_blocks, len(request.block_hashes))
if num_full > 0:
gpu_block_pool.cache_full_blocks(
request=request,
blocks=blocks,
num_cached_blocks=0,
num_full_blocks=num_full,
block_size=virtual_block_size,
kv_cache_group_id=group_id,
)
return blocks
# ---------------------------------------------------------------------------
# Test 15: CP block size scaling is correct
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"dcp_world_size, pcp_world_size",
[
(2, 1), # DCP only
(1, 2), # PCP only
(2, 2), # DCP + PCP
],
)
def test_cp_block_size_scaling(dcp_world_size: int, pcp_world_size: int) -> None:
"""Verify that the scheduler's block_size and cp_world_size are correctly
scaled when context parallelism is enabled."""
fix = _make_cp_scheduler(
dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size
)
sched = fix.scheduler
expected_cp = dcp_world_size * pcp_world_size
assert sched.cp_world_size == expected_cp
assert sched.block_size == BLOCK_SIZE * expected_cp
# ---------------------------------------------------------------------------
# Test 16: CP eager store-and-load roundtrip
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"dcp_world_size, pcp_world_size",
[
(2, 1),
(1, 2),
],
)
def test_cp_eager_store_and_load_roundtrip(
dcp_world_size: int, pcp_world_size: int
) -> None:
"""With CP enabled, store blocks to CPU and reload them for a new request
with matching tokens. Verifies that hash matching and transfer-pair
construction work with the virtual block size."""
fix = _make_cp_scheduler(
dcp_world_size=dcp_world_size,
pcp_world_size=pcp_world_size,
num_cpu_blocks=8,
num_gpu_blocks=16,
lazy=False,
)
sched = fix.scheduler
cp = dcp_world_size * pcp_world_size
vbs = BLOCK_SIZE * cp
num_blocks = 2
req = _make_cp_request(num_blocks, vbs)
# Allocate GPU blocks and register hashes
gpu_blocks = _allocate_cp_gpu_blocks(fix.gpu_block_pool, req, num_blocks, vbs)
kv_blocks = KVCacheBlocks(blocks=(gpu_blocks,))
req.num_computed_tokens = num_blocks * vbs
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
block_ids = kv_blocks.get_block_ids()
sched_out = make_scheduler_output(
{req.request_id: num_blocks * vbs},
new_reqs={req.request_id: block_ids},
)
meta = sched.build_connector_meta(sched_out)
assert meta.store_event >= 0, "Expected a store event"
assert len(meta.store_gpu_blocks) == num_blocks
assert len(meta.store_cpu_blocks) == num_blocks
simulate_store_completion(sched, meta.store_event)
# New request with same tokens — should get a full CPU cache hit.
req2 = Request(
request_id="req-cp-load",
prompt_token_ids=req.prompt_token_ids,
sampling_params=req.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=req._block_hasher,
)
hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
assert hit_tokens == num_blocks * vbs
assert is_async is True
# Allocate fresh GPU blocks for the load.
gpu_blocks2 = fix.gpu_block_pool.get_new_blocks(num_blocks)
kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,))
sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens)
sched_out2 = make_scheduler_output(
{req2.request_id: 1},
new_reqs={req2.request_id: kv_blocks2.get_block_ids()},
)
meta2 = sched.build_connector_meta(sched_out2)
assert meta2.load_event >= 0, "Expected a load event"
assert len(meta2.load_gpu_blocks) == num_blocks
assert len(meta2.load_cpu_blocks) == num_blocks
# ---------------------------------------------------------------------------
# Test 17: CP lazy target blocks are scaled correctly
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("cp_world_size", [1, 2, 4])
def test_cp_lazy_target_blocks_scaling(cp_world_size: int) -> None:
"""_estimate_lazy_target_blocks returns fewer blocks when cp_world_size > 1
because each virtual block covers more tokens."""
kv_cache_config = _make_kv_cache_config(num_blocks=16)
max_batched = 64
target_base = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks(
kv_cache_config, max_batched, cp_world_size=1
)
target_cp = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks(
kv_cache_config, max_batched, cp_world_size=cp_world_size
)
if cp_world_size == 1:
assert target_cp == target_base
else:
assert target_cp < target_base, (
f"cp_world_size={cp_world_size}: target_cp={target_cp} should be "
f"less than target_base={target_base}"
)
+13 -7
View File
@@ -82,6 +82,9 @@ class SimpleCPUOffloadScheduler:
vllm_config.kv_events_config is not None
and vllm_config.kv_events_config.enable_kv_cache_events
)
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
pcp_world_size = vllm_config.parallel_config.prefill_context_parallel_size
self.cp_world_size = dcp_world_size * pcp_world_size
self.block_size = scheduler_block_size
self.hash_block_size = hash_block_size
assert self.block_size % self.hash_block_size == 0
@@ -113,9 +116,6 @@ class SimpleCPUOffloadScheduler:
)
# TODO (yifan): maybe need to enable kv_cache_events and metrics_collector here.
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
pcp_world_size = vllm_config.parallel_config.prefill_context_parallel_size
assert dcp_world_size == 1 and pcp_world_size == 1
self.cpu_coordinator: KVCacheCoordinator = get_kv_cache_coordinator(
kv_cache_config=self.cpu_kv_cache_config,
max_model_len=vllm_config.model_config.max_model_len,
@@ -155,6 +155,7 @@ class SimpleCPUOffloadScheduler:
self._target_free = self._estimate_lazy_target_blocks(
kv_cache_config,
vllm_config.scheduler_config.max_num_batched_tokens,
self.cp_world_size,
)
else:
self._target_free = 0
@@ -215,19 +216,22 @@ class SimpleCPUOffloadScheduler:
@staticmethod
def _estimate_lazy_target_blocks(
kv_cache_config: "KVCacheConfig", max_num_batched_tokens: int
kv_cache_config: "KVCacheConfig",
max_num_batched_tokens: int,
cp_world_size: int = 1,
) -> int:
"""GPU blocks to keep available (free/offloaded) per step in lazy mode."""
WATERMARK_RATIO = 1.0 # Reserve larger space to avoid running out of GPU blocks
target = 0
for g in kv_cache_config.kv_cache_groups:
spec = g.kv_cache_spec
block_size = spec.block_size * cp_world_size
if isinstance(spec, MambaSpec):
target += 2
elif isinstance(spec, SlidingWindowSpec):
target += cdiv(spec.sliding_window, spec.block_size) + 1
target += cdiv(spec.sliding_window, block_size) + 1
else:
target += cdiv(max_num_batched_tokens, spec.block_size)
target += cdiv(max_num_batched_tokens, block_size)
return int(target * (1 + WATERMARK_RATIO))
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
@@ -363,7 +367,9 @@ class SimpleCPUOffloadScheduler:
continue
# Number of blocks in the computed range for this group.
g_block_size = kv_cache_groups[g].kv_cache_spec.block_size
g_block_size = (
kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size
)
n_computed_g = cdiv(total_computed_tokens, g_block_size)
# Back-trace: ext blocks sit at the tail of the computed range.