[SimpleCPUOffloadConnector] PCP + DCP support (#39831)
Signed-off-by: Jonathan Chen <chenleejonathan@gmail.com>
This commit is contained in:
@@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
@@ -1528,3 +1529,232 @@ def test_reset_pending_loads() -> None:
|
||||
# All GPU blocks free
|
||||
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
|
||||
assert num_used == 1, f"Expected only null block in use, got {num_used}"
|
||||
|
||||
|
||||
def _make_cp_vllm_config(
|
||||
dcp_world_size: int = 1,
|
||||
pcp_world_size: int = 1,
|
||||
) -> VllmConfig:
|
||||
"""VllmConfig with context-parallel sizes set for scheduler-only tests."""
|
||||
cfg = _make_vllm_config()
|
||||
|
||||
cfg.parallel_config.decode_context_parallel_size = dcp_world_size
|
||||
cfg.parallel_config.prefill_context_parallel_size = pcp_world_size
|
||||
return cfg
|
||||
|
||||
|
||||
def _make_cp_scheduler(
|
||||
*,
|
||||
dcp_world_size: int = 1,
|
||||
pcp_world_size: int = 1,
|
||||
num_cpu_blocks: int = 8,
|
||||
num_gpu_blocks: int = 16,
|
||||
lazy: bool = False,
|
||||
) -> SchedulerFixture:
|
||||
"""Build a SimpleCPUOffloadScheduler with CP-scaled virtual block size."""
|
||||
cp_world_size = dcp_world_size * pcp_world_size
|
||||
virtual_block_size = BLOCK_SIZE * cp_world_size
|
||||
|
||||
kv_cache_config = _make_kv_cache_config(num_gpu_blocks)
|
||||
vllm_config = _make_cp_vllm_config(dcp_world_size, pcp_world_size)
|
||||
cpu_capacity_bytes = _BYTES_PER_BLOCK * num_cpu_blocks
|
||||
|
||||
sched = SimpleCPUOffloadScheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
cpu_capacity_bytes=cpu_capacity_bytes,
|
||||
scheduler_block_size=virtual_block_size,
|
||||
hash_block_size=virtual_block_size,
|
||||
lazy_offload=lazy,
|
||||
)
|
||||
|
||||
gpu_block_pool = BlockPool(
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
enable_caching=True,
|
||||
hash_block_size=virtual_block_size,
|
||||
)
|
||||
sched.bind_gpu_block_pool(gpu_block_pool)
|
||||
|
||||
return SchedulerFixture(
|
||||
scheduler=sched,
|
||||
gpu_block_pool=gpu_block_pool,
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
|
||||
|
||||
def _make_cp_request(
|
||||
num_blocks: int,
|
||||
virtual_block_size: int,
|
||||
request_id: str | None = None,
|
||||
) -> Request:
|
||||
"""Create a request whose block hashes are computed at the virtual
|
||||
(CP-scaled) block size, matching what the real scheduler does.
|
||||
"""
|
||||
global _req_counter
|
||||
_req_counter += 1
|
||||
if request_id is None:
|
||||
request_id = f"req-cp-{_req_counter}"
|
||||
|
||||
num_tokens = num_blocks * virtual_block_size + 1
|
||||
start = _req_counter * 10000
|
||||
prompt_token_ids = list(range(start, start + num_tokens))
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
return Request(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=get_request_block_hasher(virtual_block_size, sha256),
|
||||
)
|
||||
|
||||
|
||||
def _allocate_cp_gpu_blocks(
|
||||
gpu_block_pool: BlockPool,
|
||||
request: Request,
|
||||
num_blocks: int,
|
||||
virtual_block_size: int,
|
||||
group_id: int = 0,
|
||||
) -> list:
|
||||
"""Allocate GPU blocks and cache them using the CP-scaled block size."""
|
||||
blocks = gpu_block_pool.get_new_blocks(num_blocks)
|
||||
num_full = min(num_blocks, len(request.block_hashes))
|
||||
if num_full > 0:
|
||||
gpu_block_pool.cache_full_blocks(
|
||||
request=request,
|
||||
blocks=blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=num_full,
|
||||
block_size=virtual_block_size,
|
||||
kv_cache_group_id=group_id,
|
||||
)
|
||||
return blocks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 15: CP block size scaling is correct
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.parametrize(
|
||||
"dcp_world_size, pcp_world_size",
|
||||
[
|
||||
(2, 1), # DCP only
|
||||
(1, 2), # PCP only
|
||||
(2, 2), # DCP + PCP
|
||||
],
|
||||
)
|
||||
def test_cp_block_size_scaling(dcp_world_size: int, pcp_world_size: int) -> None:
|
||||
"""Verify that the scheduler's block_size and cp_world_size are correctly
|
||||
scaled when context parallelism is enabled."""
|
||||
fix = _make_cp_scheduler(
|
||||
dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size
|
||||
)
|
||||
sched = fix.scheduler
|
||||
|
||||
expected_cp = dcp_world_size * pcp_world_size
|
||||
assert sched.cp_world_size == expected_cp
|
||||
assert sched.block_size == BLOCK_SIZE * expected_cp
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 16: CP eager store-and-load roundtrip
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.parametrize(
|
||||
"dcp_world_size, pcp_world_size",
|
||||
[
|
||||
(2, 1),
|
||||
(1, 2),
|
||||
],
|
||||
)
|
||||
def test_cp_eager_store_and_load_roundtrip(
|
||||
dcp_world_size: int, pcp_world_size: int
|
||||
) -> None:
|
||||
"""With CP enabled, store blocks to CPU and reload them for a new request
|
||||
with matching tokens. Verifies that hash matching and transfer-pair
|
||||
construction work with the virtual block size."""
|
||||
fix = _make_cp_scheduler(
|
||||
dcp_world_size=dcp_world_size,
|
||||
pcp_world_size=pcp_world_size,
|
||||
num_cpu_blocks=8,
|
||||
num_gpu_blocks=16,
|
||||
lazy=False,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
cp = dcp_world_size * pcp_world_size
|
||||
vbs = BLOCK_SIZE * cp
|
||||
|
||||
num_blocks = 2
|
||||
req = _make_cp_request(num_blocks, vbs)
|
||||
|
||||
# Allocate GPU blocks and register hashes
|
||||
gpu_blocks = _allocate_cp_gpu_blocks(fix.gpu_block_pool, req, num_blocks, vbs)
|
||||
kv_blocks = KVCacheBlocks(blocks=(gpu_blocks,))
|
||||
req.num_computed_tokens = num_blocks * vbs
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
|
||||
block_ids = kv_blocks.get_block_ids()
|
||||
sched_out = make_scheduler_output(
|
||||
{req.request_id: num_blocks * vbs},
|
||||
new_reqs={req.request_id: block_ids},
|
||||
)
|
||||
|
||||
meta = sched.build_connector_meta(sched_out)
|
||||
assert meta.store_event >= 0, "Expected a store event"
|
||||
assert len(meta.store_gpu_blocks) == num_blocks
|
||||
assert len(meta.store_cpu_blocks) == num_blocks
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
# New request with same tokens — should get a full CPU cache hit.
|
||||
req2 = Request(
|
||||
request_id="req-cp-load",
|
||||
prompt_token_ids=req.prompt_token_ids,
|
||||
sampling_params=req.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=req._block_hasher,
|
||||
)
|
||||
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
|
||||
assert hit_tokens == num_blocks * vbs
|
||||
assert is_async is True
|
||||
|
||||
# Allocate fresh GPU blocks for the load.
|
||||
gpu_blocks2 = fix.gpu_block_pool.get_new_blocks(num_blocks)
|
||||
kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,))
|
||||
sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens)
|
||||
|
||||
sched_out2 = make_scheduler_output(
|
||||
{req2.request_id: 1},
|
||||
new_reqs={req2.request_id: kv_blocks2.get_block_ids()},
|
||||
)
|
||||
meta2 = sched.build_connector_meta(sched_out2)
|
||||
assert meta2.load_event >= 0, "Expected a load event"
|
||||
assert len(meta2.load_gpu_blocks) == num_blocks
|
||||
assert len(meta2.load_cpu_blocks) == num_blocks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 17: CP lazy target blocks are scaled correctly
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.parametrize("cp_world_size", [1, 2, 4])
|
||||
def test_cp_lazy_target_blocks_scaling(cp_world_size: int) -> None:
|
||||
"""_estimate_lazy_target_blocks returns fewer blocks when cp_world_size > 1
|
||||
because each virtual block covers more tokens."""
|
||||
kv_cache_config = _make_kv_cache_config(num_blocks=16)
|
||||
max_batched = 64
|
||||
|
||||
target_base = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks(
|
||||
kv_cache_config, max_batched, cp_world_size=1
|
||||
)
|
||||
target_cp = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks(
|
||||
kv_cache_config, max_batched, cp_world_size=cp_world_size
|
||||
)
|
||||
|
||||
if cp_world_size == 1:
|
||||
assert target_cp == target_base
|
||||
else:
|
||||
assert target_cp < target_base, (
|
||||
f"cp_world_size={cp_world_size}: target_cp={target_cp} should be "
|
||||
f"less than target_base={target_base}"
|
||||
)
|
||||
|
||||
@@ -82,6 +82,9 @@ class SimpleCPUOffloadScheduler:
|
||||
vllm_config.kv_events_config is not None
|
||||
and vllm_config.kv_events_config.enable_kv_cache_events
|
||||
)
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
pcp_world_size = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
self.cp_world_size = dcp_world_size * pcp_world_size
|
||||
self.block_size = scheduler_block_size
|
||||
self.hash_block_size = hash_block_size
|
||||
assert self.block_size % self.hash_block_size == 0
|
||||
@@ -113,9 +116,6 @@ class SimpleCPUOffloadScheduler:
|
||||
)
|
||||
|
||||
# TODO (yifan): maybe need to enable kv_cache_events and metrics_collector here.
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
pcp_world_size = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
assert dcp_world_size == 1 and pcp_world_size == 1
|
||||
self.cpu_coordinator: KVCacheCoordinator = get_kv_cache_coordinator(
|
||||
kv_cache_config=self.cpu_kv_cache_config,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
@@ -155,6 +155,7 @@ class SimpleCPUOffloadScheduler:
|
||||
self._target_free = self._estimate_lazy_target_blocks(
|
||||
kv_cache_config,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
self.cp_world_size,
|
||||
)
|
||||
else:
|
||||
self._target_free = 0
|
||||
@@ -215,19 +216,22 @@ class SimpleCPUOffloadScheduler:
|
||||
|
||||
@staticmethod
|
||||
def _estimate_lazy_target_blocks(
|
||||
kv_cache_config: "KVCacheConfig", max_num_batched_tokens: int
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
max_num_batched_tokens: int,
|
||||
cp_world_size: int = 1,
|
||||
) -> int:
|
||||
"""GPU blocks to keep available (free/offloaded) per step in lazy mode."""
|
||||
WATERMARK_RATIO = 1.0 # Reserve larger space to avoid running out of GPU blocks
|
||||
target = 0
|
||||
for g in kv_cache_config.kv_cache_groups:
|
||||
spec = g.kv_cache_spec
|
||||
block_size = spec.block_size * cp_world_size
|
||||
if isinstance(spec, MambaSpec):
|
||||
target += 2
|
||||
elif isinstance(spec, SlidingWindowSpec):
|
||||
target += cdiv(spec.sliding_window, spec.block_size) + 1
|
||||
target += cdiv(spec.sliding_window, block_size) + 1
|
||||
else:
|
||||
target += cdiv(max_num_batched_tokens, spec.block_size)
|
||||
target += cdiv(max_num_batched_tokens, block_size)
|
||||
return int(target * (1 + WATERMARK_RATIO))
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
@@ -363,7 +367,9 @@ class SimpleCPUOffloadScheduler:
|
||||
continue
|
||||
|
||||
# Number of blocks in the computed range for this group.
|
||||
g_block_size = kv_cache_groups[g].kv_cache_spec.block_size
|
||||
g_block_size = (
|
||||
kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size
|
||||
)
|
||||
n_computed_g = cdiv(total_computed_tokens, g_block_size)
|
||||
|
||||
# Back-trace: ext blocks sit at the tail of the computed range.
|
||||
|
||||
Reference in New Issue
Block a user