[Bugfix][KV Offload] Allow explicit SimpleCPUOffload block counts (#61674)

Signed-off-by: Sherif Waly <sherif.waly@mistral.ai>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Sherif Waly
2026-10-17 02:11:12 +00:00
committed by GitHub
co-authored by vllm-agent
parent 71c20f73f2
commit ce8853d27a
6 changed files with 195 additions and 5 deletions
@@ -245,6 +245,55 @@ def test_connector_reports_configured_cache_source(
assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type]
@pytest.mark.parametrize("num_layers", [1, 2])
@pytest.mark.parametrize("backend", ["cpu", "disk"])
def test_connector_explicit_cpu_blocks_align_uneven_stages(
tmp_path, num_layers: int, backend: str
):
"""The same configured pool size must fit lighter and heavier PP stages."""
config = _make_kv_cache_config(num_blocks=16)
names = [f"layer_{i}" for i in range(num_layers)]
config.kv_cache_groups[0].layer_names = names
config.kv_cache_tensors[0].layers = names
config.kv_cache_tensors[0].size *= num_layers
vllm_config = _make_vllm_config()
extra = {
"cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 8,
"num_cpu_blocks": 4,
"kv_offload_backend": backend,
}
if backend == "disk":
extra.update(
disk_path=str(tmp_path / "offload.bin"),
disk_capacity_bytes=_BYTES_PER_BLOCK * 8,
)
assert vllm_config.kv_transfer_config is not None
vllm_config.kv_transfer_config.kv_connector_extra_config = extra
connector = SimpleCPUOffloadConnector(
vllm_config, KVConnectorRole.SCHEDULER, config
)
assert connector.scheduler_manager is not None
assert connector.scheduler_manager.num_cpu_blocks == 4
assert len(connector.scheduler_manager.cpu_block_pool.blocks) == 4
@pytest.mark.parametrize(
"count, capacity_bytes",
[(count, _BYTES_PER_BLOCK * 8) for count in [0, -1, True, 1.5, "4", 9]]
+ [(1, _BYTES_PER_BLOCK // 2)],
)
def test_connector_rejects_invalid_explicit_cpu_blocks(count, capacity_bytes):
config = _make_kv_cache_config(num_blocks=16)
vllm_config = _make_vllm_config()
assert vllm_config.kv_transfer_config is not None
vllm_config.kv_transfer_config.kv_connector_extra_config = {
"cpu_bytes_to_use_per_rank": capacity_bytes,
"num_cpu_blocks": count,
}
with pytest.raises(ValueError, match="num_cpu_blocks"):
SimpleCPUOffloadConnector(vllm_config, KVConnectorRole.SCHEDULER, config)
def make_scheduler(
num_cpu_blocks: int = 8,
num_gpu_blocks: int = 16,
+63
View File
@@ -371,6 +371,69 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
}
@pytest.mark.parametrize("num_layers", [1, 2])
def test_explicit_cpu_blocks_bound_live_worker_allocation(monkeypatch, num_layers):
"""Different live KV strides must honor the same explicit CPU pool size."""
num_blocks = 4
names = [f"layer.{i}" for i in range(num_layers)]
raw = torch.zeros(
num_layers * num_blocks * BLOCK_BYTES, dtype=torch.int8, device="cuda"
)
config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=raw.nbytes,
layers=names,
layer_stride=num_blocks * BLOCK_BYTES,
block_stride=BLOCK_BYTES,
)
],
kv_cache_groups=[],
)
caches = dict(zip(names, raw.view(num_layers, num_blocks, BLOCK_BYTES)))
worker = SimpleCPUOffloadWorker(None, config, 8 * BLOCK_BYTES, num_cpu_blocks=4)
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
try:
worker.register_kv_caches(caches)
assert worker.num_cpu_blocks == 4
assert worker.cpu_kv_caches is not None
assert all(t.shape[0] == 4 for t in worker.cpu_kv_caches.values())
assert sum(t.nbytes for t in worker.cpu_kv_caches.values()) <= 8 * BLOCK_BYTES
finally:
if worker._backend is not None:
worker._backend.shutdown()
@pytest.mark.parametrize(
"capacity_bytes, count, local_capacity",
[(8 * BLOCK_BYTES, 5, 4), (BLOCK_BYTES, 1, 0)],
)
def test_explicit_cpu_blocks_reject_heavier_worker_before_allocation(
capacity_bytes, count, local_capacity
):
"""A count that fits the scheduler's stage must fail on a smaller worker."""
names = ["layer.0", "layer.1"]
raw = torch.zeros(2 * 4 * BLOCK_BYTES, dtype=torch.int8, device="cuda")
config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(
size=raw.nbytes,
layers=names,
layer_stride=4 * BLOCK_BYTES,
block_stride=BLOCK_BYTES,
)
],
kv_cache_groups=[],
)
worker = SimpleCPUOffloadWorker(None, config, capacity_bytes, num_cpu_blocks=count)
caches = dict(zip(names, raw.view(2, 4, BLOCK_BYTES)))
with pytest.raises(ValueError, match=f"local offload capacity {local_capacity}"):
worker.register_kv_caches(caches)
assert worker.cpu_kv_caches is None
def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
num_blocks = 4
block_bytes = 32