[Bugfix][KV Offload] Allow explicit SimpleCPUOffload block counts (#61674)
Signed-off-by: Sherif Waly <sherif.waly@mistral.ai> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
vllm-agent
parent
71c20f73f2
commit
ce8853d27a
@@ -245,6 +245,55 @@ def test_connector_reports_configured_cache_source(
|
||||
assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_layers", [1, 2])
|
||||
@pytest.mark.parametrize("backend", ["cpu", "disk"])
|
||||
def test_connector_explicit_cpu_blocks_align_uneven_stages(
|
||||
tmp_path, num_layers: int, backend: str
|
||||
):
|
||||
"""The same configured pool size must fit lighter and heavier PP stages."""
|
||||
config = _make_kv_cache_config(num_blocks=16)
|
||||
names = [f"layer_{i}" for i in range(num_layers)]
|
||||
config.kv_cache_groups[0].layer_names = names
|
||||
config.kv_cache_tensors[0].layers = names
|
||||
config.kv_cache_tensors[0].size *= num_layers
|
||||
vllm_config = _make_vllm_config()
|
||||
extra = {
|
||||
"cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 8,
|
||||
"num_cpu_blocks": 4,
|
||||
"kv_offload_backend": backend,
|
||||
}
|
||||
if backend == "disk":
|
||||
extra.update(
|
||||
disk_path=str(tmp_path / "offload.bin"),
|
||||
disk_capacity_bytes=_BYTES_PER_BLOCK * 8,
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config = extra
|
||||
connector = SimpleCPUOffloadConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, config
|
||||
)
|
||||
assert connector.scheduler_manager is not None
|
||||
assert connector.scheduler_manager.num_cpu_blocks == 4
|
||||
assert len(connector.scheduler_manager.cpu_block_pool.blocks) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"count, capacity_bytes",
|
||||
[(count, _BYTES_PER_BLOCK * 8) for count in [0, -1, True, 1.5, "4", 9]]
|
||||
+ [(1, _BYTES_PER_BLOCK // 2)],
|
||||
)
|
||||
def test_connector_rejects_invalid_explicit_cpu_blocks(count, capacity_bytes):
|
||||
config = _make_kv_cache_config(num_blocks=16)
|
||||
vllm_config = _make_vllm_config()
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config = {
|
||||
"cpu_bytes_to_use_per_rank": capacity_bytes,
|
||||
"num_cpu_blocks": count,
|
||||
}
|
||||
with pytest.raises(ValueError, match="num_cpu_blocks"):
|
||||
SimpleCPUOffloadConnector(vllm_config, KVConnectorRole.SCHEDULER, config)
|
||||
|
||||
|
||||
def make_scheduler(
|
||||
num_cpu_blocks: int = 8,
|
||||
num_gpu_blocks: int = 16,
|
||||
|
||||
@@ -371,6 +371,69 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_layers", [1, 2])
|
||||
def test_explicit_cpu_blocks_bound_live_worker_allocation(monkeypatch, num_layers):
|
||||
"""Different live KV strides must honor the same explicit CPU pool size."""
|
||||
num_blocks = 4
|
||||
names = [f"layer.{i}" for i in range(num_layers)]
|
||||
raw = torch.zeros(
|
||||
num_layers * num_blocks * BLOCK_BYTES, dtype=torch.int8, device="cuda"
|
||||
)
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=raw.nbytes,
|
||||
layers=names,
|
||||
layer_stride=num_blocks * BLOCK_BYTES,
|
||||
block_stride=BLOCK_BYTES,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[],
|
||||
)
|
||||
caches = dict(zip(names, raw.view(num_layers, num_blocks, BLOCK_BYTES)))
|
||||
worker = SimpleCPUOffloadWorker(None, config, 8 * BLOCK_BYTES, num_cpu_blocks=4)
|
||||
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
|
||||
try:
|
||||
worker.register_kv_caches(caches)
|
||||
assert worker.num_cpu_blocks == 4
|
||||
assert worker.cpu_kv_caches is not None
|
||||
assert all(t.shape[0] == 4 for t in worker.cpu_kv_caches.values())
|
||||
assert sum(t.nbytes for t in worker.cpu_kv_caches.values()) <= 8 * BLOCK_BYTES
|
||||
finally:
|
||||
if worker._backend is not None:
|
||||
worker._backend.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"capacity_bytes, count, local_capacity",
|
||||
[(8 * BLOCK_BYTES, 5, 4), (BLOCK_BYTES, 1, 0)],
|
||||
)
|
||||
def test_explicit_cpu_blocks_reject_heavier_worker_before_allocation(
|
||||
capacity_bytes, count, local_capacity
|
||||
):
|
||||
"""A count that fits the scheduler's stage must fail on a smaller worker."""
|
||||
names = ["layer.0", "layer.1"]
|
||||
raw = torch.zeros(2 * 4 * BLOCK_BYTES, dtype=torch.int8, device="cuda")
|
||||
config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=raw.nbytes,
|
||||
layers=names,
|
||||
layer_stride=4 * BLOCK_BYTES,
|
||||
block_stride=BLOCK_BYTES,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[],
|
||||
)
|
||||
worker = SimpleCPUOffloadWorker(None, config, capacity_bytes, num_cpu_blocks=count)
|
||||
caches = dict(zip(names, raw.view(2, 4, BLOCK_BYTES)))
|
||||
with pytest.raises(ValueError, match=f"local offload capacity {local_capacity}"):
|
||||
worker.register_kv_caches(caches)
|
||||
assert worker.cpu_kv_caches is None
|
||||
|
||||
|
||||
def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
|
||||
num_blocks = 4
|
||||
block_bytes = 32
|
||||
|
||||
Reference in New Issue
Block a user