[KV Offload] Define clean backend configuration boundary (#48150)

Signed-off-by: Change72 <changg@nvidia.com>
Signed-off-by: Chang Guo <cguo51@asu.edu>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Chang Guo
2026-07-16 13:27:05 +03:00
committed by GitHub
co-authored by Codex Claude Fable 5 Or Ozeri OpenAI Codex
parent 8c3393f373
commit a9531edfa6
27 changed files with 1287 additions and 855 deletions
+20 -20
View File
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
NUM_GPU_BLOCKS = [64]
NUM_CPU_BLOCKS = [256]
GPU_PAGE_SIZES = [512, 1024]
BLOCK_SIZE_FACTORS = [1, 3]
BLOCKS_PER_CHUNK_VALUES = [1, 3]
NUM_TENSORS = [4]
SEEDS = [0]
DEVICE_TYPE = current_platform.device_type
@@ -35,7 +35,7 @@ NUM_MAPPINGS_PER_GROUP = [2]
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
@@ -48,7 +48,7 @@ def test_transfer(
gpu_to_cpu: bool,
num_mappings: int,
gpu_page_size_bytes: int,
block_size_factor: int,
blocks_per_chunk: int,
num_gpu_blocks: int,
num_cpu_blocks: int,
num_tensors: int,
@@ -92,7 +92,7 @@ def test_transfer(
mmap_region: SharedOffloadRegion | None = None
if use_shared_memory:
cpu_page_size = round_up(
gpu_page_size_bytes * num_tensors * block_size_factor,
gpu_page_size_bytes * num_tensors * blocks_per_chunk,
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
)
mmap_region = SharedOffloadRegion(
@@ -105,25 +105,25 @@ def test_transfer(
worker = CPUOffloadingWorker(
kv_caches=kv_caches,
block_size_factor=block_size_factor,
blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks,
mmap_region=mmap_region,
)
# select block mappings
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * block_size_factor)
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
# expand cpu blocks to gpu-page granularity for uniform comparison:
# each cpu block maps to block_size_factor consecutive sub-blocks
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
cpu_blocks_expanded = [
cpu_block * block_size_factor + j
cpu_block * blocks_per_chunk + j
for cpu_block in cpu_blocks
for j in range(block_size_factor)
for j in range(blocks_per_chunk)
]
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
blocks_to_skip = block_size_factor - 1
blocks_to_skip = blocks_per_chunk - 1
if blocks_to_skip > 0:
gpu_blocks = gpu_blocks[blocks_to_skip:]
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
@@ -214,7 +214,7 @@ def test_transfer(
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
@pytest.mark.parametrize("num_mappings_per_group", NUM_MAPPINGS_PER_GROUP)
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
@pytest.mark.parametrize("seed", SEEDS)
@@ -225,7 +225,7 @@ def test_transfer_multi_group(
gpu_to_cpu: bool,
num_mappings_per_group: int,
gpu_page_size_bytes: int,
block_size_factor: int,
blocks_per_chunk: int,
num_gpu_blocks: int,
num_cpu_blocks: int,
seed: int,
@@ -234,7 +234,7 @@ def test_transfer_multi_group(
"""Test transfers with three KV cache groups:
- Group 0: aligned transfer with num_mappings_per_group blocks
- Group 1: zero blocks (empty group)
- Group 2: unaligned CPU->GPU transfer (logical_offset=block_size_factor-1,
- Group 2: unaligned CPU->GPU transfer (logical_offset=blocks_per_chunk-1,
causing the implementation to skip source sub-blocks) with
num_mappings_per_group blocks
"""
@@ -275,7 +275,7 @@ def test_transfer_multi_group(
worker = CPUOffloadingWorker(
kv_caches=canonical_kv_caches,
block_size_factor=block_size_factor,
blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks,
)
@@ -283,7 +283,7 @@ def test_transfer_multi_group(
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
total_gpu_blocks_needed = total_cpu_blocks * block_size_factor
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
@@ -293,7 +293,7 @@ def test_transfer_multi_group(
gpu_offset = 0
cpu_offset = 0
for size in group_sizes_in_cpu_blocks:
gpu_count = size * block_size_factor
gpu_count = size * blocks_per_chunk
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
gpu_offset += gpu_count
@@ -302,15 +302,15 @@ def test_transfer_multi_group(
# expand cpu blocks to gpu-page granularity
cpu_blocks_expanded_per_group = [
[
cpu_block * block_size_factor + j
cpu_block * blocks_per_chunk + j
for cpu_block in cpu_blocks
for j in range(block_size_factor)
for j in range(blocks_per_chunk)
]
for cpu_blocks in cpu_blocks_per_group
]
# skip sub-blocks from group 2 to test unaligned transfers.
sub_blocks_to_skip = block_size_factor - 1 # e.g. 2 when block_size_factor=3
sub_blocks_to_skip = blocks_per_chunk - 1 # e.g. 2 when blocks_per_chunk=3
if sub_blocks_to_skip > 0:
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
sub_blocks_to_skip:-sub_blocks_to_skip
@@ -347,7 +347,7 @@ def test_transfer_multi_group(
cpu_blocks_expanded_per_group, gpu_blocks_per_group
)
]
num_dst_sub_blocks = num_cpu_blocks * block_size_factor
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
else:
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)
+329 -13
View File
@@ -11,17 +11,33 @@ These tests verify:
4. Error paths — unregistered specs, missing config, duplicate registration.
"""
from typing import cast
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import KVTransferConfig
from vllm.config import KVTransferConfig, ParallelConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
build_offloading_config,
)
from vllm.platforms import current_platform
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
MLAAttentionSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_offload.base import OffloadingHistogramMetadata, OffloadingSpec
from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
OffloadingHistogramMetadata,
OffloadingManager,
OffloadingSpec,
OffloadingWorker,
)
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
@@ -39,6 +55,17 @@ def restore_registry():
OffloadingSpecFactory._registry = original
def _get_extra_config(config: VllmConfig) -> dict:
assert config.kv_transfer_config is not None
return config.kv_transfer_config.kv_connector_extra_config
def _create_spec(config: VllmConfig, kv_cache_config: KVCacheConfig) -> OffloadingSpec:
return OffloadingSpecFactory.create_spec(
build_offloading_config(config, kv_cache_config)
)
def _make_vllm_config(
spec_name: str | None = "CPUOffloadingSpec",
cpu_bytes_to_use: int | None = None,
@@ -95,6 +122,46 @@ def _make_vllm_config(
)
def _make_layout_vllm_config(
spec_name: str = "CPUOffloadingSpec",
cpu_bytes_to_use: int | None = None,
extra_config: dict | None = None,
tensor_parallel_size: int = 1,
pipeline_parallel_size: int = 1,
prefill_context_parallel_size: int = 1,
decode_context_parallel_size: int = 1,
) -> VllmConfig:
config = MagicMock()
config.cache_config.block_size = 16
config.cache_config.enable_prefix_caching = True
config.cache_config.prefix_match_unit = None
config.cache_config.cache_dtype = torch.float16
config.model_config.model = "test-model"
world_size = (
tensor_parallel_size * pipeline_parallel_size * prefill_context_parallel_size
)
with patch.object(current_platform, "device_count", return_value=world_size):
config.parallel_config = ParallelConfig(
tensor_parallel_size=tensor_parallel_size,
pipeline_parallel_size=pipeline_parallel_size,
prefill_context_parallel_size=prefill_context_parallel_size,
decode_context_parallel_size=decode_context_parallel_size,
)
config.kv_events_config = None
config.use_v2_model_runner = False
connector_extra_config = dict(extra_config or {})
connector_extra_config["spec_name"] = spec_name
if cpu_bytes_to_use is not None:
connector_extra_config["cpu_bytes_to_use"] = cpu_bytes_to_use
config.kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
kv_connector_extra_config=connector_extra_config,
)
return cast(VllmConfig, config)
def _make_kv_cache_config():
"""Build a minimal KVCacheConfig with one KV cache tensor."""
num_blocks = 16
@@ -122,6 +189,78 @@ def _make_kv_cache_config():
)
def _make_sizing_kv_cache_config(packed: bool) -> KVCacheConfig:
num_blocks = 4
if packed:
kv_cache_tensors = [
KVCacheTensor(
size=64,
shared_by=[layer_name],
block_stride=16,
)
for layer_name in ("layer0", "layer1")
]
else:
kv_cache_tensors = [
KVCacheTensor(size=40, shared_by=["layer0"]),
KVCacheTensor(size=24, shared_by=["layer1"]),
]
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=[
KVCacheGroupSpec(
["layer0", "layer1"],
FullAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
],
)
def _make_hybrid_kv_cache_config() -> KVCacheConfig:
return KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(size=40, shared_by=["full_layer"]),
KVCacheTensor(size=24, shared_by=["mla_layer"]),
],
kv_cache_groups=[
KVCacheGroupSpec(
["full_layer"],
FullAttentionSpec(
block_size=12,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mla_layer"],
MLAAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=576,
dtype=torch.float32,
),
),
],
)
class SingleArgExternalOffloadingSpec(OffloadingSpec):
def get_manager(self) -> OffloadingManager:
raise NotImplementedError
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
raise NotImplementedError
# ---------------------------------------------------------------------------
# Pre-registration integrity (CI sentinel)
# ---------------------------------------------------------------------------
@@ -154,7 +293,7 @@ def test_tiering_spec_registered():
def test_get_spec_cls_returns_registered_class():
"""Registered spec_name returns correct class."""
config = _make_vllm_config(spec_name="CPUOffloadingSpec")
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
assert spec_cls is CPUOffloadingSpec
@@ -162,7 +301,7 @@ def test_get_spec_cls_default_to_cpu():
"""Default spec_name (absent from config) resolves to CPUOffloadingSpec."""
config = _make_vllm_config(spec_name=None)
config.kv_transfer_config.kv_connector_extra_config.pop("spec_name", None)
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
assert spec_cls is CPUOffloadingSpec
@@ -176,16 +315,193 @@ def test_create_cpu_offloading_spec_end_to_end():
Verifies:
- cpu_bytes_to_use validation and num_blocks calculation
- block_size % hash_block_size assertion
- block_size % tokens_per_hash assertion
- spec instance is CPUOffloadingSpec
"""
config = _make_vllm_config(cpu_bytes_to_use=65536)
kv_cache_config = _make_kv_cache_config()
spec = OffloadingSpecFactory.create_spec(config, kv_cache_config)
spec = _create_spec(config, kv_cache_config)
assert isinstance(spec, CPUOffloadingSpec)
assert spec.num_blocks > 0
@pytest.mark.parametrize("packed", [False, True])
def test_cpu_spec_sizing_preserves_tensor_layout(packed: bool):
cpu_bytes_to_use = 1920
config = _make_layout_vllm_config(
cpu_bytes_to_use=cpu_bytes_to_use,
extra_config={"block_size": 32},
tensor_parallel_size=3,
pipeline_parallel_size=2,
)
spec = _create_spec(config, _make_sizing_kv_cache_config(packed))
assert isinstance(spec, CPUOffloadingSpec)
assert spec.cpu_page_size_per_worker == 32
assert spec.kv_bytes_per_chunk == 192
assert spec.num_blocks == cpu_bytes_to_use // 192
def test_cpu_spec_rejects_partially_packed_tensor_layout():
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
kv_cache_config.kv_cache_tensors[0].block_stride = 16
with pytest.raises(AssertionError):
_create_spec(config, kv_cache_config)
def test_cpu_spec_zero_blocks_skips_tensor_layout_validation():
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
kv_cache_config.num_blocks = 0
kv_cache_config.kv_cache_tensors[0].block_stride = 16
spec = _create_spec(config, kv_cache_config)
assert isinstance(spec, CPUOffloadingSpec)
assert spec.cpu_page_size_per_worker == 0
assert spec.kv_bytes_per_chunk == 0
assert spec.num_blocks == 0
def test_tiering_spec_aligns_row_size():
alignment = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT
cpu_bytes_to_use = alignment * 3
config = _make_layout_vllm_config(
spec_name="TieringOffloadingSpec",
cpu_bytes_to_use=cpu_bytes_to_use,
extra_config={"block_size": 32},
tensor_parallel_size=3,
pipeline_parallel_size=2,
)
spec = _create_spec(config, _make_sizing_kv_cache_config(packed=False))
assert isinstance(spec, TieringOffloadingSpec)
assert spec.cpu_page_size_per_worker == 32
assert spec.kv_bytes_per_chunk == alignment
assert spec.num_blocks == cpu_bytes_to_use // alignment
def test_offloading_spec_resolves_prefill_context_parallel_block_sizes():
config = _make_layout_vllm_config(
cpu_bytes_to_use=65536,
extra_config={"block_size": 64},
prefill_context_parallel_size=2,
)
spec = _create_spec(config, _make_kv_cache_config())
assert spec.tokens_per_block == (32,)
assert spec.tokens_per_hash == 32
assert spec.blocks_per_chunk == 2
def test_offloading_config_preserves_data_parallel_index():
config = _make_layout_vllm_config()
config.parallel_config.data_parallel_index = 2
offloading_config = build_offloading_config(config, _make_kv_cache_config())
assert offloading_config.parallel.data_parallel_index == 2
def test_offloading_spec_resolves_heterogeneous_hybrid_block_sizes():
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
config.cache_config.block_size = 4
spec = _create_spec(config, _make_hybrid_kv_cache_config())
assert spec.tokens_per_block == (12, 16)
assert spec.tokens_per_hash == 4
assert spec.blocks_per_chunk == 1
def _full_attention_spec(block_size: int = 16) -> FullAttentionSpec:
return FullAttentionSpec(
block_size=block_size, num_kv_heads=4, head_size=128, dtype=torch.float32
)
def _parallelism_agnostic(kv_cache_groups: list[KVCacheGroupSpec]) -> bool:
config = _make_layout_vllm_config()
kv_cache_config = KVCacheConfig(
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
)
offloading_config = build_offloading_config(config, kv_cache_config)
return offloading_config.parallel.is_parallelism_agnostic
def test_parallelism_agnostic_for_single_full_attention_group():
assert _parallelism_agnostic([KVCacheGroupSpec(["l0"], _full_attention_spec())])
@pytest.mark.parametrize(
"kv_cache_groups",
[
# MLA latent KV is replicated per rank, never head-sharded.
[
KVCacheGroupSpec(
["l0"],
MLAAttentionSpec(
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
),
)
],
# Sliding window is not full attention.
[
KVCacheGroupSpec(
["l0"],
SlidingWindowSpec(
block_size=16,
num_kv_heads=4,
head_size=128,
dtype=torch.float32,
sliding_window=128,
),
)
],
# Hybrid model: more than one KV cache group.
[
KVCacheGroupSpec(["l0"], _full_attention_spec()),
KVCacheGroupSpec(["l1"], _full_attention_spec()),
],
],
)
def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
assert not _parallelism_agnostic(kv_cache_groups)
def test_parallelism_agnostic_disabled_on_v2_model_runner():
config = _make_layout_vllm_config()
config.use_v2_model_runner = True
kv_cache_config = KVCacheConfig(
num_blocks=0,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["l0"], _full_attention_spec())],
)
offloading_config = build_offloading_config(config, kv_cache_config)
assert not offloading_config.parallel.is_parallelism_agnostic
def test_create_dynamic_spec_receives_translated_config():
config = _make_layout_vllm_config(
spec_name="SingleArgExternalOffloadingSpec",
extra_config={
"spec_module_path": "tests.v1.kv_offload.test_factory",
},
)
kv_cache_config = _make_kv_cache_config()
offloading_config = build_offloading_config(config, kv_cache_config)
spec = OffloadingSpecFactory.create_spec(offloading_config)
assert isinstance(spec, SingleArgExternalOffloadingSpec)
assert spec.config is offloading_config
# ---------------------------------------------------------------------------
# Dynamic import via spec_module_path
# ---------------------------------------------------------------------------
@@ -205,7 +521,7 @@ def test_dynamic_load_via_spec_module_path():
config.kv_transfer_config.kv_connector_extra_config["spec_module_path"] = (
"vllm.v1.kv_offload.cpu.spec"
)
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
assert spec_cls is CPUOffloadingSpec
@@ -218,12 +534,12 @@ def test_unregistered_spec_without_module_path_raises():
"""spec_name not in registry + no spec_module_path → ValueError."""
config = _make_vllm_config(spec_name="NonexistentSpec")
with pytest.raises(ValueError, match="Unsupported spec type"):
OffloadingSpecFactory.get_spec_cls(config)
OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
# create_spec should also fail (calls get_spec_cls internally)
kv_cache_config = _make_kv_cache_config()
with pytest.raises(ValueError, match="Unsupported spec type"):
OffloadingSpecFactory.create_spec(config, kv_cache_config)
_create_spec(config, kv_cache_config)
def test_cpu_spec_missing_cpu_bytes_to_use_raises():
@@ -232,7 +548,7 @@ def test_cpu_spec_missing_cpu_bytes_to_use_raises():
config.kv_transfer_config.kv_connector_extra_config.pop("cpu_bytes_to_use", None)
kv_cache_config = _make_kv_cache_config()
with pytest.raises(Exception, match="cpu_bytes_to_use must be specified"):
OffloadingSpecFactory.create_spec(config, kv_cache_config)
_create_spec(config, kv_cache_config)
def test_duplicate_registration_raises():
@@ -253,7 +569,7 @@ def test_build_metric_definitions_below_threshold():
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
config = _make_vllm_config(store_threshold=1)
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
metrics = spec_cls.build_metric_definitions(
config.kv_transfer_config.kv_connector_extra_config
)
@@ -266,7 +582,7 @@ def test_build_metric_definitions_allocation_size_histogram():
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
config = _make_vllm_config(store_threshold=0)
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
metrics = spec_cls.build_metric_definitions(
config.kv_transfer_config.kv_connector_extra_config
)
@@ -291,7 +607,7 @@ def test_build_metric_definitions_returns_counter_at_threshold():
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
config = _make_vllm_config(store_threshold=2)
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
metrics = spec_cls.build_metric_definitions(
config.kv_transfer_config.kv_connector_extra_config
)
+106 -144
View File
@@ -4,80 +4,60 @@
from unittest.mock import MagicMock
import torch
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
MLAAttentionSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_offload.base import (
OffloadingSpec,
make_offload_key,
from vllm.v1.kv_offload.base import OffloadingSpec, make_offload_key
from vllm.v1.kv_offload.config import (
OffloadingCacheConfig,
OffloadingConfig,
OffloadingGroupConfig,
OffloadingModelConfig,
OffloadingParallelConfig,
)
from vllm.v1.kv_offload.file_mapper import FileMapper
# ---------------------------------------------------------------------------
# Shared mocks (mirrors test_fs_tier.py pattern)
# ---------------------------------------------------------------------------
_MOCK_VLLM_CONFIG = MagicMock()
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
_MOCK_KV_CACHE_CONFIG = MagicMock()
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
_MOCK_OFFLOADING_SPEC = MagicMock(spec=OffloadingSpec)
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
"""Helper to create FileMapper with customizable mock config."""
# Create a copy of the mock config to avoid modifying the global one
mock_vllm_config = MagicMock()
mock_vllm_config.model_config.model = kwargs.get("model_name", "test-model")
mock_vllm_config.cache_config.block_size = kwargs.get("hash_block_size", 16)
mock_vllm_config.cache_config.cache_dtype = (
f"torch.{kwargs.get('dtype', 'float16')}"
"""Build a FileMapper from a mocked spec carrying a hand-built config."""
config = OffloadingConfig(
groups=tuple(
OffloadingGroupConfig(
tokens_per_block=tokens_per_block,
layer_names=(layer_name,),
)
for tokens_per_block, layer_name in kwargs.get("groups", ())
),
worker_kv_bytes_per_block=0,
enable_kv_cache_events=False,
extra_config={},
engine_id="test-engine",
model=OffloadingModelConfig(
name=kwargs.get("model_name", "test-model"),
dtype=kwargs.get("dtype", "float16"),
),
cache=OffloadingCacheConfig(
tokens_per_hash=kwargs.get("tokens_per_hash", 16),
blocks_per_chunk=kwargs.get("blocks_per_chunk", 1),
),
parallel=OffloadingParallelConfig(
rank=kwargs.get("rank", 0),
world_size=kwargs.get("world_size", 1),
tp_size=kwargs.get("tp_size", 1),
pp_size=kwargs.get("pp_size", 1),
pcp_size=kwargs.get("pcp_size", 1),
dcp_size=kwargs.get("dcp_size", 1),
data_parallel_index=0,
is_parallelism_agnostic=kwargs.get("is_parallelism_agnostic", False),
),
)
mock_vllm_config.parallel_config.tensor_parallel_size = kwargs.get("tp_size", 1)
mock_vllm_config.parallel_config.pipeline_parallel_size = kwargs.get("pp_size", 1)
mock_vllm_config.parallel_config.prefill_context_parallel_size = kwargs.get(
"pcp_size", 1
)
mock_vllm_config.parallel_config.decode_context_parallel_size = kwargs.get(
"dcp_size", 1
)
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)
mock_vllm_config.use_v2_model_runner = kwargs.get("use_v2_model_runner", False)
mock_kv_cache_config = MagicMock()
mock_kv_cache_config.kv_cache_groups = kwargs.get("kv_cache_groups", [])
mock_offloading_spec = MagicMock(spec=OffloadingSpec)
mock_offloading_spec.vllm_config = mock_vllm_config
mock_offloading_spec.kv_cache_config = mock_kv_cache_config
mock_offloading_spec.block_size_factor = kwargs.get("block_size_factor", 1)
spec = MagicMock(spec=OffloadingSpec)
spec.config = config
return FileMapper.from_offloading_spec(
root_dir=kwargs.get("root_dir", "/tmp/cache"),
offloading_spec=mock_offloading_spec,
gpu_blocks_per_file=mock_offloading_spec.block_size_factor,
offloading_spec=spec,
blocks_per_file=config.cache.blocks_per_chunk,
parallel_agnostic=kwargs.get("parallel_agnostic", False),
)
@@ -92,7 +72,7 @@ def test_get_file_name_full_structure():
Path must match: <base_path>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash_hex>.bin
Concretely:
- The segment immediately after base_path must end with `_r0`
- The segment immediately after base_path must end with `_r3`
- The next segment is the first 3 hex chars of the block hash
- The next segment is <2 hex chars>_g<group_idx>
- The final segment is <full hash hex>.bin
@@ -105,7 +85,7 @@ def test_get_file_name_full_structure():
path = fm.get_file_name(key)
expected_path = (
"/tmp/cache/test-model_588656ebcc66_r3/000/10_g2/0001020304050607.bin"
"/tmp/cache/test-model_42b94bdc9933_r3/000/10_g2/0001020304050607.bin"
)
assert path == expected_path
@@ -114,19 +94,30 @@ def test_get_run_config_fields():
fm = make_mapper_from_offloading_spec(
model_name="my-model",
dtype="bfloat16",
tp_size=2,
tp_size=4,
pp_size=3,
pcp_size=2,
dcp_size=2,
groups=((64, "layer0"),),
tokens_per_hash=64,
blocks_per_chunk=3,
)
cfg = fm.get_run_config()
assert cfg == {
"model_name": "my-model",
"hash_block_size": 16,
"gpu_blocks_per_file": 1,
"tp_size": 2,
"pp_size": 1,
"pcp_size": 1,
"dcp_size": 1,
"tokens_per_hash": 64,
"blocks_per_file": 3,
"tp_size": 4,
"pp_size": 3,
"pcp_size": 2,
"dcp_size": 2,
"dtype": "bfloat16",
"kv_cache_groups": [],
"kv_cache_groups": [
{
"tokens_per_block": 64,
"layer_names": ["layer0"],
}
],
"inference_engine": "vllm",
}
@@ -137,90 +128,61 @@ def test_get_config_file_path():
assert config_path == f"{fm.base_path}/config.json"
# ---------------------------------------------------------------------------
# parallel_agnostic: honored only for a single non-MLA full-attention group
# ---------------------------------------------------------------------------
def _full_attention_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=128, dtype=torch.float32
),
)
def _sliding_window_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=SlidingWindowSpec(
block_size=16,
num_kv_heads=4,
head_size=128,
dtype=torch.float32,
sliding_window=128,
),
)
def test_parallel_agnostic_enabled_for_single_full_attention():
# tp/rank are collapsed out of the namespace so the cache is shared
# across tensor-parallel sizes.
def test_hybrid_file_identity_uses_resolved_tokens_per_hash():
# For heterogeneous groups the namespace records the resolved hash
# granularity (GCD of the group block sizes), which is the actual
# granularity of the offload block hashes.
fm = make_mapper_from_offloading_spec(
tp_size=2,
groups=((12, "full_layer"), (16, "mla_layer")),
tokens_per_hash=4,
)
assert fm.fields["tokens_per_hash"] == 4
assert fm.fields["kv_cache_groups"] == [
{"tokens_per_block": 12, "layer_names": ["full_layer"]},
{"tokens_per_block": 16, "layer_names": ["mla_layer"]},
]
# ---------------------------------------------------------------------------
# parallel_agnostic: opt-in honored only when the config marks the layout
# parallelism-agnostic (predicate computation is covered in test_factory.py)
# ---------------------------------------------------------------------------
def test_parallel_agnostic_collapses_namespace_when_config_allows():
fm = make_mapper_from_offloading_spec(
tp_size=4,
pp_size=3,
pcp_size=2,
dcp_size=2,
rank=1,
kv_cache_groups=[_full_attention_group()],
is_parallelism_agnostic=True,
parallel_agnostic=True,
)
assert fm.fields["tp_size"] == 1
assert fm.fields["pp_size"] == 1
assert fm.fields["pcp_size"] == 1
assert fm.fields["dcp_size"] == 1
assert fm.rank == 0
def test_parallel_agnostic_disabled_for_multiple_groups():
# More than one KV-cache group (hybrid model) => keep per-layout namespacing.
fm = make_mapper_from_offloading_spec(
tp_size=2,
kv_cache_groups=[_full_attention_group(), _full_attention_group()],
parallel_agnostic=True,
)
assert fm.fields["tp_size"] == 2
def test_parallel_agnostic_disabled_for_non_full_attention():
# Single group but not full attention (sliding window) => keep namespacing.
fm = make_mapper_from_offloading_spec(
tp_size=2,
kv_cache_groups=[_sliding_window_group()],
parallel_agnostic=True,
)
assert fm.fields["tp_size"] == 2
def test_parallel_agnostic_excludes_mla():
# MLA latent KV is replicated per rank, so its offloaded blocks are not
# parallelism-invariant: the opt-in must not collapse tp/rank.
group = KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=MLAAttentionSpec(
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
),
)
fm = make_mapper_from_offloading_spec(
tp_size=2, rank=1, kv_cache_groups=[group], parallel_agnostic=True
)
assert fm.fields["tp_size"] == 2
assert fm.rank == 1
def test_parallel_agnostic_disabled_on_v2_model_runner():
# V2's KV layout is not known to be parallelism-invariant: don't collapse.
def test_parallel_agnostic_ignored_when_config_disallows():
fm = make_mapper_from_offloading_spec(
tp_size=2,
rank=1,
kv_cache_groups=[_full_attention_group()],
use_v2_model_runner=True,
is_parallelism_agnostic=False,
parallel_agnostic=True,
)
assert fm.fields["tp_size"] == 2
assert fm.rank == 1
def test_namespace_kept_without_parallel_agnostic_opt_in():
fm = make_mapper_from_offloading_spec(
tp_size=2,
rank=1,
is_parallelism_agnostic=True,
parallel_agnostic=False,
)
assert fm.fields["tp_size"] == 2
assert fm.rank == 1
@@ -47,7 +47,7 @@ class TestDataTransportBase:
def test_config_fingerprint_deterministic(self):
"""Same config fields → same fingerprint."""
view = self._make_view()
fields = {"model": "llama", "dtype": "float16", "block_size_factor": 1}
fields = {"model": "llama", "dtype": "float16", "blocks_per_chunk": 1}
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
t1 = NixlTransport("test:1", view, config_fields=fields)
t2 = NixlTransport("test:2", view, config_fields=fields)
@@ -1419,9 +1419,9 @@ class TestBindHostPortDefaults:
or SimpleNamespace(),
)
spec = SimpleNamespace(
block_size_factor=1,
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(data_parallel_index=dp_index)
blocks_per_chunk=1,
config=SimpleNamespace(
parallel=SimpleNamespace(data_parallel_index=dp_index)
),
)
mgr = P2PSecondaryTierManager(spec, memoryview(b""), **kwargs)
+34 -22
View File
@@ -22,11 +22,18 @@ from vllm.distributed.kv_events import MEDIUM_FS
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingEvent,
OffloadingKVEventsConfig,
OffloadKey,
ReqContext,
ScheduleEndContext,
make_offload_key,
)
from vllm.v1.kv_offload.config import (
OffloadingCacheConfig,
OffloadingConfig,
OffloadingModelConfig,
OffloadingParallelConfig,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.fs.manager import (
FileSystemTierManager,
@@ -41,35 +48,40 @@ _BLOCK_ELEMENTS = 128 * mmap.PAGESIZE # 2MB per block for pagesize 4096.
_DTYPE: torch.dtype = torch.float32
_CTX = ReqContext(req_id="test")
_MOCK_VLLM_CONFIG = MagicMock()
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
_MOCK_KV_CACHE_CONFIG = MagicMock()
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
_MOCK_OFFLOADING_SPEC = MagicMock()
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
def _make_offloading_spec(enable_kv_cache_events: bool) -> MagicMock:
"""Mock spec with an explicit global KV events flag."""
spec = MagicMock()
spec.vllm_config = _MOCK_VLLM_CONFIG
spec.kv_cache_config = _MOCK_KV_CACHE_CONFIG
spec.block_size_factor = 1
spec.kv_events_config.enable_kv_cache_events = enable_kv_cache_events
spec.config = OffloadingConfig(
groups=(),
worker_kv_bytes_per_block=0,
enable_kv_cache_events=enable_kv_cache_events,
extra_config={},
engine_id="test-engine",
model=OffloadingModelConfig(name="test-model", dtype="float32"),
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
parallel=OffloadingParallelConfig(
rank=0,
world_size=1,
tp_size=1,
pp_size=1,
pcp_size=1,
dcp_size=1,
data_parallel_index=0,
is_parallelism_agnostic=False,
),
)
spec.blocks_per_chunk = 1
spec.kv_events_config = OffloadingKVEventsConfig(
enable_kv_cache_events=enable_kv_cache_events,
self_describing_kv_events=False,
)
return spec
_MOCK_OFFLOADING_SPEC = _make_offloading_spec(enable_kv_cache_events=False)
def key(n: int) -> OffloadKey:
return make_offload_key(n.to_bytes(8, "big"), 0)
+30 -15
View File
@@ -19,11 +19,18 @@ import torch
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingKVEventsConfig,
OffloadKey,
ReqContext,
ScheduleEndContext,
make_offload_key,
)
from vllm.v1.kv_offload.config import (
OffloadingCacheConfig,
OffloadingConfig,
OffloadingModelConfig,
OffloadingParallelConfig,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig
from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManager
@@ -33,24 +40,30 @@ from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManag
# ---------------------------------------------------------------------------
def _make_vllm_config():
return SimpleNamespace(
model_config=SimpleNamespace(model="test/model"),
cache_config=SimpleNamespace(block_size=16, cache_dtype="float16"),
parallel_config=SimpleNamespace(
tensor_parallel_size=1,
pipeline_parallel_size=1,
prefill_context_parallel_size=1,
decode_context_parallel_size=1,
def _make_offloading_config(enable_kv_cache_events: bool) -> OffloadingConfig:
return OffloadingConfig(
groups=(),
worker_kv_bytes_per_block=0,
enable_kv_cache_events=enable_kv_cache_events,
extra_config={},
engine_id="test-engine",
model=OffloadingModelConfig(name="test/model", dtype="float16"),
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
parallel=OffloadingParallelConfig(
rank=0,
world_size=1,
tp_size=1,
pp_size=1,
pcp_size=1,
dcp_size=1,
data_parallel_index=0,
is_parallelism_agnostic=False,
),
use_v2_model_runner=False,
)
_OFFLOADING_SPEC = SimpleNamespace(
vllm_config=_make_vllm_config(),
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
config=_make_offloading_config(enable_kv_cache_events=False),
)
_STORE_CONFIG = {
@@ -182,9 +195,11 @@ class MockNixlAgent:
def _make_events_spec(enable_kv_cache_events: bool) -> SimpleNamespace:
"""Offloading spec stub with an explicit global KV events flag."""
return SimpleNamespace(
vllm_config=_make_vllm_config(),
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
kv_events_config=SimpleNamespace(enable_kv_cache_events=enable_kv_cache_events),
config=_make_offloading_config(enable_kv_cache_events),
kv_events_config=OffloadingKVEventsConfig(
enable_kv_cache_events=enable_kv_cache_events,
self_describing_kv_events=False,
),
)