[KV Offload] Define clean backend configuration boundary (#48150)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <cguo51@asu.edu> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
Or Ozeri
OpenAI Codex
parent
8c3393f373
commit
a9531edfa6
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
NUM_GPU_BLOCKS = [64]
|
||||
NUM_CPU_BLOCKS = [256]
|
||||
GPU_PAGE_SIZES = [512, 1024]
|
||||
BLOCK_SIZE_FACTORS = [1, 3]
|
||||
BLOCKS_PER_CHUNK_VALUES = [1, 3]
|
||||
NUM_TENSORS = [4]
|
||||
SEEDS = [0]
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
@@ -35,7 +35,7 @@ NUM_MAPPINGS_PER_GROUP = [2]
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
|
||||
@@ -48,7 +48,7 @@ def test_transfer(
|
||||
gpu_to_cpu: bool,
|
||||
num_mappings: int,
|
||||
gpu_page_size_bytes: int,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
num_tensors: int,
|
||||
@@ -92,7 +92,7 @@ def test_transfer(
|
||||
mmap_region: SharedOffloadRegion | None = None
|
||||
if use_shared_memory:
|
||||
cpu_page_size = round_up(
|
||||
gpu_page_size_bytes * num_tensors * block_size_factor,
|
||||
gpu_page_size_bytes * num_tensors * blocks_per_chunk,
|
||||
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
|
||||
)
|
||||
mmap_region = SharedOffloadRegion(
|
||||
@@ -105,25 +105,25 @@ def test_transfer(
|
||||
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
|
||||
# select block mappings
|
||||
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * block_size_factor)
|
||||
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
|
||||
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
|
||||
|
||||
# expand cpu blocks to gpu-page granularity for uniform comparison:
|
||||
# each cpu block maps to block_size_factor consecutive sub-blocks
|
||||
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
|
||||
cpu_blocks_expanded = [
|
||||
cpu_block * block_size_factor + j
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
for j in range(block_size_factor)
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
|
||||
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
|
||||
blocks_to_skip = block_size_factor - 1
|
||||
blocks_to_skip = blocks_per_chunk - 1
|
||||
if blocks_to_skip > 0:
|
||||
gpu_blocks = gpu_blocks[blocks_to_skip:]
|
||||
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
|
||||
@@ -214,7 +214,7 @@ def test_transfer(
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
@pytest.mark.parametrize("num_mappings_per_group", NUM_MAPPINGS_PER_GROUP)
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@@ -225,7 +225,7 @@ def test_transfer_multi_group(
|
||||
gpu_to_cpu: bool,
|
||||
num_mappings_per_group: int,
|
||||
gpu_page_size_bytes: int,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
seed: int,
|
||||
@@ -234,7 +234,7 @@ def test_transfer_multi_group(
|
||||
"""Test transfers with three KV cache groups:
|
||||
- Group 0: aligned transfer with num_mappings_per_group blocks
|
||||
- Group 1: zero blocks (empty group)
|
||||
- Group 2: unaligned CPU->GPU transfer (logical_offset=block_size_factor-1,
|
||||
- Group 2: unaligned CPU->GPU transfer (logical_offset=blocks_per_chunk-1,
|
||||
causing the implementation to skip source sub-blocks) with
|
||||
num_mappings_per_group blocks
|
||||
"""
|
||||
@@ -275,7 +275,7 @@ def test_transfer_multi_group(
|
||||
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=canonical_kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
|
||||
@@ -283,7 +283,7 @@ def test_transfer_multi_group(
|
||||
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
|
||||
|
||||
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
|
||||
total_gpu_blocks_needed = total_cpu_blocks * block_size_factor
|
||||
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
|
||||
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
|
||||
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
|
||||
|
||||
@@ -293,7 +293,7 @@ def test_transfer_multi_group(
|
||||
gpu_offset = 0
|
||||
cpu_offset = 0
|
||||
for size in group_sizes_in_cpu_blocks:
|
||||
gpu_count = size * block_size_factor
|
||||
gpu_count = size * blocks_per_chunk
|
||||
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
|
||||
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
|
||||
gpu_offset += gpu_count
|
||||
@@ -302,15 +302,15 @@ def test_transfer_multi_group(
|
||||
# expand cpu blocks to gpu-page granularity
|
||||
cpu_blocks_expanded_per_group = [
|
||||
[
|
||||
cpu_block * block_size_factor + j
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
for j in range(block_size_factor)
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
for cpu_blocks in cpu_blocks_per_group
|
||||
]
|
||||
|
||||
# skip sub-blocks from group 2 to test unaligned transfers.
|
||||
sub_blocks_to_skip = block_size_factor - 1 # e.g. 2 when block_size_factor=3
|
||||
sub_blocks_to_skip = blocks_per_chunk - 1 # e.g. 2 when blocks_per_chunk=3
|
||||
if sub_blocks_to_skip > 0:
|
||||
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
|
||||
sub_blocks_to_skip:-sub_blocks_to_skip
|
||||
@@ -347,7 +347,7 @@ def test_transfer_multi_group(
|
||||
cpu_blocks_expanded_per_group, gpu_blocks_per_group
|
||||
)
|
||||
]
|
||||
num_dst_sub_blocks = num_cpu_blocks * block_size_factor
|
||||
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
|
||||
else:
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
|
||||
@@ -11,17 +11,33 @@ These tests verify:
|
||||
4. Error paths — unregistered specs, missing config, duplicate registration.
|
||||
"""
|
||||
|
||||
from typing import cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.config import KVTransferConfig, ParallelConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
build_offloading_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import OffloadingHistogramMetadata, OffloadingSpec
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
OffloadingHistogramMetadata,
|
||||
OffloadingManager,
|
||||
OffloadingSpec,
|
||||
OffloadingWorker,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
@@ -39,6 +55,17 @@ def restore_registry():
|
||||
OffloadingSpecFactory._registry = original
|
||||
|
||||
|
||||
def _get_extra_config(config: VllmConfig) -> dict:
|
||||
assert config.kv_transfer_config is not None
|
||||
return config.kv_transfer_config.kv_connector_extra_config
|
||||
|
||||
|
||||
def _create_spec(config: VllmConfig, kv_cache_config: KVCacheConfig) -> OffloadingSpec:
|
||||
return OffloadingSpecFactory.create_spec(
|
||||
build_offloading_config(config, kv_cache_config)
|
||||
)
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
spec_name: str | None = "CPUOffloadingSpec",
|
||||
cpu_bytes_to_use: int | None = None,
|
||||
@@ -95,6 +122,46 @@ def _make_vllm_config(
|
||||
)
|
||||
|
||||
|
||||
def _make_layout_vllm_config(
|
||||
spec_name: str = "CPUOffloadingSpec",
|
||||
cpu_bytes_to_use: int | None = None,
|
||||
extra_config: dict | None = None,
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
prefill_context_parallel_size: int = 1,
|
||||
decode_context_parallel_size: int = 1,
|
||||
) -> VllmConfig:
|
||||
config = MagicMock()
|
||||
config.cache_config.block_size = 16
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
config.cache_config.prefix_match_unit = None
|
||||
config.cache_config.cache_dtype = torch.float16
|
||||
config.model_config.model = "test-model"
|
||||
world_size = (
|
||||
tensor_parallel_size * pipeline_parallel_size * prefill_context_parallel_size
|
||||
)
|
||||
with patch.object(current_platform, "device_count", return_value=world_size):
|
||||
config.parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
prefill_context_parallel_size=prefill_context_parallel_size,
|
||||
decode_context_parallel_size=decode_context_parallel_size,
|
||||
)
|
||||
config.kv_events_config = None
|
||||
config.use_v2_model_runner = False
|
||||
|
||||
connector_extra_config = dict(extra_config or {})
|
||||
connector_extra_config["spec_name"] = spec_name
|
||||
if cpu_bytes_to_use is not None:
|
||||
connector_extra_config["cpu_bytes_to_use"] = cpu_bytes_to_use
|
||||
config.kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=connector_extra_config,
|
||||
)
|
||||
return cast(VllmConfig, config)
|
||||
|
||||
|
||||
def _make_kv_cache_config():
|
||||
"""Build a minimal KVCacheConfig with one KV cache tensor."""
|
||||
num_blocks = 16
|
||||
@@ -122,6 +189,78 @@ def _make_kv_cache_config():
|
||||
)
|
||||
|
||||
|
||||
def _make_sizing_kv_cache_config(packed: bool) -> KVCacheConfig:
|
||||
num_blocks = 4
|
||||
if packed:
|
||||
kv_cache_tensors = [
|
||||
KVCacheTensor(
|
||||
size=64,
|
||||
shared_by=[layer_name],
|
||||
block_stride=16,
|
||||
)
|
||||
for layer_name in ("layer0", "layer1")
|
||||
]
|
||||
else:
|
||||
kv_cache_tensors = [
|
||||
KVCacheTensor(size=40, shared_by=["layer0"]),
|
||||
KVCacheTensor(size=24, shared_by=["layer1"]),
|
||||
]
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer0", "layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _make_hybrid_kv_cache_config() -> KVCacheConfig:
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=40, shared_by=["full_layer"]),
|
||||
KVCacheTensor(size=24, shared_by=["mla_layer"]),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full_layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=12,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mla_layer"],
|
||||
MLAAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class SingleArgExternalOffloadingSpec(OffloadingSpec):
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-registration integrity (CI sentinel)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -154,7 +293,7 @@ def test_tiering_spec_registered():
|
||||
def test_get_spec_cls_returns_registered_class():
|
||||
"""Registered spec_name returns correct class."""
|
||||
config = _make_vllm_config(spec_name="CPUOffloadingSpec")
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
assert spec_cls is CPUOffloadingSpec
|
||||
|
||||
|
||||
@@ -162,7 +301,7 @@ def test_get_spec_cls_default_to_cpu():
|
||||
"""Default spec_name (absent from config) resolves to CPUOffloadingSpec."""
|
||||
config = _make_vllm_config(spec_name=None)
|
||||
config.kv_transfer_config.kv_connector_extra_config.pop("spec_name", None)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
assert spec_cls is CPUOffloadingSpec
|
||||
|
||||
|
||||
@@ -176,16 +315,193 @@ def test_create_cpu_offloading_spec_end_to_end():
|
||||
|
||||
Verifies:
|
||||
- cpu_bytes_to_use validation and num_blocks calculation
|
||||
- block_size % hash_block_size assertion
|
||||
- block_size % tokens_per_hash assertion
|
||||
- spec instance is CPUOffloadingSpec
|
||||
"""
|
||||
config = _make_vllm_config(cpu_bytes_to_use=65536)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
spec = OffloadingSpecFactory.create_spec(config, kv_cache_config)
|
||||
spec = _create_spec(config, kv_cache_config)
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.num_blocks > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("packed", [False, True])
|
||||
def test_cpu_spec_sizing_preserves_tensor_layout(packed: bool):
|
||||
cpu_bytes_to_use = 1920
|
||||
config = _make_layout_vllm_config(
|
||||
cpu_bytes_to_use=cpu_bytes_to_use,
|
||||
extra_config={"block_size": 32},
|
||||
tensor_parallel_size=3,
|
||||
pipeline_parallel_size=2,
|
||||
)
|
||||
|
||||
spec = _create_spec(config, _make_sizing_kv_cache_config(packed))
|
||||
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == 192
|
||||
assert spec.num_blocks == cpu_bytes_to_use // 192
|
||||
|
||||
|
||||
def test_cpu_spec_rejects_partially_packed_tensor_layout():
|
||||
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
|
||||
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
|
||||
kv_cache_config.kv_cache_tensors[0].block_stride = 16
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
_create_spec(config, kv_cache_config)
|
||||
|
||||
|
||||
def test_cpu_spec_zero_blocks_skips_tensor_layout_validation():
|
||||
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
|
||||
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
|
||||
kv_cache_config.num_blocks = 0
|
||||
kv_cache_config.kv_cache_tensors[0].block_stride = 16
|
||||
|
||||
spec = _create_spec(config, kv_cache_config)
|
||||
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 0
|
||||
assert spec.kv_bytes_per_chunk == 0
|
||||
assert spec.num_blocks == 0
|
||||
|
||||
|
||||
def test_tiering_spec_aligns_row_size():
|
||||
alignment = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT
|
||||
cpu_bytes_to_use = alignment * 3
|
||||
config = _make_layout_vllm_config(
|
||||
spec_name="TieringOffloadingSpec",
|
||||
cpu_bytes_to_use=cpu_bytes_to_use,
|
||||
extra_config={"block_size": 32},
|
||||
tensor_parallel_size=3,
|
||||
pipeline_parallel_size=2,
|
||||
)
|
||||
|
||||
spec = _create_spec(config, _make_sizing_kv_cache_config(packed=False))
|
||||
|
||||
assert isinstance(spec, TieringOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == alignment
|
||||
assert spec.num_blocks == cpu_bytes_to_use // alignment
|
||||
|
||||
|
||||
def test_offloading_spec_resolves_prefill_context_parallel_block_sizes():
|
||||
config = _make_layout_vllm_config(
|
||||
cpu_bytes_to_use=65536,
|
||||
extra_config={"block_size": 64},
|
||||
prefill_context_parallel_size=2,
|
||||
)
|
||||
|
||||
spec = _create_spec(config, _make_kv_cache_config())
|
||||
|
||||
assert spec.tokens_per_block == (32,)
|
||||
assert spec.tokens_per_hash == 32
|
||||
assert spec.blocks_per_chunk == 2
|
||||
|
||||
|
||||
def test_offloading_config_preserves_data_parallel_index():
|
||||
config = _make_layout_vllm_config()
|
||||
config.parallel_config.data_parallel_index = 2
|
||||
|
||||
offloading_config = build_offloading_config(config, _make_kv_cache_config())
|
||||
|
||||
assert offloading_config.parallel.data_parallel_index == 2
|
||||
|
||||
|
||||
def test_offloading_spec_resolves_heterogeneous_hybrid_block_sizes():
|
||||
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
|
||||
config.cache_config.block_size = 4
|
||||
|
||||
spec = _create_spec(config, _make_hybrid_kv_cache_config())
|
||||
|
||||
assert spec.tokens_per_block == (12, 16)
|
||||
assert spec.tokens_per_hash == 4
|
||||
assert spec.blocks_per_chunk == 1
|
||||
|
||||
|
||||
def _full_attention_spec(block_size: int = 16) -> FullAttentionSpec:
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=4, head_size=128, dtype=torch.float32
|
||||
)
|
||||
|
||||
|
||||
def _parallelism_agnostic(kv_cache_groups: list[KVCacheGroupSpec]) -> bool:
|
||||
config = _make_layout_vllm_config()
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
|
||||
)
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
return offloading_config.parallel.is_parallelism_agnostic
|
||||
|
||||
|
||||
def test_parallelism_agnostic_for_single_full_attention_group():
|
||||
assert _parallelism_agnostic([KVCacheGroupSpec(["l0"], _full_attention_spec())])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_groups",
|
||||
[
|
||||
# MLA latent KV is replicated per rank, never head-sharded.
|
||||
[
|
||||
KVCacheGroupSpec(
|
||||
["l0"],
|
||||
MLAAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
],
|
||||
# Sliding window is not full attention.
|
||||
[
|
||||
KVCacheGroupSpec(
|
||||
["l0"],
|
||||
SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=128,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
)
|
||||
],
|
||||
# Hybrid model: more than one KV cache group.
|
||||
[
|
||||
KVCacheGroupSpec(["l0"], _full_attention_spec()),
|
||||
KVCacheGroupSpec(["l1"], _full_attention_spec()),
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
|
||||
assert not _parallelism_agnostic(kv_cache_groups)
|
||||
|
||||
|
||||
def test_parallelism_agnostic_disabled_on_v2_model_runner():
|
||||
config = _make_layout_vllm_config()
|
||||
config.use_v2_model_runner = True
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["l0"], _full_attention_spec())],
|
||||
)
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
assert not offloading_config.parallel.is_parallelism_agnostic
|
||||
|
||||
|
||||
def test_create_dynamic_spec_receives_translated_config():
|
||||
config = _make_layout_vllm_config(
|
||||
spec_name="SingleArgExternalOffloadingSpec",
|
||||
extra_config={
|
||||
"spec_module_path": "tests.v1.kv_offload.test_factory",
|
||||
},
|
||||
)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
|
||||
spec = OffloadingSpecFactory.create_spec(offloading_config)
|
||||
|
||||
assert isinstance(spec, SingleArgExternalOffloadingSpec)
|
||||
assert spec.config is offloading_config
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dynamic import via spec_module_path
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -205,7 +521,7 @@ def test_dynamic_load_via_spec_module_path():
|
||||
config.kv_transfer_config.kv_connector_extra_config["spec_module_path"] = (
|
||||
"vllm.v1.kv_offload.cpu.spec"
|
||||
)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
assert spec_cls is CPUOffloadingSpec
|
||||
|
||||
|
||||
@@ -218,12 +534,12 @@ def test_unregistered_spec_without_module_path_raises():
|
||||
"""spec_name not in registry + no spec_module_path → ValueError."""
|
||||
config = _make_vllm_config(spec_name="NonexistentSpec")
|
||||
with pytest.raises(ValueError, match="Unsupported spec type"):
|
||||
OffloadingSpecFactory.get_spec_cls(config)
|
||||
OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
|
||||
# create_spec should also fail (calls get_spec_cls internally)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
with pytest.raises(ValueError, match="Unsupported spec type"):
|
||||
OffloadingSpecFactory.create_spec(config, kv_cache_config)
|
||||
_create_spec(config, kv_cache_config)
|
||||
|
||||
|
||||
def test_cpu_spec_missing_cpu_bytes_to_use_raises():
|
||||
@@ -232,7 +548,7 @@ def test_cpu_spec_missing_cpu_bytes_to_use_raises():
|
||||
config.kv_transfer_config.kv_connector_extra_config.pop("cpu_bytes_to_use", None)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
with pytest.raises(Exception, match="cpu_bytes_to_use must be specified"):
|
||||
OffloadingSpecFactory.create_spec(config, kv_cache_config)
|
||||
_create_spec(config, kv_cache_config)
|
||||
|
||||
|
||||
def test_duplicate_registration_raises():
|
||||
@@ -253,7 +569,7 @@ def test_build_metric_definitions_below_threshold():
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=1)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
@@ -266,7 +582,7 @@ def test_build_metric_definitions_allocation_size_histogram():
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=0)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
@@ -291,7 +607,7 @@ def test_build_metric_definitions_returns_counter_at_threshold():
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=2)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
|
||||
@@ -4,80 +4,60 @@
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingSpec,
|
||||
make_offload_key,
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec, make_offload_key
|
||||
from vllm.v1.kv_offload.config import (
|
||||
OffloadingCacheConfig,
|
||||
OffloadingConfig,
|
||||
OffloadingGroupConfig,
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared mocks (mirrors test_fs_tier.py pattern)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MOCK_VLLM_CONFIG = MagicMock()
|
||||
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
|
||||
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
|
||||
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
|
||||
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
|
||||
|
||||
_MOCK_KV_CACHE_CONFIG = MagicMock()
|
||||
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
|
||||
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock(spec=OffloadingSpec)
|
||||
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
|
||||
"""Helper to create FileMapper with customizable mock config."""
|
||||
# Create a copy of the mock config to avoid modifying the global one
|
||||
mock_vllm_config = MagicMock()
|
||||
mock_vllm_config.model_config.model = kwargs.get("model_name", "test-model")
|
||||
mock_vllm_config.cache_config.block_size = kwargs.get("hash_block_size", 16)
|
||||
mock_vllm_config.cache_config.cache_dtype = (
|
||||
f"torch.{kwargs.get('dtype', 'float16')}"
|
||||
"""Build a FileMapper from a mocked spec carrying a hand-built config."""
|
||||
config = OffloadingConfig(
|
||||
groups=tuple(
|
||||
OffloadingGroupConfig(
|
||||
tokens_per_block=tokens_per_block,
|
||||
layer_names=(layer_name,),
|
||||
)
|
||||
for tokens_per_block, layer_name in kwargs.get("groups", ())
|
||||
),
|
||||
worker_kv_bytes_per_block=0,
|
||||
enable_kv_cache_events=False,
|
||||
extra_config={},
|
||||
engine_id="test-engine",
|
||||
model=OffloadingModelConfig(
|
||||
name=kwargs.get("model_name", "test-model"),
|
||||
dtype=kwargs.get("dtype", "float16"),
|
||||
),
|
||||
cache=OffloadingCacheConfig(
|
||||
tokens_per_hash=kwargs.get("tokens_per_hash", 16),
|
||||
blocks_per_chunk=kwargs.get("blocks_per_chunk", 1),
|
||||
),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=kwargs.get("rank", 0),
|
||||
world_size=kwargs.get("world_size", 1),
|
||||
tp_size=kwargs.get("tp_size", 1),
|
||||
pp_size=kwargs.get("pp_size", 1),
|
||||
pcp_size=kwargs.get("pcp_size", 1),
|
||||
dcp_size=kwargs.get("dcp_size", 1),
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=kwargs.get("is_parallelism_agnostic", False),
|
||||
),
|
||||
)
|
||||
mock_vllm_config.parallel_config.tensor_parallel_size = kwargs.get("tp_size", 1)
|
||||
mock_vllm_config.parallel_config.pipeline_parallel_size = kwargs.get("pp_size", 1)
|
||||
mock_vllm_config.parallel_config.prefill_context_parallel_size = kwargs.get(
|
||||
"pcp_size", 1
|
||||
)
|
||||
mock_vllm_config.parallel_config.decode_context_parallel_size = kwargs.get(
|
||||
"dcp_size", 1
|
||||
)
|
||||
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)
|
||||
mock_vllm_config.use_v2_model_runner = kwargs.get("use_v2_model_runner", False)
|
||||
|
||||
mock_kv_cache_config = MagicMock()
|
||||
mock_kv_cache_config.kv_cache_groups = kwargs.get("kv_cache_groups", [])
|
||||
|
||||
mock_offloading_spec = MagicMock(spec=OffloadingSpec)
|
||||
mock_offloading_spec.vllm_config = mock_vllm_config
|
||||
mock_offloading_spec.kv_cache_config = mock_kv_cache_config
|
||||
mock_offloading_spec.block_size_factor = kwargs.get("block_size_factor", 1)
|
||||
|
||||
spec = MagicMock(spec=OffloadingSpec)
|
||||
spec.config = config
|
||||
return FileMapper.from_offloading_spec(
|
||||
root_dir=kwargs.get("root_dir", "/tmp/cache"),
|
||||
offloading_spec=mock_offloading_spec,
|
||||
gpu_blocks_per_file=mock_offloading_spec.block_size_factor,
|
||||
offloading_spec=spec,
|
||||
blocks_per_file=config.cache.blocks_per_chunk,
|
||||
parallel_agnostic=kwargs.get("parallel_agnostic", False),
|
||||
)
|
||||
|
||||
@@ -92,7 +72,7 @@ def test_get_file_name_full_structure():
|
||||
Path must match: <base_path>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash_hex>.bin
|
||||
|
||||
Concretely:
|
||||
- The segment immediately after base_path must end with `_r0`
|
||||
- The segment immediately after base_path must end with `_r3`
|
||||
- The next segment is the first 3 hex chars of the block hash
|
||||
- The next segment is <2 hex chars>_g<group_idx>
|
||||
- The final segment is <full hash hex>.bin
|
||||
@@ -105,7 +85,7 @@ def test_get_file_name_full_structure():
|
||||
path = fm.get_file_name(key)
|
||||
|
||||
expected_path = (
|
||||
"/tmp/cache/test-model_588656ebcc66_r3/000/10_g2/0001020304050607.bin"
|
||||
"/tmp/cache/test-model_42b94bdc9933_r3/000/10_g2/0001020304050607.bin"
|
||||
)
|
||||
assert path == expected_path
|
||||
|
||||
@@ -114,19 +94,30 @@ def test_get_run_config_fields():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
model_name="my-model",
|
||||
dtype="bfloat16",
|
||||
tp_size=2,
|
||||
tp_size=4,
|
||||
pp_size=3,
|
||||
pcp_size=2,
|
||||
dcp_size=2,
|
||||
groups=((64, "layer0"),),
|
||||
tokens_per_hash=64,
|
||||
blocks_per_chunk=3,
|
||||
)
|
||||
cfg = fm.get_run_config()
|
||||
assert cfg == {
|
||||
"model_name": "my-model",
|
||||
"hash_block_size": 16,
|
||||
"gpu_blocks_per_file": 1,
|
||||
"tp_size": 2,
|
||||
"pp_size": 1,
|
||||
"pcp_size": 1,
|
||||
"dcp_size": 1,
|
||||
"tokens_per_hash": 64,
|
||||
"blocks_per_file": 3,
|
||||
"tp_size": 4,
|
||||
"pp_size": 3,
|
||||
"pcp_size": 2,
|
||||
"dcp_size": 2,
|
||||
"dtype": "bfloat16",
|
||||
"kv_cache_groups": [],
|
||||
"kv_cache_groups": [
|
||||
{
|
||||
"tokens_per_block": 64,
|
||||
"layer_names": ["layer0"],
|
||||
}
|
||||
],
|
||||
"inference_engine": "vllm",
|
||||
}
|
||||
|
||||
@@ -137,90 +128,61 @@ def test_get_config_file_path():
|
||||
assert config_path == f"{fm.base_path}/config.json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parallel_agnostic: honored only for a single non-MLA full-attention group
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _full_attention_group() -> KVCacheGroupSpec:
|
||||
return KVCacheGroupSpec(
|
||||
layer_names=["layer0"],
|
||||
kv_cache_spec=FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=128, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _sliding_window_group() -> KVCacheGroupSpec:
|
||||
return KVCacheGroupSpec(
|
||||
layer_names=["layer0"],
|
||||
kv_cache_spec=SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=128,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_parallel_agnostic_enabled_for_single_full_attention():
|
||||
# tp/rank are collapsed out of the namespace so the cache is shared
|
||||
# across tensor-parallel sizes.
|
||||
def test_hybrid_file_identity_uses_resolved_tokens_per_hash():
|
||||
# For heterogeneous groups the namespace records the resolved hash
|
||||
# granularity (GCD of the group block sizes), which is the actual
|
||||
# granularity of the offload block hashes.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
groups=((12, "full_layer"), (16, "mla_layer")),
|
||||
tokens_per_hash=4,
|
||||
)
|
||||
assert fm.fields["tokens_per_hash"] == 4
|
||||
assert fm.fields["kv_cache_groups"] == [
|
||||
{"tokens_per_block": 12, "layer_names": ["full_layer"]},
|
||||
{"tokens_per_block": 16, "layer_names": ["mla_layer"]},
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parallel_agnostic: opt-in honored only when the config marks the layout
|
||||
# parallelism-agnostic (predicate computation is covered in test_factory.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parallel_agnostic_collapses_namespace_when_config_allows():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=4,
|
||||
pp_size=3,
|
||||
pcp_size=2,
|
||||
dcp_size=2,
|
||||
rank=1,
|
||||
kv_cache_groups=[_full_attention_group()],
|
||||
is_parallelism_agnostic=True,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 1
|
||||
assert fm.fields["pp_size"] == 1
|
||||
assert fm.fields["pcp_size"] == 1
|
||||
assert fm.fields["dcp_size"] == 1
|
||||
assert fm.rank == 0
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_for_multiple_groups():
|
||||
# More than one KV-cache group (hybrid model) => keep per-layout namespacing.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
kv_cache_groups=[_full_attention_group(), _full_attention_group()],
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_for_non_full_attention():
|
||||
# Single group but not full attention (sliding window) => keep namespacing.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
kv_cache_groups=[_sliding_window_group()],
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
|
||||
|
||||
def test_parallel_agnostic_excludes_mla():
|
||||
# MLA latent KV is replicated per rank, so its offloaded blocks are not
|
||||
# parallelism-invariant: the opt-in must not collapse tp/rank.
|
||||
group = KVCacheGroupSpec(
|
||||
layer_names=["layer0"],
|
||||
kv_cache_spec=MLAAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2, rank=1, kv_cache_groups=[group], parallel_agnostic=True
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_on_v2_model_runner():
|
||||
# V2's KV layout is not known to be parallelism-invariant: don't collapse.
|
||||
def test_parallel_agnostic_ignored_when_config_disallows():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
rank=1,
|
||||
kv_cache_groups=[_full_attention_group()],
|
||||
use_v2_model_runner=True,
|
||||
is_parallelism_agnostic=False,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
|
||||
def test_namespace_kept_without_parallel_agnostic_opt_in():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
rank=1,
|
||||
is_parallelism_agnostic=True,
|
||||
parallel_agnostic=False,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
@@ -47,7 +47,7 @@ class TestDataTransportBase:
|
||||
def test_config_fingerprint_deterministic(self):
|
||||
"""Same config fields → same fingerprint."""
|
||||
view = self._make_view()
|
||||
fields = {"model": "llama", "dtype": "float16", "block_size_factor": 1}
|
||||
fields = {"model": "llama", "dtype": "float16", "blocks_per_chunk": 1}
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
t1 = NixlTransport("test:1", view, config_fields=fields)
|
||||
t2 = NixlTransport("test:2", view, config_fields=fields)
|
||||
|
||||
@@ -1419,9 +1419,9 @@ class TestBindHostPortDefaults:
|
||||
or SimpleNamespace(),
|
||||
)
|
||||
spec = SimpleNamespace(
|
||||
block_size_factor=1,
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(data_parallel_index=dp_index)
|
||||
blocks_per_chunk=1,
|
||||
config=SimpleNamespace(
|
||||
parallel=SimpleNamespace(data_parallel_index=dp_index)
|
||||
),
|
||||
)
|
||||
mgr = P2PSecondaryTierManager(spec, memoryview(b""), **kwargs)
|
||||
|
||||
@@ -22,11 +22,18 @@ from vllm.distributed.kv_events import MEDIUM_FS
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
ScheduleEndContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import (
|
||||
OffloadingCacheConfig,
|
||||
OffloadingConfig,
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
||||
from vllm.v1.kv_offload.tiering.fs.manager import (
|
||||
FileSystemTierManager,
|
||||
@@ -41,35 +48,40 @@ _BLOCK_ELEMENTS = 128 * mmap.PAGESIZE # 2MB per block for pagesize 4096.
|
||||
_DTYPE: torch.dtype = torch.float32
|
||||
_CTX = ReqContext(req_id="test")
|
||||
|
||||
_MOCK_VLLM_CONFIG = MagicMock()
|
||||
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
|
||||
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
|
||||
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
|
||||
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
|
||||
|
||||
_MOCK_KV_CACHE_CONFIG = MagicMock()
|
||||
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
|
||||
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
|
||||
|
||||
|
||||
def _make_offloading_spec(enable_kv_cache_events: bool) -> MagicMock:
|
||||
"""Mock spec with an explicit global KV events flag."""
|
||||
spec = MagicMock()
|
||||
spec.vllm_config = _MOCK_VLLM_CONFIG
|
||||
spec.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
spec.block_size_factor = 1
|
||||
spec.kv_events_config.enable_kv_cache_events = enable_kv_cache_events
|
||||
spec.config = OffloadingConfig(
|
||||
groups=(),
|
||||
worker_kv_bytes_per_block=0,
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
extra_config={},
|
||||
engine_id="test-engine",
|
||||
model=OffloadingModelConfig(name="test-model", dtype="float32"),
|
||||
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=0,
|
||||
world_size=1,
|
||||
tp_size=1,
|
||||
pp_size=1,
|
||||
pcp_size=1,
|
||||
dcp_size=1,
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=False,
|
||||
),
|
||||
)
|
||||
spec.blocks_per_chunk = 1
|
||||
spec.kv_events_config = OffloadingKVEventsConfig(
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
self_describing_kv_events=False,
|
||||
)
|
||||
return spec
|
||||
|
||||
|
||||
_MOCK_OFFLOADING_SPEC = _make_offloading_spec(enable_kv_cache_events=False)
|
||||
|
||||
|
||||
def key(n: int) -> OffloadKey:
|
||||
return make_offload_key(n.to_bytes(8, "big"), 0)
|
||||
|
||||
|
||||
@@ -19,11 +19,18 @@ import torch
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
ScheduleEndContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import (
|
||||
OffloadingCacheConfig,
|
||||
OffloadingConfig,
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
||||
from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig
|
||||
from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManager
|
||||
@@ -33,24 +40,30 @@ from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManag
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_vllm_config():
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(model="test/model"),
|
||||
cache_config=SimpleNamespace(block_size=16, cache_dtype="float16"),
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
decode_context_parallel_size=1,
|
||||
def _make_offloading_config(enable_kv_cache_events: bool) -> OffloadingConfig:
|
||||
return OffloadingConfig(
|
||||
groups=(),
|
||||
worker_kv_bytes_per_block=0,
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
extra_config={},
|
||||
engine_id="test-engine",
|
||||
model=OffloadingModelConfig(name="test/model", dtype="float16"),
|
||||
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=0,
|
||||
world_size=1,
|
||||
tp_size=1,
|
||||
pp_size=1,
|
||||
pcp_size=1,
|
||||
dcp_size=1,
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=False,
|
||||
),
|
||||
use_v2_model_runner=False,
|
||||
)
|
||||
|
||||
|
||||
_OFFLOADING_SPEC = SimpleNamespace(
|
||||
vllm_config=_make_vllm_config(),
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
|
||||
config=_make_offloading_config(enable_kv_cache_events=False),
|
||||
)
|
||||
|
||||
_STORE_CONFIG = {
|
||||
@@ -182,9 +195,11 @@ class MockNixlAgent:
|
||||
def _make_events_spec(enable_kv_cache_events: bool) -> SimpleNamespace:
|
||||
"""Offloading spec stub with an explicit global KV events flag."""
|
||||
return SimpleNamespace(
|
||||
vllm_config=_make_vllm_config(),
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=enable_kv_cache_events),
|
||||
config=_make_offloading_config(enable_kv_cache_events),
|
||||
kv_events_config=OffloadingKVEventsConfig(
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
self_describing_kv_events=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user