[KV Offload] Define clean backend configuration boundary (#48150)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <cguo51@asu.edu> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
Or Ozeri
OpenAI Codex
parent
8c3393f373
commit
a9531edfa6
@@ -11,17 +11,33 @@ These tests verify:
|
||||
4. Error paths — unregistered specs, missing config, duplicate registration.
|
||||
"""
|
||||
|
||||
from typing import cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.config import KVTransferConfig, ParallelConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
build_offloading_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import OffloadingHistogramMetadata, OffloadingSpec
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
OffloadingHistogramMetadata,
|
||||
OffloadingManager,
|
||||
OffloadingSpec,
|
||||
OffloadingWorker,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
@@ -39,6 +55,17 @@ def restore_registry():
|
||||
OffloadingSpecFactory._registry = original
|
||||
|
||||
|
||||
def _get_extra_config(config: VllmConfig) -> dict:
|
||||
assert config.kv_transfer_config is not None
|
||||
return config.kv_transfer_config.kv_connector_extra_config
|
||||
|
||||
|
||||
def _create_spec(config: VllmConfig, kv_cache_config: KVCacheConfig) -> OffloadingSpec:
|
||||
return OffloadingSpecFactory.create_spec(
|
||||
build_offloading_config(config, kv_cache_config)
|
||||
)
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
spec_name: str | None = "CPUOffloadingSpec",
|
||||
cpu_bytes_to_use: int | None = None,
|
||||
@@ -95,6 +122,46 @@ def _make_vllm_config(
|
||||
)
|
||||
|
||||
|
||||
def _make_layout_vllm_config(
|
||||
spec_name: str = "CPUOffloadingSpec",
|
||||
cpu_bytes_to_use: int | None = None,
|
||||
extra_config: dict | None = None,
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
prefill_context_parallel_size: int = 1,
|
||||
decode_context_parallel_size: int = 1,
|
||||
) -> VllmConfig:
|
||||
config = MagicMock()
|
||||
config.cache_config.block_size = 16
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
config.cache_config.prefix_match_unit = None
|
||||
config.cache_config.cache_dtype = torch.float16
|
||||
config.model_config.model = "test-model"
|
||||
world_size = (
|
||||
tensor_parallel_size * pipeline_parallel_size * prefill_context_parallel_size
|
||||
)
|
||||
with patch.object(current_platform, "device_count", return_value=world_size):
|
||||
config.parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
prefill_context_parallel_size=prefill_context_parallel_size,
|
||||
decode_context_parallel_size=decode_context_parallel_size,
|
||||
)
|
||||
config.kv_events_config = None
|
||||
config.use_v2_model_runner = False
|
||||
|
||||
connector_extra_config = dict(extra_config or {})
|
||||
connector_extra_config["spec_name"] = spec_name
|
||||
if cpu_bytes_to_use is not None:
|
||||
connector_extra_config["cpu_bytes_to_use"] = cpu_bytes_to_use
|
||||
config.kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=connector_extra_config,
|
||||
)
|
||||
return cast(VllmConfig, config)
|
||||
|
||||
|
||||
def _make_kv_cache_config():
|
||||
"""Build a minimal KVCacheConfig with one KV cache tensor."""
|
||||
num_blocks = 16
|
||||
@@ -122,6 +189,78 @@ def _make_kv_cache_config():
|
||||
)
|
||||
|
||||
|
||||
def _make_sizing_kv_cache_config(packed: bool) -> KVCacheConfig:
|
||||
num_blocks = 4
|
||||
if packed:
|
||||
kv_cache_tensors = [
|
||||
KVCacheTensor(
|
||||
size=64,
|
||||
shared_by=[layer_name],
|
||||
block_stride=16,
|
||||
)
|
||||
for layer_name in ("layer0", "layer1")
|
||||
]
|
||||
else:
|
||||
kv_cache_tensors = [
|
||||
KVCacheTensor(size=40, shared_by=["layer0"]),
|
||||
KVCacheTensor(size=24, shared_by=["layer1"]),
|
||||
]
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer0", "layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _make_hybrid_kv_cache_config() -> KVCacheConfig:
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=40, shared_by=["full_layer"]),
|
||||
KVCacheTensor(size=24, shared_by=["mla_layer"]),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full_layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=12,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mla_layer"],
|
||||
MLAAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class SingleArgExternalOffloadingSpec(OffloadingSpec):
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-registration integrity (CI sentinel)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -154,7 +293,7 @@ def test_tiering_spec_registered():
|
||||
def test_get_spec_cls_returns_registered_class():
|
||||
"""Registered spec_name returns correct class."""
|
||||
config = _make_vllm_config(spec_name="CPUOffloadingSpec")
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
assert spec_cls is CPUOffloadingSpec
|
||||
|
||||
|
||||
@@ -162,7 +301,7 @@ def test_get_spec_cls_default_to_cpu():
|
||||
"""Default spec_name (absent from config) resolves to CPUOffloadingSpec."""
|
||||
config = _make_vllm_config(spec_name=None)
|
||||
config.kv_transfer_config.kv_connector_extra_config.pop("spec_name", None)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
assert spec_cls is CPUOffloadingSpec
|
||||
|
||||
|
||||
@@ -176,16 +315,193 @@ def test_create_cpu_offloading_spec_end_to_end():
|
||||
|
||||
Verifies:
|
||||
- cpu_bytes_to_use validation and num_blocks calculation
|
||||
- block_size % hash_block_size assertion
|
||||
- block_size % tokens_per_hash assertion
|
||||
- spec instance is CPUOffloadingSpec
|
||||
"""
|
||||
config = _make_vllm_config(cpu_bytes_to_use=65536)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
spec = OffloadingSpecFactory.create_spec(config, kv_cache_config)
|
||||
spec = _create_spec(config, kv_cache_config)
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.num_blocks > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("packed", [False, True])
|
||||
def test_cpu_spec_sizing_preserves_tensor_layout(packed: bool):
|
||||
cpu_bytes_to_use = 1920
|
||||
config = _make_layout_vllm_config(
|
||||
cpu_bytes_to_use=cpu_bytes_to_use,
|
||||
extra_config={"block_size": 32},
|
||||
tensor_parallel_size=3,
|
||||
pipeline_parallel_size=2,
|
||||
)
|
||||
|
||||
spec = _create_spec(config, _make_sizing_kv_cache_config(packed))
|
||||
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == 192
|
||||
assert spec.num_blocks == cpu_bytes_to_use // 192
|
||||
|
||||
|
||||
def test_cpu_spec_rejects_partially_packed_tensor_layout():
|
||||
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
|
||||
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
|
||||
kv_cache_config.kv_cache_tensors[0].block_stride = 16
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
_create_spec(config, kv_cache_config)
|
||||
|
||||
|
||||
def test_cpu_spec_zero_blocks_skips_tensor_layout_validation():
|
||||
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
|
||||
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
|
||||
kv_cache_config.num_blocks = 0
|
||||
kv_cache_config.kv_cache_tensors[0].block_stride = 16
|
||||
|
||||
spec = _create_spec(config, kv_cache_config)
|
||||
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 0
|
||||
assert spec.kv_bytes_per_chunk == 0
|
||||
assert spec.num_blocks == 0
|
||||
|
||||
|
||||
def test_tiering_spec_aligns_row_size():
|
||||
alignment = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT
|
||||
cpu_bytes_to_use = alignment * 3
|
||||
config = _make_layout_vllm_config(
|
||||
spec_name="TieringOffloadingSpec",
|
||||
cpu_bytes_to_use=cpu_bytes_to_use,
|
||||
extra_config={"block_size": 32},
|
||||
tensor_parallel_size=3,
|
||||
pipeline_parallel_size=2,
|
||||
)
|
||||
|
||||
spec = _create_spec(config, _make_sizing_kv_cache_config(packed=False))
|
||||
|
||||
assert isinstance(spec, TieringOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == alignment
|
||||
assert spec.num_blocks == cpu_bytes_to_use // alignment
|
||||
|
||||
|
||||
def test_offloading_spec_resolves_prefill_context_parallel_block_sizes():
|
||||
config = _make_layout_vllm_config(
|
||||
cpu_bytes_to_use=65536,
|
||||
extra_config={"block_size": 64},
|
||||
prefill_context_parallel_size=2,
|
||||
)
|
||||
|
||||
spec = _create_spec(config, _make_kv_cache_config())
|
||||
|
||||
assert spec.tokens_per_block == (32,)
|
||||
assert spec.tokens_per_hash == 32
|
||||
assert spec.blocks_per_chunk == 2
|
||||
|
||||
|
||||
def test_offloading_config_preserves_data_parallel_index():
|
||||
config = _make_layout_vllm_config()
|
||||
config.parallel_config.data_parallel_index = 2
|
||||
|
||||
offloading_config = build_offloading_config(config, _make_kv_cache_config())
|
||||
|
||||
assert offloading_config.parallel.data_parallel_index == 2
|
||||
|
||||
|
||||
def test_offloading_spec_resolves_heterogeneous_hybrid_block_sizes():
|
||||
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
|
||||
config.cache_config.block_size = 4
|
||||
|
||||
spec = _create_spec(config, _make_hybrid_kv_cache_config())
|
||||
|
||||
assert spec.tokens_per_block == (12, 16)
|
||||
assert spec.tokens_per_hash == 4
|
||||
assert spec.blocks_per_chunk == 1
|
||||
|
||||
|
||||
def _full_attention_spec(block_size: int = 16) -> FullAttentionSpec:
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=4, head_size=128, dtype=torch.float32
|
||||
)
|
||||
|
||||
|
||||
def _parallelism_agnostic(kv_cache_groups: list[KVCacheGroupSpec]) -> bool:
|
||||
config = _make_layout_vllm_config()
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
|
||||
)
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
return offloading_config.parallel.is_parallelism_agnostic
|
||||
|
||||
|
||||
def test_parallelism_agnostic_for_single_full_attention_group():
|
||||
assert _parallelism_agnostic([KVCacheGroupSpec(["l0"], _full_attention_spec())])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_groups",
|
||||
[
|
||||
# MLA latent KV is replicated per rank, never head-sharded.
|
||||
[
|
||||
KVCacheGroupSpec(
|
||||
["l0"],
|
||||
MLAAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
],
|
||||
# Sliding window is not full attention.
|
||||
[
|
||||
KVCacheGroupSpec(
|
||||
["l0"],
|
||||
SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=128,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
)
|
||||
],
|
||||
# Hybrid model: more than one KV cache group.
|
||||
[
|
||||
KVCacheGroupSpec(["l0"], _full_attention_spec()),
|
||||
KVCacheGroupSpec(["l1"], _full_attention_spec()),
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
|
||||
assert not _parallelism_agnostic(kv_cache_groups)
|
||||
|
||||
|
||||
def test_parallelism_agnostic_disabled_on_v2_model_runner():
|
||||
config = _make_layout_vllm_config()
|
||||
config.use_v2_model_runner = True
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["l0"], _full_attention_spec())],
|
||||
)
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
assert not offloading_config.parallel.is_parallelism_agnostic
|
||||
|
||||
|
||||
def test_create_dynamic_spec_receives_translated_config():
|
||||
config = _make_layout_vllm_config(
|
||||
spec_name="SingleArgExternalOffloadingSpec",
|
||||
extra_config={
|
||||
"spec_module_path": "tests.v1.kv_offload.test_factory",
|
||||
},
|
||||
)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
|
||||
spec = OffloadingSpecFactory.create_spec(offloading_config)
|
||||
|
||||
assert isinstance(spec, SingleArgExternalOffloadingSpec)
|
||||
assert spec.config is offloading_config
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dynamic import via spec_module_path
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -205,7 +521,7 @@ def test_dynamic_load_via_spec_module_path():
|
||||
config.kv_transfer_config.kv_connector_extra_config["spec_module_path"] = (
|
||||
"vllm.v1.kv_offload.cpu.spec"
|
||||
)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
assert spec_cls is CPUOffloadingSpec
|
||||
|
||||
|
||||
@@ -218,12 +534,12 @@ def test_unregistered_spec_without_module_path_raises():
|
||||
"""spec_name not in registry + no spec_module_path → ValueError."""
|
||||
config = _make_vllm_config(spec_name="NonexistentSpec")
|
||||
with pytest.raises(ValueError, match="Unsupported spec type"):
|
||||
OffloadingSpecFactory.get_spec_cls(config)
|
||||
OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
|
||||
# create_spec should also fail (calls get_spec_cls internally)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
with pytest.raises(ValueError, match="Unsupported spec type"):
|
||||
OffloadingSpecFactory.create_spec(config, kv_cache_config)
|
||||
_create_spec(config, kv_cache_config)
|
||||
|
||||
|
||||
def test_cpu_spec_missing_cpu_bytes_to_use_raises():
|
||||
@@ -232,7 +548,7 @@ def test_cpu_spec_missing_cpu_bytes_to_use_raises():
|
||||
config.kv_transfer_config.kv_connector_extra_config.pop("cpu_bytes_to_use", None)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
with pytest.raises(Exception, match="cpu_bytes_to_use must be specified"):
|
||||
OffloadingSpecFactory.create_spec(config, kv_cache_config)
|
||||
_create_spec(config, kv_cache_config)
|
||||
|
||||
|
||||
def test_duplicate_registration_raises():
|
||||
@@ -253,7 +569,7 @@ def test_build_metric_definitions_below_threshold():
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=1)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
@@ -266,7 +582,7 @@ def test_build_metric_definitions_allocation_size_histogram():
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=0)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
@@ -291,7 +607,7 @@ def test_build_metric_definitions_returns_counter_at_threshold():
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=2)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user