[KV Offload] Define clean backend configuration boundary (#48150)

Signed-off-by: Change72 <changg@nvidia.com>
Signed-off-by: Chang Guo <cguo51@asu.edu>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Chang Guo
2026-07-16 13:27:05 +03:00
committed by GitHub
co-authored by Codex Claude Fable 5 Or Ozeri OpenAI Codex
parent 8c3393f373
commit a9531edfa6
27 changed files with 1287 additions and 855 deletions
+106 -144
View File
@@ -4,80 +4,60 @@
from unittest.mock import MagicMock
import torch
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
MLAAttentionSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_offload.base import (
OffloadingSpec,
make_offload_key,
from vllm.v1.kv_offload.base import OffloadingSpec, make_offload_key
from vllm.v1.kv_offload.config import (
OffloadingCacheConfig,
OffloadingConfig,
OffloadingGroupConfig,
OffloadingModelConfig,
OffloadingParallelConfig,
)
from vllm.v1.kv_offload.file_mapper import FileMapper
# ---------------------------------------------------------------------------
# Shared mocks (mirrors test_fs_tier.py pattern)
# ---------------------------------------------------------------------------
_MOCK_VLLM_CONFIG = MagicMock()
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
_MOCK_KV_CACHE_CONFIG = MagicMock()
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
_MOCK_OFFLOADING_SPEC = MagicMock(spec=OffloadingSpec)
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
"""Helper to create FileMapper with customizable mock config."""
# Create a copy of the mock config to avoid modifying the global one
mock_vllm_config = MagicMock()
mock_vllm_config.model_config.model = kwargs.get("model_name", "test-model")
mock_vllm_config.cache_config.block_size = kwargs.get("hash_block_size", 16)
mock_vllm_config.cache_config.cache_dtype = (
f"torch.{kwargs.get('dtype', 'float16')}"
"""Build a FileMapper from a mocked spec carrying a hand-built config."""
config = OffloadingConfig(
groups=tuple(
OffloadingGroupConfig(
tokens_per_block=tokens_per_block,
layer_names=(layer_name,),
)
for tokens_per_block, layer_name in kwargs.get("groups", ())
),
worker_kv_bytes_per_block=0,
enable_kv_cache_events=False,
extra_config={},
engine_id="test-engine",
model=OffloadingModelConfig(
name=kwargs.get("model_name", "test-model"),
dtype=kwargs.get("dtype", "float16"),
),
cache=OffloadingCacheConfig(
tokens_per_hash=kwargs.get("tokens_per_hash", 16),
blocks_per_chunk=kwargs.get("blocks_per_chunk", 1),
),
parallel=OffloadingParallelConfig(
rank=kwargs.get("rank", 0),
world_size=kwargs.get("world_size", 1),
tp_size=kwargs.get("tp_size", 1),
pp_size=kwargs.get("pp_size", 1),
pcp_size=kwargs.get("pcp_size", 1),
dcp_size=kwargs.get("dcp_size", 1),
data_parallel_index=0,
is_parallelism_agnostic=kwargs.get("is_parallelism_agnostic", False),
),
)
mock_vllm_config.parallel_config.tensor_parallel_size = kwargs.get("tp_size", 1)
mock_vllm_config.parallel_config.pipeline_parallel_size = kwargs.get("pp_size", 1)
mock_vllm_config.parallel_config.prefill_context_parallel_size = kwargs.get(
"pcp_size", 1
)
mock_vllm_config.parallel_config.decode_context_parallel_size = kwargs.get(
"dcp_size", 1
)
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)
mock_vllm_config.use_v2_model_runner = kwargs.get("use_v2_model_runner", False)
mock_kv_cache_config = MagicMock()
mock_kv_cache_config.kv_cache_groups = kwargs.get("kv_cache_groups", [])
mock_offloading_spec = MagicMock(spec=OffloadingSpec)
mock_offloading_spec.vllm_config = mock_vllm_config
mock_offloading_spec.kv_cache_config = mock_kv_cache_config
mock_offloading_spec.block_size_factor = kwargs.get("block_size_factor", 1)
spec = MagicMock(spec=OffloadingSpec)
spec.config = config
return FileMapper.from_offloading_spec(
root_dir=kwargs.get("root_dir", "/tmp/cache"),
offloading_spec=mock_offloading_spec,
gpu_blocks_per_file=mock_offloading_spec.block_size_factor,
offloading_spec=spec,
blocks_per_file=config.cache.blocks_per_chunk,
parallel_agnostic=kwargs.get("parallel_agnostic", False),
)
@@ -92,7 +72,7 @@ def test_get_file_name_full_structure():
Path must match: <base_path>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash_hex>.bin
Concretely:
- The segment immediately after base_path must end with `_r0`
- The segment immediately after base_path must end with `_r3`
- The next segment is the first 3 hex chars of the block hash
- The next segment is <2 hex chars>_g<group_idx>
- The final segment is <full hash hex>.bin
@@ -105,7 +85,7 @@ def test_get_file_name_full_structure():
path = fm.get_file_name(key)
expected_path = (
"/tmp/cache/test-model_588656ebcc66_r3/000/10_g2/0001020304050607.bin"
"/tmp/cache/test-model_42b94bdc9933_r3/000/10_g2/0001020304050607.bin"
)
assert path == expected_path
@@ -114,19 +94,30 @@ def test_get_run_config_fields():
fm = make_mapper_from_offloading_spec(
model_name="my-model",
dtype="bfloat16",
tp_size=2,
tp_size=4,
pp_size=3,
pcp_size=2,
dcp_size=2,
groups=((64, "layer0"),),
tokens_per_hash=64,
blocks_per_chunk=3,
)
cfg = fm.get_run_config()
assert cfg == {
"model_name": "my-model",
"hash_block_size": 16,
"gpu_blocks_per_file": 1,
"tp_size": 2,
"pp_size": 1,
"pcp_size": 1,
"dcp_size": 1,
"tokens_per_hash": 64,
"blocks_per_file": 3,
"tp_size": 4,
"pp_size": 3,
"pcp_size": 2,
"dcp_size": 2,
"dtype": "bfloat16",
"kv_cache_groups": [],
"kv_cache_groups": [
{
"tokens_per_block": 64,
"layer_names": ["layer0"],
}
],
"inference_engine": "vllm",
}
@@ -137,90 +128,61 @@ def test_get_config_file_path():
assert config_path == f"{fm.base_path}/config.json"
# ---------------------------------------------------------------------------
# parallel_agnostic: honored only for a single non-MLA full-attention group
# ---------------------------------------------------------------------------
def _full_attention_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=128, dtype=torch.float32
),
)
def _sliding_window_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=SlidingWindowSpec(
block_size=16,
num_kv_heads=4,
head_size=128,
dtype=torch.float32,
sliding_window=128,
),
)
def test_parallel_agnostic_enabled_for_single_full_attention():
# tp/rank are collapsed out of the namespace so the cache is shared
# across tensor-parallel sizes.
def test_hybrid_file_identity_uses_resolved_tokens_per_hash():
# For heterogeneous groups the namespace records the resolved hash
# granularity (GCD of the group block sizes), which is the actual
# granularity of the offload block hashes.
fm = make_mapper_from_offloading_spec(
tp_size=2,
groups=((12, "full_layer"), (16, "mla_layer")),
tokens_per_hash=4,
)
assert fm.fields["tokens_per_hash"] == 4
assert fm.fields["kv_cache_groups"] == [
{"tokens_per_block": 12, "layer_names": ["full_layer"]},
{"tokens_per_block": 16, "layer_names": ["mla_layer"]},
]
# ---------------------------------------------------------------------------
# parallel_agnostic: opt-in honored only when the config marks the layout
# parallelism-agnostic (predicate computation is covered in test_factory.py)
# ---------------------------------------------------------------------------
def test_parallel_agnostic_collapses_namespace_when_config_allows():
fm = make_mapper_from_offloading_spec(
tp_size=4,
pp_size=3,
pcp_size=2,
dcp_size=2,
rank=1,
kv_cache_groups=[_full_attention_group()],
is_parallelism_agnostic=True,
parallel_agnostic=True,
)
assert fm.fields["tp_size"] == 1
assert fm.fields["pp_size"] == 1
assert fm.fields["pcp_size"] == 1
assert fm.fields["dcp_size"] == 1
assert fm.rank == 0
def test_parallel_agnostic_disabled_for_multiple_groups():
# More than one KV-cache group (hybrid model) => keep per-layout namespacing.
fm = make_mapper_from_offloading_spec(
tp_size=2,
kv_cache_groups=[_full_attention_group(), _full_attention_group()],
parallel_agnostic=True,
)
assert fm.fields["tp_size"] == 2
def test_parallel_agnostic_disabled_for_non_full_attention():
# Single group but not full attention (sliding window) => keep namespacing.
fm = make_mapper_from_offloading_spec(
tp_size=2,
kv_cache_groups=[_sliding_window_group()],
parallel_agnostic=True,
)
assert fm.fields["tp_size"] == 2
def test_parallel_agnostic_excludes_mla():
# MLA latent KV is replicated per rank, so its offloaded blocks are not
# parallelism-invariant: the opt-in must not collapse tp/rank.
group = KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=MLAAttentionSpec(
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
),
)
fm = make_mapper_from_offloading_spec(
tp_size=2, rank=1, kv_cache_groups=[group], parallel_agnostic=True
)
assert fm.fields["tp_size"] == 2
assert fm.rank == 1
def test_parallel_agnostic_disabled_on_v2_model_runner():
# V2's KV layout is not known to be parallelism-invariant: don't collapse.
def test_parallel_agnostic_ignored_when_config_disallows():
fm = make_mapper_from_offloading_spec(
tp_size=2,
rank=1,
kv_cache_groups=[_full_attention_group()],
use_v2_model_runner=True,
is_parallelism_agnostic=False,
parallel_agnostic=True,
)
assert fm.fields["tp_size"] == 2
assert fm.rank == 1
def test_namespace_kept_without_parallel_agnostic_opt_in():
fm = make_mapper_from_offloading_spec(
tp_size=2,
rank=1,
is_parallelism_agnostic=True,
parallel_agnostic=False,
)
assert fm.fields["tp_size"] == 2
assert fm.rank == 1