[KV Offload] Define clean backend configuration boundary (#48150)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <cguo51@asu.edu> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
Or Ozeri
OpenAI Codex
parent
8c3393f373
commit
a9531edfa6
@@ -4,80 +4,60 @@
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingSpec,
|
||||
make_offload_key,
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec, make_offload_key
|
||||
from vllm.v1.kv_offload.config import (
|
||||
OffloadingCacheConfig,
|
||||
OffloadingConfig,
|
||||
OffloadingGroupConfig,
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared mocks (mirrors test_fs_tier.py pattern)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MOCK_VLLM_CONFIG = MagicMock()
|
||||
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
|
||||
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
|
||||
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
|
||||
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
|
||||
|
||||
_MOCK_KV_CACHE_CONFIG = MagicMock()
|
||||
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
|
||||
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock(spec=OffloadingSpec)
|
||||
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
|
||||
"""Helper to create FileMapper with customizable mock config."""
|
||||
# Create a copy of the mock config to avoid modifying the global one
|
||||
mock_vllm_config = MagicMock()
|
||||
mock_vllm_config.model_config.model = kwargs.get("model_name", "test-model")
|
||||
mock_vllm_config.cache_config.block_size = kwargs.get("hash_block_size", 16)
|
||||
mock_vllm_config.cache_config.cache_dtype = (
|
||||
f"torch.{kwargs.get('dtype', 'float16')}"
|
||||
"""Build a FileMapper from a mocked spec carrying a hand-built config."""
|
||||
config = OffloadingConfig(
|
||||
groups=tuple(
|
||||
OffloadingGroupConfig(
|
||||
tokens_per_block=tokens_per_block,
|
||||
layer_names=(layer_name,),
|
||||
)
|
||||
for tokens_per_block, layer_name in kwargs.get("groups", ())
|
||||
),
|
||||
worker_kv_bytes_per_block=0,
|
||||
enable_kv_cache_events=False,
|
||||
extra_config={},
|
||||
engine_id="test-engine",
|
||||
model=OffloadingModelConfig(
|
||||
name=kwargs.get("model_name", "test-model"),
|
||||
dtype=kwargs.get("dtype", "float16"),
|
||||
),
|
||||
cache=OffloadingCacheConfig(
|
||||
tokens_per_hash=kwargs.get("tokens_per_hash", 16),
|
||||
blocks_per_chunk=kwargs.get("blocks_per_chunk", 1),
|
||||
),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=kwargs.get("rank", 0),
|
||||
world_size=kwargs.get("world_size", 1),
|
||||
tp_size=kwargs.get("tp_size", 1),
|
||||
pp_size=kwargs.get("pp_size", 1),
|
||||
pcp_size=kwargs.get("pcp_size", 1),
|
||||
dcp_size=kwargs.get("dcp_size", 1),
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=kwargs.get("is_parallelism_agnostic", False),
|
||||
),
|
||||
)
|
||||
mock_vllm_config.parallel_config.tensor_parallel_size = kwargs.get("tp_size", 1)
|
||||
mock_vllm_config.parallel_config.pipeline_parallel_size = kwargs.get("pp_size", 1)
|
||||
mock_vllm_config.parallel_config.prefill_context_parallel_size = kwargs.get(
|
||||
"pcp_size", 1
|
||||
)
|
||||
mock_vllm_config.parallel_config.decode_context_parallel_size = kwargs.get(
|
||||
"dcp_size", 1
|
||||
)
|
||||
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)
|
||||
mock_vllm_config.use_v2_model_runner = kwargs.get("use_v2_model_runner", False)
|
||||
|
||||
mock_kv_cache_config = MagicMock()
|
||||
mock_kv_cache_config.kv_cache_groups = kwargs.get("kv_cache_groups", [])
|
||||
|
||||
mock_offloading_spec = MagicMock(spec=OffloadingSpec)
|
||||
mock_offloading_spec.vllm_config = mock_vllm_config
|
||||
mock_offloading_spec.kv_cache_config = mock_kv_cache_config
|
||||
mock_offloading_spec.block_size_factor = kwargs.get("block_size_factor", 1)
|
||||
|
||||
spec = MagicMock(spec=OffloadingSpec)
|
||||
spec.config = config
|
||||
return FileMapper.from_offloading_spec(
|
||||
root_dir=kwargs.get("root_dir", "/tmp/cache"),
|
||||
offloading_spec=mock_offloading_spec,
|
||||
gpu_blocks_per_file=mock_offloading_spec.block_size_factor,
|
||||
offloading_spec=spec,
|
||||
blocks_per_file=config.cache.blocks_per_chunk,
|
||||
parallel_agnostic=kwargs.get("parallel_agnostic", False),
|
||||
)
|
||||
|
||||
@@ -92,7 +72,7 @@ def test_get_file_name_full_structure():
|
||||
Path must match: <base_path>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash_hex>.bin
|
||||
|
||||
Concretely:
|
||||
- The segment immediately after base_path must end with `_r0`
|
||||
- The segment immediately after base_path must end with `_r3`
|
||||
- The next segment is the first 3 hex chars of the block hash
|
||||
- The next segment is <2 hex chars>_g<group_idx>
|
||||
- The final segment is <full hash hex>.bin
|
||||
@@ -105,7 +85,7 @@ def test_get_file_name_full_structure():
|
||||
path = fm.get_file_name(key)
|
||||
|
||||
expected_path = (
|
||||
"/tmp/cache/test-model_588656ebcc66_r3/000/10_g2/0001020304050607.bin"
|
||||
"/tmp/cache/test-model_42b94bdc9933_r3/000/10_g2/0001020304050607.bin"
|
||||
)
|
||||
assert path == expected_path
|
||||
|
||||
@@ -114,19 +94,30 @@ def test_get_run_config_fields():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
model_name="my-model",
|
||||
dtype="bfloat16",
|
||||
tp_size=2,
|
||||
tp_size=4,
|
||||
pp_size=3,
|
||||
pcp_size=2,
|
||||
dcp_size=2,
|
||||
groups=((64, "layer0"),),
|
||||
tokens_per_hash=64,
|
||||
blocks_per_chunk=3,
|
||||
)
|
||||
cfg = fm.get_run_config()
|
||||
assert cfg == {
|
||||
"model_name": "my-model",
|
||||
"hash_block_size": 16,
|
||||
"gpu_blocks_per_file": 1,
|
||||
"tp_size": 2,
|
||||
"pp_size": 1,
|
||||
"pcp_size": 1,
|
||||
"dcp_size": 1,
|
||||
"tokens_per_hash": 64,
|
||||
"blocks_per_file": 3,
|
||||
"tp_size": 4,
|
||||
"pp_size": 3,
|
||||
"pcp_size": 2,
|
||||
"dcp_size": 2,
|
||||
"dtype": "bfloat16",
|
||||
"kv_cache_groups": [],
|
||||
"kv_cache_groups": [
|
||||
{
|
||||
"tokens_per_block": 64,
|
||||
"layer_names": ["layer0"],
|
||||
}
|
||||
],
|
||||
"inference_engine": "vllm",
|
||||
}
|
||||
|
||||
@@ -137,90 +128,61 @@ def test_get_config_file_path():
|
||||
assert config_path == f"{fm.base_path}/config.json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parallel_agnostic: honored only for a single non-MLA full-attention group
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _full_attention_group() -> KVCacheGroupSpec:
|
||||
return KVCacheGroupSpec(
|
||||
layer_names=["layer0"],
|
||||
kv_cache_spec=FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=128, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _sliding_window_group() -> KVCacheGroupSpec:
|
||||
return KVCacheGroupSpec(
|
||||
layer_names=["layer0"],
|
||||
kv_cache_spec=SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=128,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_parallel_agnostic_enabled_for_single_full_attention():
|
||||
# tp/rank are collapsed out of the namespace so the cache is shared
|
||||
# across tensor-parallel sizes.
|
||||
def test_hybrid_file_identity_uses_resolved_tokens_per_hash():
|
||||
# For heterogeneous groups the namespace records the resolved hash
|
||||
# granularity (GCD of the group block sizes), which is the actual
|
||||
# granularity of the offload block hashes.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
groups=((12, "full_layer"), (16, "mla_layer")),
|
||||
tokens_per_hash=4,
|
||||
)
|
||||
assert fm.fields["tokens_per_hash"] == 4
|
||||
assert fm.fields["kv_cache_groups"] == [
|
||||
{"tokens_per_block": 12, "layer_names": ["full_layer"]},
|
||||
{"tokens_per_block": 16, "layer_names": ["mla_layer"]},
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parallel_agnostic: opt-in honored only when the config marks the layout
|
||||
# parallelism-agnostic (predicate computation is covered in test_factory.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parallel_agnostic_collapses_namespace_when_config_allows():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=4,
|
||||
pp_size=3,
|
||||
pcp_size=2,
|
||||
dcp_size=2,
|
||||
rank=1,
|
||||
kv_cache_groups=[_full_attention_group()],
|
||||
is_parallelism_agnostic=True,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 1
|
||||
assert fm.fields["pp_size"] == 1
|
||||
assert fm.fields["pcp_size"] == 1
|
||||
assert fm.fields["dcp_size"] == 1
|
||||
assert fm.rank == 0
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_for_multiple_groups():
|
||||
# More than one KV-cache group (hybrid model) => keep per-layout namespacing.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
kv_cache_groups=[_full_attention_group(), _full_attention_group()],
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_for_non_full_attention():
|
||||
# Single group but not full attention (sliding window) => keep namespacing.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
kv_cache_groups=[_sliding_window_group()],
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
|
||||
|
||||
def test_parallel_agnostic_excludes_mla():
|
||||
# MLA latent KV is replicated per rank, so its offloaded blocks are not
|
||||
# parallelism-invariant: the opt-in must not collapse tp/rank.
|
||||
group = KVCacheGroupSpec(
|
||||
layer_names=["layer0"],
|
||||
kv_cache_spec=MLAAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2, rank=1, kv_cache_groups=[group], parallel_agnostic=True
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_on_v2_model_runner():
|
||||
# V2's KV layout is not known to be parallelism-invariant: don't collapse.
|
||||
def test_parallel_agnostic_ignored_when_config_disallows():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
rank=1,
|
||||
kv_cache_groups=[_full_attention_group()],
|
||||
use_v2_model_runner=True,
|
||||
is_parallelism_agnostic=False,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
|
||||
def test_namespace_kept_without_parallel_agnostic_opt_in():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
rank=1,
|
||||
is_parallelism_agnostic=True,
|
||||
parallel_agnostic=False,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
Reference in New Issue
Block a user