[KV Connector][Offloading] Disable parallel-agnostic fs-tier cache on V2 model runner (#46044)
Signed-off-by: Itay Etelis <etelis2019@gmail.com> Co-authored-by: Itay Etelis <etelis2019@gmail.com>
This commit is contained in:
co-authored by
Itay Etelis
parent
a0df04e477
commit
ea6078fe6a
@@ -64,6 +64,7 @@ def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
|
||||
"dcp_size", 1
|
||||
)
|
||||
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)
|
||||
mock_vllm_config.use_v2_model_runner = kwargs.get("use_v2_model_runner", False)
|
||||
|
||||
mock_kv_cache_config = MagicMock()
|
||||
mock_kv_cache_config.kv_cache_groups = kwargs.get("kv_cache_groups", [])
|
||||
@@ -210,3 +211,16 @@ def test_parallel_agnostic_excludes_mla():
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_on_v2_model_runner():
|
||||
# V2's KV layout is not known to be parallelism-invariant: don't collapse.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
rank=1,
|
||||
kv_cache_groups=[_full_attention_group()],
|
||||
use_v2_model_runner=True,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
@@ -37,6 +37,7 @@ def _make_vllm_config():
|
||||
decode_context_parallel_size=1,
|
||||
rank=0,
|
||||
),
|
||||
use_v2_model_runner=False,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -84,10 +84,13 @@ class FileMapper:
|
||||
]
|
||||
# Only a single full-attention group is parallelism-invariant. MLA is
|
||||
# excluded: its latent KV is replicated per rank, never head-sharded.
|
||||
# The V2 model runner is excluded: its KV layout is not known to be
|
||||
# parallelism-invariant.
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
spec = groups[0].kv_cache_spec if len(groups) == 1 else None
|
||||
parallel_agnostic = (
|
||||
parallel_agnostic
|
||||
and not vllm_config.use_v2_model_runner
|
||||
and isinstance(spec, FullAttentionSpec)
|
||||
and not isinstance(spec, MLAAttentionSpec)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user