[KV Connector][Offloading] Certify attention-only hybrids in the canonical portability gate (#51689)
Signed-off-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Itay Etelis
Or Ozeri
parent
4c58a0c398
commit
c5d840ff6a
@@ -8,7 +8,10 @@ from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import (
|
||||
OffloadingCacheConfig,
|
||||
@@ -139,35 +142,42 @@ def build_offloading_config(
|
||||
and parallel_config.decode_context_parallel_size == 1
|
||||
and parallel_config.prefill_context_parallel_size == 1
|
||||
)
|
||||
# Canonical pages are topology-free by construction, so the canonical
|
||||
# layout widens the gate to every config whose mappings derive portable:
|
||||
# exactly-sharded or replicated GQA heads (writer rotation) and the
|
||||
# TP-replicated MLA latent (one canonical copy for all replicas). The
|
||||
# model-runner version is irrelevant here — certification happens per
|
||||
# layer against live tensor strides at registration, and create_worker
|
||||
# fails closed against this flag if any layer cannot be certified.
|
||||
# Canonical pages are topology-free, so the gate widens to every config
|
||||
# whose mappings derive portable, group by group; certification happens
|
||||
# per layer at registration and create_worker fails closed on this flag.
|
||||
if canonical_layout and not is_parallelism_agnostic:
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
if isinstance(single_group_spec, FullAttentionSpec):
|
||||
if type(single_group_spec) is MLAAttentionSpec:
|
||||
spec_certifiable = (
|
||||
single_group_spec.tokens_per_state == 1
|
||||
and single_group_spec.real_page_size_bytes
|
||||
% single_group_spec.block_size
|
||||
== 0
|
||||
total_kv_heads = vllm_config.model_config.get_total_num_kv_heads()
|
||||
|
||||
def spec_certifiable(spec: KVCacheSpec) -> bool:
|
||||
"""Conservative static mirror of _layer_mapping's per-layer checks."""
|
||||
if not isinstance(spec, AttentionSpec):
|
||||
return False
|
||||
if spec.kv_quant_mode.is_per_token_head:
|
||||
return False
|
||||
if type(spec) is MLAAttentionSpec:
|
||||
return (
|
||||
spec.tokens_per_state == 1
|
||||
and spec.real_page_size_bytes % spec.block_size == 0
|
||||
)
|
||||
else:
|
||||
total_kv_heads = vllm_config.model_config.get_total_num_kv_heads()
|
||||
spec_certifiable = (
|
||||
total_kv_heads % tp_size == 0 or tp_size % total_kv_heads == 0
|
||||
)
|
||||
is_parallelism_agnostic = (
|
||||
spec_certifiable
|
||||
and not single_group_spec.kv_quant_mode.is_per_token_head
|
||||
and parallel_config.decode_context_parallel_size == 1
|
||||
and parallel_config.prefill_context_parallel_size == 1
|
||||
and parallel_config.world_size == tp_size
|
||||
if isinstance(spec, (SlidingWindowMLASpec, MLAAttentionSpec)):
|
||||
return False
|
||||
if not isinstance(spec, (FullAttentionSpec, SlidingWindowSpec)):
|
||||
return False
|
||||
return (
|
||||
total_kv_heads % tp_size == 0 or tp_size % total_kv_heads == 0
|
||||
) and spec.num_kv_heads == max(1, total_kv_heads // tp_size)
|
||||
|
||||
is_parallelism_agnostic = (
|
||||
len(kv_cache_config.kv_cache_groups) > 0
|
||||
and all(
|
||||
spec_certifiable(group.kv_cache_spec)
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
and parallel_config.decode_context_parallel_size == 1
|
||||
and parallel_config.prefill_context_parallel_size == 1
|
||||
and parallel_config.world_size == tp_size
|
||||
)
|
||||
|
||||
kv_events_config = vllm_config.kv_events_config
|
||||
cache_dtype = (
|
||||
|
||||
Reference in New Issue
Block a user