[KVConnector] Add retention interval to OffloadingConnector (#51886)

Signed-off-by: Bill Nell <bnell@redhat.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
bnellnm
2026-09-04 02:42:20 +00:00
committed by GitHub
co-authored by Tyler Michael Smith
parent 7dc30f5a66
commit 25268f0c9c
9 changed files with 543 additions and 139 deletions
@@ -298,6 +298,10 @@ class MooncakeStoreCoordinator:
use_eagle=use_eagle,
retention_interval=retention_interval,
reachable_boundaries=reachable_boundaries,
# ``spec`` is already DCP-resolved (worker.py applies
# resolve_dcp_kv_cache_spec) and ``end_chunk`` is indexed in
# that scaled block size, so the mask must not scale again.
dcp_world_size=1,
)
if mask is not None:
assert len(mask) == end_chunk - start_chunk
@@ -30,6 +30,7 @@ from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_down
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.single_type_kv_cache_manager import SingleTypeKVCacheManager
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
@@ -38,6 +39,7 @@ from vllm.v1.kv_cache_interface import (
MambaSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
Locality,
@@ -91,15 +93,12 @@ class GroupOffloadConfig(NamedTuple):
kv_event_group_spec: OffloadingEventGroupSpec
# None below means full attention
sliding_window_size_in_chunks: int | None
kv_cache_spec: KVCacheSpec
# Cached manager class for this group's KV cache spec, resolved at init time
manager_cls: type[SingleTypeKVCacheManager]
# Partial-tail data for this group comes from the scheduler's CoW hand-off
# rather than the request block table.
requires_cow_source: bool = False
# Number of this group's offloaded chunks per full-attention alignment
# segment. Used to skip storing SWA chunks that can never serve a load
# hit (e.g. DeepSeek V4 where SWA groups have much smaller block sizes
# than the MLA full-attention group).
# None for full-attention groups or when the optimization doesn't apply.
alignment_chunk_count: int | None = None
# True for EAGLE/MTP draft-model attention groups. The trailing chunk
# of these groups is volatile and lacks a stable hash, so it must
# be excluded from store and load scheduling.
@@ -126,26 +125,6 @@ def get_sliding_window_size_in_chunks(
return None
def is_store_reachable_swa_chunk(
absolute_chunk_index: int,
storable_chunk_count: int,
alignment_chunk_count: int | None,
sliding_window_chunks: int | None,
is_eagle_group: bool,
) -> bool:
"""Return whether an SWA chunk can participate in an external-cache hit."""
if alignment_chunk_count is None:
return True
assert sliding_window_chunks is not None
position_in_segment = absolute_chunk_index % alignment_chunk_count
segment_start = absolute_chunk_index - position_in_segment
actual_segment_length = min(
alignment_chunk_count, storable_chunk_count - segment_start
)
reachable_tail = sliding_window_chunks + int(is_eagle_group)
return position_in_segment >= actual_segment_length - reachable_tail
def resolve_mamba_align_size(
spec: "OffloadingSpec", kv_cache_config: KVCacheConfig
) -> int | None:
@@ -176,6 +155,9 @@ class SchedulerOffloadConfig(NamedTuple):
num_workers: int
offload_prompt_only: bool
supports_partial_tail: bool
alignment_tokens: int | None = None
retention_interval: int | None = None
dcp_world_size: int = 1
@classmethod
def from_spec(
@@ -199,23 +181,21 @@ class SchedulerOffloadConfig(NamedTuple):
full_attn_tokens_per_chunk.add(tokens_per_block * spec.blocks_per_chunk)
# Only apply the optimization if there's a single consistent
# full-attention alignment size.
# full-attention alignment size. For pure SWA/Mamba models (no
# full-attention groups), fall back to the largest SWA/Mamba
# tokens_per_chunk so retention_interval has a boundary
# granularity to work with.
alignment_tokens: int | None = None
if len(full_attn_tokens_per_chunk) == 1:
alignment_tokens = full_attn_tokens_per_chunk.pop()
elif not full_attn_tokens_per_chunk:
all_tokens_per_chunk = [
tpb * spec.blocks_per_chunk for tpb in spec.tokens_per_block
]
if all_tokens_per_chunk:
alignment_tokens = max(all_tokens_per_chunk)
def _alignment_chunk_count(
tokens_per_chunk: int,
sliding_window_size_in_chunks: int | None,
) -> int | None:
if alignment_tokens is None or sliding_window_size_in_chunks is None:
return None
if alignment_tokens <= tokens_per_chunk:
return None
per_segment = alignment_tokens // tokens_per_chunk
if sliding_window_size_in_chunks >= per_segment:
return None
return per_segment
retention_interval = vllm_config.cache_config.prefix_cache_retention_interval
eagle_groups = {
idx
@@ -245,6 +225,10 @@ class SchedulerOffloadConfig(NamedTuple):
sw = get_sliding_window_size_in_chunks(
kv_spec, tokens_per_block * spec.blocks_per_chunk
)
manager_cls = KVCacheSpecRegistry.get_manager_class(kv_spec)
assert manager_cls is not None, (
f"No manager found for KV cache spec {type(kv_spec).__name__}"
)
kv_group_configs_list.append(
GroupOffloadConfig(
group_idx=idx,
@@ -255,9 +239,8 @@ class SchedulerOffloadConfig(NamedTuple):
// spec.tokens_per_hash
),
sliding_window_size_in_chunks=sw,
alignment_chunk_count=_alignment_chunk_count(
tokens_per_block * spec.blocks_per_chunk, sw
),
kv_cache_spec=kv_spec,
manager_cls=manager_cls,
kv_event_group_spec=get_offloading_event_group_spec(kv_cache_group),
is_eagle_group=idx in eagle_groups,
requires_cow_source=(
@@ -289,6 +272,23 @@ class SchedulerOffloadConfig(NamedTuple):
and vllm_config.parallel_config.decode_context_parallel_size == 1
)
if retention_interval is not None:
if retention_interval < 0:
raise ValueError(
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "
f"({retention_interval}) must be non-negative."
)
for config in kv_group_configs:
if (
config.sliding_window_size_in_chunks is not None
and retention_interval % config.tokens_per_chunk != 0
):
raise ValueError(
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "
f"({retention_interval}) must be a multiple of "
f"tokens_per_chunk ({config.tokens_per_chunk})."
)
return cls(
num_workers=vllm_config.parallel_config.world_size,
kv_group_configs=kv_group_configs,
@@ -296,6 +296,9 @@ class SchedulerOffloadConfig(NamedTuple):
tokens_per_hash=spec.tokens_per_hash,
offload_prompt_only=spec.offload_prompt_only,
supports_partial_tail=supports_partial_tail,
alignment_tokens=alignment_tokens,
retention_interval=retention_interval,
dcp_world_size=vllm_config.parallel_config.decode_context_parallel_size,
)
@@ -1353,6 +1356,13 @@ class OffloadingConnectorScheduler:
# Filter out chunks skipped due to sliding window attention / SSM
# or unreachable by the load path's alignment constraints.
new_offload_keys: list[OffloadKey] = []
reachable_boundaries: tuple[int, ...] = ()
if self.config.retention_interval is not None:
reachable_boundaries = (req.num_prompt_tokens - 1,)
if req.shared_prefix_boundary:
reachable_boundaries += (req.shared_prefix_boundary,)
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
@@ -1379,23 +1389,31 @@ class OffloadingConnectorScheduler:
]
assert len(offload_keys) == len(offload_block_ids)
# Use reachable_block_mask to filter unreachable chunks
# (SWA/Mamba sparsity + retention interval).
# reachable_block_mask operates in KV-block coordinates,
# so convert chunk indices to block indices.
block_mask = group_config.manager_cls.reachable_block_mask(
start_block=start_chunk_idx * blocks_per_chunk,
end_block=num_chunks * blocks_per_chunk,
alignment_tokens=self.config.alignment_tokens,
kv_cache_spec=group_config.kv_cache_spec,
use_eagle=group_config.is_eagle_group,
retention_interval=self.config.retention_interval,
reachable_boundaries=reachable_boundaries,
dcp_world_size=self.config.dcp_world_size,
)
for key_idx, (offload_key, block_id) in enumerate(
zip(offload_keys, offload_block_ids)
):
if block_id == 0:
continue
# Skip SWA chunks that can never serve a load hit:
# within each full-attention alignment segment, only the
# trailing chunks queried by _sliding_window_lookup are
# reachable. EAGLE/MTP requires one additional chunk that
# lookup later drops as its volatile draft tail.
abs_chunk_idx = start_chunk_idx + key_idx
if not is_store_reachable_swa_chunk(
abs_chunk_idx,
num_chunks,
group_config.alignment_chunk_count,
group_config.sliding_window_size_in_chunks,
group_config.is_eagle_group,
# A chunk is reachable if any of its constituent
# blocks is reachable.
if block_mask is not None and not any(
block_mask[key_idx * blocks_per_chunk + b]
for b in range(blocks_per_chunk)
):
continue
new_offload_keys.append(offload_key)