[KVConnector] Add retention interval to OffloadingConnector (#51886)
Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
7dc30f5a66
commit
25268f0c9c
@@ -298,6 +298,10 @@ class MooncakeStoreCoordinator:
|
||||
use_eagle=use_eagle,
|
||||
retention_interval=retention_interval,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
# ``spec`` is already DCP-resolved (worker.py applies
|
||||
# resolve_dcp_kv_cache_spec) and ``end_chunk`` is indexed in
|
||||
# that scaled block size, so the mask must not scale again.
|
||||
dcp_world_size=1,
|
||||
)
|
||||
if mask is not None:
|
||||
assert len(mask) == end_chunk - start_chunk
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SingleTypeKVCacheManager
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
@@ -38,6 +39,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
from vllm.v1.kv_offload.base import (
|
||||
GPULoadStoreSpec,
|
||||
Locality,
|
||||
@@ -91,15 +93,12 @@ class GroupOffloadConfig(NamedTuple):
|
||||
kv_event_group_spec: OffloadingEventGroupSpec
|
||||
# None below means full attention
|
||||
sliding_window_size_in_chunks: int | None
|
||||
kv_cache_spec: KVCacheSpec
|
||||
# Cached manager class for this group's KV cache spec, resolved at init time
|
||||
manager_cls: type[SingleTypeKVCacheManager]
|
||||
# Partial-tail data for this group comes from the scheduler's CoW hand-off
|
||||
# rather than the request block table.
|
||||
requires_cow_source: bool = False
|
||||
# Number of this group's offloaded chunks per full-attention alignment
|
||||
# segment. Used to skip storing SWA chunks that can never serve a load
|
||||
# hit (e.g. DeepSeek V4 where SWA groups have much smaller block sizes
|
||||
# than the MLA full-attention group).
|
||||
# None for full-attention groups or when the optimization doesn't apply.
|
||||
alignment_chunk_count: int | None = None
|
||||
# True for EAGLE/MTP draft-model attention groups. The trailing chunk
|
||||
# of these groups is volatile and lacks a stable hash, so it must
|
||||
# be excluded from store and load scheduling.
|
||||
@@ -126,26 +125,6 @@ def get_sliding_window_size_in_chunks(
|
||||
return None
|
||||
|
||||
|
||||
def is_store_reachable_swa_chunk(
|
||||
absolute_chunk_index: int,
|
||||
storable_chunk_count: int,
|
||||
alignment_chunk_count: int | None,
|
||||
sliding_window_chunks: int | None,
|
||||
is_eagle_group: bool,
|
||||
) -> bool:
|
||||
"""Return whether an SWA chunk can participate in an external-cache hit."""
|
||||
if alignment_chunk_count is None:
|
||||
return True
|
||||
assert sliding_window_chunks is not None
|
||||
position_in_segment = absolute_chunk_index % alignment_chunk_count
|
||||
segment_start = absolute_chunk_index - position_in_segment
|
||||
actual_segment_length = min(
|
||||
alignment_chunk_count, storable_chunk_count - segment_start
|
||||
)
|
||||
reachable_tail = sliding_window_chunks + int(is_eagle_group)
|
||||
return position_in_segment >= actual_segment_length - reachable_tail
|
||||
|
||||
|
||||
def resolve_mamba_align_size(
|
||||
spec: "OffloadingSpec", kv_cache_config: KVCacheConfig
|
||||
) -> int | None:
|
||||
@@ -176,6 +155,9 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
num_workers: int
|
||||
offload_prompt_only: bool
|
||||
supports_partial_tail: bool
|
||||
alignment_tokens: int | None = None
|
||||
retention_interval: int | None = None
|
||||
dcp_world_size: int = 1
|
||||
|
||||
@classmethod
|
||||
def from_spec(
|
||||
@@ -199,23 +181,21 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
full_attn_tokens_per_chunk.add(tokens_per_block * spec.blocks_per_chunk)
|
||||
|
||||
# Only apply the optimization if there's a single consistent
|
||||
# full-attention alignment size.
|
||||
# full-attention alignment size. For pure SWA/Mamba models (no
|
||||
# full-attention groups), fall back to the largest SWA/Mamba
|
||||
# tokens_per_chunk so retention_interval has a boundary
|
||||
# granularity to work with.
|
||||
alignment_tokens: int | None = None
|
||||
if len(full_attn_tokens_per_chunk) == 1:
|
||||
alignment_tokens = full_attn_tokens_per_chunk.pop()
|
||||
elif not full_attn_tokens_per_chunk:
|
||||
all_tokens_per_chunk = [
|
||||
tpb * spec.blocks_per_chunk for tpb in spec.tokens_per_block
|
||||
]
|
||||
if all_tokens_per_chunk:
|
||||
alignment_tokens = max(all_tokens_per_chunk)
|
||||
|
||||
def _alignment_chunk_count(
|
||||
tokens_per_chunk: int,
|
||||
sliding_window_size_in_chunks: int | None,
|
||||
) -> int | None:
|
||||
if alignment_tokens is None or sliding_window_size_in_chunks is None:
|
||||
return None
|
||||
if alignment_tokens <= tokens_per_chunk:
|
||||
return None
|
||||
per_segment = alignment_tokens // tokens_per_chunk
|
||||
if sliding_window_size_in_chunks >= per_segment:
|
||||
return None
|
||||
return per_segment
|
||||
retention_interval = vllm_config.cache_config.prefix_cache_retention_interval
|
||||
|
||||
eagle_groups = {
|
||||
idx
|
||||
@@ -245,6 +225,10 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
sw = get_sliding_window_size_in_chunks(
|
||||
kv_spec, tokens_per_block * spec.blocks_per_chunk
|
||||
)
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(kv_spec)
|
||||
assert manager_cls is not None, (
|
||||
f"No manager found for KV cache spec {type(kv_spec).__name__}"
|
||||
)
|
||||
kv_group_configs_list.append(
|
||||
GroupOffloadConfig(
|
||||
group_idx=idx,
|
||||
@@ -255,9 +239,8 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
// spec.tokens_per_hash
|
||||
),
|
||||
sliding_window_size_in_chunks=sw,
|
||||
alignment_chunk_count=_alignment_chunk_count(
|
||||
tokens_per_block * spec.blocks_per_chunk, sw
|
||||
),
|
||||
kv_cache_spec=kv_spec,
|
||||
manager_cls=manager_cls,
|
||||
kv_event_group_spec=get_offloading_event_group_spec(kv_cache_group),
|
||||
is_eagle_group=idx in eagle_groups,
|
||||
requires_cow_source=(
|
||||
@@ -289,6 +272,23 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
and vllm_config.parallel_config.decode_context_parallel_size == 1
|
||||
)
|
||||
|
||||
if retention_interval is not None:
|
||||
if retention_interval < 0:
|
||||
raise ValueError(
|
||||
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "
|
||||
f"({retention_interval}) must be non-negative."
|
||||
)
|
||||
for config in kv_group_configs:
|
||||
if (
|
||||
config.sliding_window_size_in_chunks is not None
|
||||
and retention_interval % config.tokens_per_chunk != 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "
|
||||
f"({retention_interval}) must be a multiple of "
|
||||
f"tokens_per_chunk ({config.tokens_per_chunk})."
|
||||
)
|
||||
|
||||
return cls(
|
||||
num_workers=vllm_config.parallel_config.world_size,
|
||||
kv_group_configs=kv_group_configs,
|
||||
@@ -296,6 +296,9 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
tokens_per_hash=spec.tokens_per_hash,
|
||||
offload_prompt_only=spec.offload_prompt_only,
|
||||
supports_partial_tail=supports_partial_tail,
|
||||
alignment_tokens=alignment_tokens,
|
||||
retention_interval=retention_interval,
|
||||
dcp_world_size=vllm_config.parallel_config.decode_context_parallel_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -1353,6 +1356,13 @@ class OffloadingConnectorScheduler:
|
||||
# Filter out chunks skipped due to sliding window attention / SSM
|
||||
# or unreachable by the load path's alignment constraints.
|
||||
new_offload_keys: list[OffloadKey] = []
|
||||
|
||||
reachable_boundaries: tuple[int, ...] = ()
|
||||
if self.config.retention_interval is not None:
|
||||
reachable_boundaries = (req.num_prompt_tokens - 1,)
|
||||
if req.shared_prefix_boundary:
|
||||
reachable_boundaries += (req.shared_prefix_boundary,)
|
||||
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
@@ -1379,23 +1389,31 @@ class OffloadingConnectorScheduler:
|
||||
]
|
||||
assert len(offload_keys) == len(offload_block_ids)
|
||||
|
||||
# Use reachable_block_mask to filter unreachable chunks
|
||||
# (SWA/Mamba sparsity + retention interval).
|
||||
# reachable_block_mask operates in KV-block coordinates,
|
||||
# so convert chunk indices to block indices.
|
||||
block_mask = group_config.manager_cls.reachable_block_mask(
|
||||
start_block=start_chunk_idx * blocks_per_chunk,
|
||||
end_block=num_chunks * blocks_per_chunk,
|
||||
alignment_tokens=self.config.alignment_tokens,
|
||||
kv_cache_spec=group_config.kv_cache_spec,
|
||||
use_eagle=group_config.is_eagle_group,
|
||||
retention_interval=self.config.retention_interval,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
dcp_world_size=self.config.dcp_world_size,
|
||||
)
|
||||
|
||||
for key_idx, (offload_key, block_id) in enumerate(
|
||||
zip(offload_keys, offload_block_ids)
|
||||
):
|
||||
if block_id == 0:
|
||||
continue
|
||||
# Skip SWA chunks that can never serve a load hit:
|
||||
# within each full-attention alignment segment, only the
|
||||
# trailing chunks queried by _sliding_window_lookup are
|
||||
# reachable. EAGLE/MTP requires one additional chunk that
|
||||
# lookup later drops as its volatile draft tail.
|
||||
abs_chunk_idx = start_chunk_idx + key_idx
|
||||
if not is_store_reachable_swa_chunk(
|
||||
abs_chunk_idx,
|
||||
num_chunks,
|
||||
group_config.alignment_chunk_count,
|
||||
group_config.sliding_window_size_in_chunks,
|
||||
group_config.is_eagle_group,
|
||||
# A chunk is reachable if any of its constituent
|
||||
# blocks is reachable.
|
||||
if block_mask is not None and not any(
|
||||
block_mask[key_idx * blocks_per_chunk + b]
|
||||
for b in range(blocks_per_chunk)
|
||||
):
|
||||
continue
|
||||
new_offload_keys.append(offload_key)
|
||||
|
||||
Reference in New Issue
Block a user