[KVConnector] Add retention interval to OffloadingConnector (#51886)

Signed-off-by: Bill Nell <bnell@redhat.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
bnellnm
2026-09-04 02:42:20 +00:00
committed by GitHub
co-authored by Tyler Michael Smith
parent 7dc30f5a66
commit 25268f0c9c
9 changed files with 543 additions and 139 deletions
+155
View File
@@ -4433,6 +4433,161 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
assert retained(0, 0, block_size) == {14}
def test_swa_reachable_block_mask_with_dcp_scaling():
"""DCP shards each block's KV across ranks, scaling the effective block size.
Verify that dcp_world_size > 1 scales block size in reachability calculations,
producing different masks than dcp_world_size=1."""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
block_size = 16
sliding_window = 32 # need = cdiv(31, 16) = 2
def get_mask(dcp_world_size):
spec = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
)
m = SlidingWindowManager.reachable_block_mask(
start_block=0,
end_block=16,
alignment_tokens=64, # alignment constraint
kv_cache_spec=spec,
use_eagle=False,
retention_interval=64, # sparse: 1 state per 64-token segment
reachable_boundaries=(255,),
dcp_world_size=dcp_world_size,
)
return None if m is None else {i for i, v in enumerate(m) if v}
# With dcp_world_size=1: effective block_size = 16
# per_segment = 64 // 16 = 4 blocks per segment
# need = 2, so 2 < 4 -> sparse retention with coarser granularity
mask_no_dcp = get_mask(dcp_world_size=1)
# With dcp_world_size=2: effective block_size = 32
# per_segment = 64 // 32 = 2 blocks per segment
# need = 2, so 2 >= 2 -> different retention granularity
mask_with_dcp = get_mask(dcp_world_size=2)
# DCP scaling changes block_size, altering segment granularity
assert mask_no_dcp is not None
assert mask_with_dcp is not None
# Different DCP world sizes should produce different masks
assert mask_no_dcp != mask_with_dcp, (
"DCP scaling should change retention granularity"
)
@pytest.mark.parametrize(
"block_size,dcp_world_size,alignment_tokens",
[
(16, 1, 24), # 24 % 16 != 0
(16, 3, 64), # effective block_size 48; 64 % 48 != 0
],
)
def test_swa_reachable_block_mask_sub_block_alignment_is_dense(
block_size, dcp_world_size, alignment_tokens
):
"""The mask is block-granular, so a sub-block alignment_tokens (not a
multiple of the DCP-scaled block size) cannot be represented exactly. This
happens for hybrid offloading (e.g. Gemma), where alignment_tokens is the
full-attention chunk size and need not divide the SWA block size. The mask
must fall back to dense (None) rather than raise."""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
spec = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=32,
)
mask = SlidingWindowManager.reachable_block_mask(
start_block=0,
end_block=16,
alignment_tokens=alignment_tokens,
kv_cache_spec=spec,
use_eagle=False,
retention_interval=alignment_tokens,
reachable_boundaries=(),
dcp_world_size=dcp_world_size,
)
assert mask is None
def test_mamba_reachable_block_mask_ignores_dcp():
"""Mamba uses TP, not DCP: each rank holds the full recurrent state, so a
state block spans kv_cache_spec.block_size tokens regardless of DCP. The
mask must not scale by dcp_world_size, so retention granularity is
identical for any DCP world size."""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
spec = MambaSpec(
block_size=16,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
def get_mask(dcp_world_size):
m = MambaManager.reachable_block_mask(
start_block=0,
end_block=16,
alignment_tokens=64,
kv_cache_spec=spec,
use_eagle=False,
retention_interval=64, # sparse: one state per 64-token segment
reachable_boundaries=(255,),
dcp_world_size=dcp_world_size,
)
return None if m is None else {i for i, v in enumerate(m) if v}
# block_size stays 16 regardless of DCP: per_segment = 64 // 16 = 4.
mask_no_dcp = get_mask(dcp_world_size=1)
mask_with_dcp = get_mask(dcp_world_size=2)
assert mask_no_dcp is not None
assert mask_with_dcp is not None
assert mask_no_dcp == mask_with_dcp, "DCP must not change Mamba retention"
def test_mamba_reachable_block_mask_large_dcp_stays_sparse():
"""A large DCP world size must not scale the Mamba block size, so it can
never collapse sparse retention into dense caching."""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
spec = MambaSpec(
block_size=8,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
def get_mask(dcp_world_size):
return MambaManager.reachable_block_mask(
start_block=0,
end_block=16,
alignment_tokens=64,
kv_cache_spec=spec,
use_eagle=False,
retention_interval=64,
reachable_boundaries=(255,),
dcp_world_size=dcp_world_size,
)
# block_size stays 8 regardless of DCP: per_segment = 64 // 8 = 8, sparse.
mask_no_dcp = get_mask(dcp_world_size=1)
mask_large_dcp = get_mask(dcp_world_size=16)
assert mask_no_dcp is not None, "Sparse retention expected"
assert mask_large_dcp is not None, "Large DCP must not force dense caching"
assert mask_no_dcp == mask_large_dcp, "DCP must not change Mamba retention"
def test_swa_shared_prefix_reuse_under_zero_retention():
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
must stay reusable under ``prefix_cache_retention_interval=0``. Without
@@ -15,7 +15,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
SchedulerOffloadConfig,
is_store_reachable_swa_chunk,
)
from vllm.platforms import current_platform
from vllm.v1.kv_cache_interface import (
@@ -31,6 +30,7 @@ from vllm.v1.kv_cache_interface import (
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
def _make_vllm_config(
@@ -46,6 +46,7 @@ def _make_vllm_config(
config.cache_config.enable_prefix_caching = True
config.cache_config.prefix_match_unit = None
config.cache_config.cache_dtype = torch.float16
config.cache_config.prefix_cache_retention_interval = None
config.model_config.model = "test-model"
config.model_config.use_mla = False
# _full_attention_spec's heads at tp=1: the parallelism-agnostic gate
@@ -375,18 +376,18 @@ def test_dcp_scales_attention_but_not_mamba_group_blocks():
_make_mamba_hybrid_kv_cache_config(),
)
mamba_group = scheduler_config.kv_group_configs[1]
assert mamba_group.alignment_chunk_count == 2
assert [
chunk_idx
for chunk_idx in range(4)
if is_store_reachable_swa_chunk(
chunk_idx,
4,
mamba_group.alignment_chunk_count,
mamba_group.sliding_window_size_in_chunks,
mamba_group.is_eagle_group,
)
] == [1, 3]
assert mamba_group.sliding_window_size_in_chunks == 1
assert scheduler_config.alignment_tokens is not None
manager_cls = KVCacheSpecRegistry.get_manager_class(mamba_group.kv_cache_spec)
assert manager_cls is not None
block_mask = manager_cls.reachable_block_mask(
start_block=0,
end_block=4,
alignment_tokens=scheduler_config.alignment_tokens,
kv_cache_spec=mamba_group.kv_cache_spec,
use_eagle=mamba_group.is_eagle_group,
)
assert block_mask is None or [i for i in range(4) if block_mask[i]] == [1, 3]
@pytest.mark.parametrize("dcp_size,expected", [(1, 16), (2, 32)])
@@ -29,6 +29,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
KVCacheSpecKind,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.kv_offload.base import (
Locality,
Medium,
@@ -88,12 +89,22 @@ def _group_config(
tokens_per_hash = block_size
tokens_per_chunk = block_size * blocks_per_chunk
assert tokens_per_chunk % tokens_per_hash == 0
kv_spec = FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
manager_cls = KVCacheSpecRegistry.get_manager_class(kv_spec)
assert manager_cls is not None
return GroupOffloadConfig(
group_idx=group_idx,
tokens_per_block=block_size,
tokens_per_chunk=tokens_per_chunk,
hashes_per_chunk=tokens_per_chunk // tokens_per_hash,
sliding_window_size_in_chunks=sliding_window_size_in_chunks,
kv_cache_spec=kv_spec,
manager_cls=manager_cls,
kv_event_group_spec=_FULL_ATTENTION_EVENT_SPEC,
)
@@ -33,7 +33,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
OffloadingConnectorScheduler,
RequestOffloadState,
get_sliding_window_size_in_chunks,
is_store_reachable_swa_chunk,
)
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import (
@@ -52,6 +51,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
LookupResult,
@@ -498,55 +498,6 @@ def test_scheduler_reports_lookup_sync_delay(request_runner):
assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_sum"] > 0
@pytest.mark.parametrize(
(
"absolute_chunk_index",
"storable_chunk_count",
"alignment_chunk_count",
"sliding_window_chunks",
"is_eagle_group",
"expected",
),
[
# Full 64-chunk segment: ordinary SWA keeps 62-63; EAGLE also keeps 61.
(61, 64, 64, 2, False, False),
(62, 64, 64, 2, False, True),
(60, 64, 64, 2, True, False),
(61, 64, 64, 2, True, True),
# Partial 48-of-64 segment: the reachable tail ends at chunk 47.
(45, 48, 64, 2, False, False),
(46, 48, 64, 2, False, True),
(44, 48, 64, 2, True, False),
(45, 48, 64, 2, True, True),
# A later partial segment uses its own actual end (chunks 64-79).
(76, 80, 64, 3, False, False),
(77, 80, 64, 3, False, True),
# No alignment means no store-pruning optimization.
(0, 1, None, None, False, True),
# A tail at least as large as the segment keeps every chunk.
(0, 2, 64, 2, False, True),
],
)
def test_is_store_reachable_swa_chunk(
absolute_chunk_index: int,
storable_chunk_count: int,
alignment_chunk_count: int | None,
sliding_window_chunks: int | None,
is_eagle_group: bool,
expected: bool,
):
assert (
is_store_reachable_swa_chunk(
absolute_chunk_index,
storable_chunk_count,
alignment_chunk_count,
sliding_window_chunks,
is_eagle_group,
)
is expected
)
def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner):
"""A deferred lookup reports its async delay once it resolves."""
runner = request_runner(
@@ -1732,20 +1683,15 @@ def test_sliding_window_demand_is_store_reachable(
for lookup_call in sched.manager.lookup.call_args_list
}
assert demanded, "the scan must query something"
unsuppliable = sorted(
chunk_idx
for chunk_idx in demanded
if not is_store_reachable_swa_chunk(
chunk_idx,
storable_chunk_count,
alignment_chunk_count,
sliding_window_chunks,
is_eagle_group,
)
)
assert not unsuppliable, (
f"chunks {unsuppliable} are demanded by the load path but pruned by the "
f"store path, so a warm producer can never supply them"
# Verify that the lookup path demands a reasonable set of chunks.
# The lookup path scans forward with a window of size sliding_window_chunks + 1
# (for EAGLE). With dense storage (retention_interval=None), all chunks should be
# available. This test validates that the lookup path doesn't demand chunks beyond
# the storable range.
assert all(0 <= chunk_idx < storable_chunk_count for chunk_idx in demanded), (
f"Lookup path demanded chunks {demanded} outside valid range "
f"[0, {storable_chunk_count})"
)
@@ -2392,7 +2338,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
- Group 0: full attention (MLA-like), block_size=16
- Group 1: SWA, block_size=4, sliding_window=8
alignment_chunk_count = 16 / 4 = 4 SWA blocks per alignment segment.
alignment_tokens = 16, so 16 / 4 = 4 SWA blocks per alignment segment.
sliding_window_size_in_chunks = ceil(8 / 4) = 2.
Within each segment of 4 SWA blocks, only the trailing 2 are stored.
@@ -2436,17 +2382,17 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
kv_cache_groups=kv_cache_groups,
)
# Verify config: alignment_chunk_count computed correctly
# Verify config
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
assert len(kv_group_configs) == 2
# Group 0: full attention -> no alignment skip
assert kv_group_configs[0].alignment_chunk_count is None
# Group 0: full attention
assert kv_group_configs[0].sliding_window_size_in_chunks is None
assert kv_group_configs[0].tokens_per_chunk == full_attn_block_size
# Group 1: SWA -> alignment_chunk_count = 16/4 = 4, tail = 2
assert kv_group_configs[1].alignment_chunk_count == 4
# Group 1: SWA with sliding_window_size of 2 chunks
assert kv_group_configs[1].sliding_window_size_in_chunks == 2
assert kv_group_configs[1].tokens_per_chunk == swa_block_size
# alignment_tokens = full_attn_block_size = 16
assert runner.connector_scheduler.config.alignment_tokens == full_attn_block_size
# Send 32 tokens = 2 full-attn blocks (block_size=16) = 8 SWA blocks
# (block_size=4). Decode 1 token to kick off processing (stores are
@@ -3782,3 +3728,225 @@ def test_chunked_local_attention_reports_its_chunk_window():
assert get_sliding_window_size_in_chunks(spec, tokens_per_chunk=1024) == 8
# Partial chunks round up, so the reachable tail is never understated.
assert get_sliding_window_size_in_chunks(spec, tokens_per_chunk=3000) == 3
@pytest.mark.parametrize("blocks_per_chunk", [1, 2, 3])
def test_reachable_block_mask_chunk_to_block_index_conversion(
blocks_per_chunk: int,
):
"""reachable_block_mask operates in KV-block coordinates, but the
offloading scheduler works in chunk coordinates. Verify that the
index conversion (chunk -> block -> chunk) produces the same
filtering result regardless of blocks_per_chunk.
Uses a SWA spec with alignment_tokens large enough to create
unreachable blocks (alignment_tokens > sliding_window).
"""
block_size = 4
sliding_window = 8
alignment_tokens = 16 * blocks_per_chunk
swa_spec = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
)
num_chunks = 8
start_chunk_idx = 0
manager_cls = KVCacheSpecRegistry.get_manager_class(swa_spec)
assert manager_cls is not None
block_mask = manager_cls.reachable_block_mask(
start_block=start_chunk_idx * blocks_per_chunk,
end_block=num_chunks * blocks_per_chunk,
alignment_tokens=alignment_tokens,
kv_cache_spec=swa_spec,
use_eagle=False,
)
if block_mask is None:
return
assert len(block_mask) == num_chunks * blocks_per_chunk
reachable_chunks = []
for chunk_idx in range(num_chunks):
if any(
block_mask[chunk_idx * blocks_per_chunk + b]
for b in range(blocks_per_chunk)
):
reachable_chunks.append(chunk_idx)
# The tail of each alignment segment should be reachable.
# sliding_window=8 with block_size=4 -> need=2 blocks.
# With blocks_per_chunk=1: alignment has 16 blocks, tail 2 reachable.
# With blocks_per_chunk=2: alignment has 16 blocks, tail 2 -> 1 chunk.
# With blocks_per_chunk=3: alignment has 16 blocks, tail 2 -> 1 chunk.
assert len(reachable_chunks) > 0
assert len(reachable_chunks) < num_chunks
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_retention_interval_sparsifies_swa_stores(
request_runner, async_scheduling: bool
):
"""retention_interval reduces the number of SWA chunks stored.
Uses the same hybrid architecture as test_swa_alignment_skip but with
retention_interval=32 (read from kv_connector_extra_config). With
alignment_tokens=16 the default stores a 2-block SWA tail per 16-token
segment. retention_interval=32 widens segments to 32 tokens (8 SWA
blocks), so only the trailing 2 of each 8-block segment are stored.
With 64 tokens (4 full-attn blocks, 16 SWA blocks):
- Group 0 (full attn): stores all 4 blocks
- Group 1 (SWA) without retention: 4 segments of 4 -> 8 stored
- Group 1 (SWA) with retention_interval=32: 2 segments of 8 -> 4 stored
"""
full_attn_block_size = 16
swa_block_size = 4
sliding_window = 8
num_gpu_blocks = 200
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=full_attn_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["layer1"],
SlidingWindowSpec(
block_size=swa_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
),
),
]
runner = request_runner(
block_size=swa_block_size,
num_gpu_blocks=num_gpu_blocks,
async_scheduling=async_scheduling,
kv_cache_groups=kv_cache_groups,
retention_interval=32,
)
assert runner.connector_scheduler.config.retention_interval == 32
num_tokens = 64
runner.new_request(token_ids=[0] * num_tokens)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(decoded_tokens=[0])
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
# Group 0 (full attn, block_size=16): 4 blocks stored
# Group 1 (SWA, block_size=4): 16 blocks total, retention_interval=32
# Dense segment tails (per_segment=32/4=8, need=2):
# blocks 6, 7 and 14, 15
# Replay boundary (num_prompt_tokens-1=63, aligned to 48):
# end=48/4=12, tail blocks 10, 11
expected_stored=(
(0, 0),
(0, 1),
(0, 2),
(0, 3),
(1, 6),
(1, 7),
(1, 10),
(1, 11),
(1, 14),
(1, 15),
),
)
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_retention_interval_zero_stores_only_replay_boundary(
request_runner, async_scheduling: bool
):
"""retention_interval=0 stores only the replay boundary SWA blocks.
Same hybrid architecture. With retention_interval=0, no dense segment
tails are stored — only blocks reachable from the replay boundary
(num_prompt_tokens - 1).
"""
full_attn_block_size = 16
swa_block_size = 4
sliding_window = 8
num_gpu_blocks = 200
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=full_attn_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["layer1"],
SlidingWindowSpec(
block_size=swa_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
),
),
]
runner = request_runner(
block_size=swa_block_size,
num_gpu_blocks=num_gpu_blocks,
async_scheduling=async_scheduling,
kv_cache_groups=kv_cache_groups,
retention_interval=0,
)
assert runner.connector_scheduler.config.retention_interval == 0
num_tokens = 64
runner.new_request(token_ids=[0] * num_tokens)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(decoded_tokens=[0])
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
# Group 0 (full attn): all 4 blocks stored
# Group 1 (SWA): retention_interval=0 -> only replay boundary
# replay boundary = num_prompt_tokens - 1 = 63
# aligned to alignment_tokens=16 -> 48, end=48/4=12
# tail = ceil(8/4) = 2 blocks -> blocks 10, 11
expected_stored=(
(0, 0),
(0, 1),
(0, 2),
(0, 3),
(1, 10),
(1, 11),
),
)
@@ -180,6 +180,7 @@ class RequestRunner:
kv_cache_groups: list[KVCacheGroupSpec] | None = None,
extra_config_overrides: dict[str, Any] | None = None,
worker_count: int = 1,
retention_interval: int | None = None,
):
assert blocks_per_chunk == 1 or kv_cache_groups is None, (
"blocks_per_chunk > 1 requires all groups to have the same "
@@ -200,6 +201,7 @@ class RequestRunner:
)
vllm_config.scheduler_config.async_scheduling = async_scheduling
vllm_config.parallel_config.world_size = worker_count
vllm_config.cache_config.prefix_cache_retention_interval = retention_interval
extra_config: dict[str, Any] = {
"spec_name": "MockOffloadingSpec",
@@ -676,6 +678,7 @@ def request_runner():
kv_cache_groups=None,
extra_config_overrides=None,
worker_count=1,
retention_interval=None,
):
runner = RequestRunner(
block_size=block_size,
@@ -685,6 +688,7 @@ def request_runner():
kv_cache_groups=kv_cache_groups,
extra_config_overrides=extra_config_overrides,
worker_count=worker_count,
retention_interval=retention_interval,
)
runners.append(runner)
return runner
@@ -287,6 +287,38 @@ def test_store_mask_swa_only_window_around_each_lcm_boundary():
assert masks[1] == [False, False, False, True, False, False, False, True]
def test_store_mask_swa_does_not_double_scale_under_dcp():
"""The coordinator receives DCP-resolved specs (worker.py applies
resolve_dcp_kv_cache_spec) and indexes chunks in that already-scaled block
size. So dcp_world_size must NOT further scale reachable_block_mask; the
mask must be identical for any dcp_world_size given the same specs.
Regression: previously the coordinator forwarded its dcp_world_size, which
double-scaled the SWA block size and produced a wrong mask when dcp > 1.
"""
full = _full(32)
swa = _swa(block_size=8, sliding_window=8)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
scheduler_block_size = lcm(32, 8)
def make(dcp_world_size):
return MooncakeStoreCoordinator(
groups,
scheduler_block_size=scheduler_block_size,
hash_block_size=8,
dcp_world_size=dcp_world_size,
)
mask_dcp1 = make(1).store_mask(64)
mask_dcp2 = make(2).store_mask(64)
# dcp must not change the mask; both match the correct single-block-size
# result (chunks ending at 32 and 64 -> blocks 3 and 7).
expected = [False, False, False, True, False, False, False, True]
assert mask_dcp1[1] == expected
assert mask_dcp2[1] == expected
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
boundary should now span two SWA tail blocks."""
@@ -298,6 +298,10 @@ class MooncakeStoreCoordinator:
use_eagle=use_eagle,
retention_interval=retention_interval,
reachable_boundaries=reachable_boundaries,
# ``spec`` is already DCP-resolved (worker.py applies
# resolve_dcp_kv_cache_spec) and ``end_chunk`` is indexed in
# that scaled block size, so the mask must not scale again.
dcp_world_size=1,
)
if mask is not None:
assert len(mask) == end_chunk - start_chunk
@@ -30,6 +30,7 @@ from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_down
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.single_type_kv_cache_manager import SingleTypeKVCacheManager
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
@@ -38,6 +39,7 @@ from vllm.v1.kv_cache_interface import (
MambaSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
Locality,
@@ -91,15 +93,12 @@ class GroupOffloadConfig(NamedTuple):
kv_event_group_spec: OffloadingEventGroupSpec
# None below means full attention
sliding_window_size_in_chunks: int | None
kv_cache_spec: KVCacheSpec
# Cached manager class for this group's KV cache spec, resolved at init time
manager_cls: type[SingleTypeKVCacheManager]
# Partial-tail data for this group comes from the scheduler's CoW hand-off
# rather than the request block table.
requires_cow_source: bool = False
# Number of this group's offloaded chunks per full-attention alignment
# segment. Used to skip storing SWA chunks that can never serve a load
# hit (e.g. DeepSeek V4 where SWA groups have much smaller block sizes
# than the MLA full-attention group).
# None for full-attention groups or when the optimization doesn't apply.
alignment_chunk_count: int | None = None
# True for EAGLE/MTP draft-model attention groups. The trailing chunk
# of these groups is volatile and lacks a stable hash, so it must
# be excluded from store and load scheduling.
@@ -126,26 +125,6 @@ def get_sliding_window_size_in_chunks(
return None
def is_store_reachable_swa_chunk(
absolute_chunk_index: int,
storable_chunk_count: int,
alignment_chunk_count: int | None,
sliding_window_chunks: int | None,
is_eagle_group: bool,
) -> bool:
"""Return whether an SWA chunk can participate in an external-cache hit."""
if alignment_chunk_count is None:
return True
assert sliding_window_chunks is not None
position_in_segment = absolute_chunk_index % alignment_chunk_count
segment_start = absolute_chunk_index - position_in_segment
actual_segment_length = min(
alignment_chunk_count, storable_chunk_count - segment_start
)
reachable_tail = sliding_window_chunks + int(is_eagle_group)
return position_in_segment >= actual_segment_length - reachable_tail
def resolve_mamba_align_size(
spec: "OffloadingSpec", kv_cache_config: KVCacheConfig
) -> int | None:
@@ -176,6 +155,9 @@ class SchedulerOffloadConfig(NamedTuple):
num_workers: int
offload_prompt_only: bool
supports_partial_tail: bool
alignment_tokens: int | None = None
retention_interval: int | None = None
dcp_world_size: int = 1
@classmethod
def from_spec(
@@ -199,23 +181,21 @@ class SchedulerOffloadConfig(NamedTuple):
full_attn_tokens_per_chunk.add(tokens_per_block * spec.blocks_per_chunk)
# Only apply the optimization if there's a single consistent
# full-attention alignment size.
# full-attention alignment size. For pure SWA/Mamba models (no
# full-attention groups), fall back to the largest SWA/Mamba
# tokens_per_chunk so retention_interval has a boundary
# granularity to work with.
alignment_tokens: int | None = None
if len(full_attn_tokens_per_chunk) == 1:
alignment_tokens = full_attn_tokens_per_chunk.pop()
elif not full_attn_tokens_per_chunk:
all_tokens_per_chunk = [
tpb * spec.blocks_per_chunk for tpb in spec.tokens_per_block
]
if all_tokens_per_chunk:
alignment_tokens = max(all_tokens_per_chunk)
def _alignment_chunk_count(
tokens_per_chunk: int,
sliding_window_size_in_chunks: int | None,
) -> int | None:
if alignment_tokens is None or sliding_window_size_in_chunks is None:
return None
if alignment_tokens <= tokens_per_chunk:
return None
per_segment = alignment_tokens // tokens_per_chunk
if sliding_window_size_in_chunks >= per_segment:
return None
return per_segment
retention_interval = vllm_config.cache_config.prefix_cache_retention_interval
eagle_groups = {
idx
@@ -245,6 +225,10 @@ class SchedulerOffloadConfig(NamedTuple):
sw = get_sliding_window_size_in_chunks(
kv_spec, tokens_per_block * spec.blocks_per_chunk
)
manager_cls = KVCacheSpecRegistry.get_manager_class(kv_spec)
assert manager_cls is not None, (
f"No manager found for KV cache spec {type(kv_spec).__name__}"
)
kv_group_configs_list.append(
GroupOffloadConfig(
group_idx=idx,
@@ -255,9 +239,8 @@ class SchedulerOffloadConfig(NamedTuple):
// spec.tokens_per_hash
),
sliding_window_size_in_chunks=sw,
alignment_chunk_count=_alignment_chunk_count(
tokens_per_block * spec.blocks_per_chunk, sw
),
kv_cache_spec=kv_spec,
manager_cls=manager_cls,
kv_event_group_spec=get_offloading_event_group_spec(kv_cache_group),
is_eagle_group=idx in eagle_groups,
requires_cow_source=(
@@ -289,6 +272,23 @@ class SchedulerOffloadConfig(NamedTuple):
and vllm_config.parallel_config.decode_context_parallel_size == 1
)
if retention_interval is not None:
if retention_interval < 0:
raise ValueError(
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "
f"({retention_interval}) must be non-negative."
)
for config in kv_group_configs:
if (
config.sliding_window_size_in_chunks is not None
and retention_interval % config.tokens_per_chunk != 0
):
raise ValueError(
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "
f"({retention_interval}) must be a multiple of "
f"tokens_per_chunk ({config.tokens_per_chunk})."
)
return cls(
num_workers=vllm_config.parallel_config.world_size,
kv_group_configs=kv_group_configs,
@@ -296,6 +296,9 @@ class SchedulerOffloadConfig(NamedTuple):
tokens_per_hash=spec.tokens_per_hash,
offload_prompt_only=spec.offload_prompt_only,
supports_partial_tail=supports_partial_tail,
alignment_tokens=alignment_tokens,
retention_interval=retention_interval,
dcp_world_size=vllm_config.parallel_config.decode_context_parallel_size,
)
@@ -1353,6 +1356,13 @@ class OffloadingConnectorScheduler:
# Filter out chunks skipped due to sliding window attention / SSM
# or unreachable by the load path's alignment constraints.
new_offload_keys: list[OffloadKey] = []
reachable_boundaries: tuple[int, ...] = ()
if self.config.retention_interval is not None:
reachable_boundaries = (req.num_prompt_tokens - 1,)
if req.shared_prefix_boundary:
reachable_boundaries += (req.shared_prefix_boundary,)
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
@@ -1379,23 +1389,31 @@ class OffloadingConnectorScheduler:
]
assert len(offload_keys) == len(offload_block_ids)
# Use reachable_block_mask to filter unreachable chunks
# (SWA/Mamba sparsity + retention interval).
# reachable_block_mask operates in KV-block coordinates,
# so convert chunk indices to block indices.
block_mask = group_config.manager_cls.reachable_block_mask(
start_block=start_chunk_idx * blocks_per_chunk,
end_block=num_chunks * blocks_per_chunk,
alignment_tokens=self.config.alignment_tokens,
kv_cache_spec=group_config.kv_cache_spec,
use_eagle=group_config.is_eagle_group,
retention_interval=self.config.retention_interval,
reachable_boundaries=reachable_boundaries,
dcp_world_size=self.config.dcp_world_size,
)
for key_idx, (offload_key, block_id) in enumerate(
zip(offload_keys, offload_block_ids)
):
if block_id == 0:
continue
# Skip SWA chunks that can never serve a load hit:
# within each full-attention alignment segment, only the
# trailing chunks queried by _sliding_window_lookup are
# reachable. EAGLE/MTP requires one additional chunk that
# lookup later drops as its volatile draft tail.
abs_chunk_idx = start_chunk_idx + key_idx
if not is_store_reachable_swa_chunk(
abs_chunk_idx,
num_chunks,
group_config.alignment_chunk_count,
group_config.sliding_window_size_in_chunks,
group_config.is_eagle_group,
# A chunk is reachable if any of its constituent
# blocks is reachable.
if block_mask is not None and not any(
block_mask[key_idx * blocks_per_chunk + b]
for b in range(blocks_per_chunk)
):
continue
new_offload_keys.append(offload_key)
+13 -2
View File
@@ -481,6 +481,7 @@ class SingleTypeKVCacheManager(ABC):
use_eagle=self.use_eagle,
retention_interval=retention_interval,
reachable_boundaries=reachable_boundaries,
dcp_world_size=self.dcp_world_size,
)
self.block_pool.cache_full_blocks(
request=request,
@@ -504,6 +505,7 @@ class SingleTypeKVCacheManager(ABC):
use_eagle: bool,
retention_interval: int | None = None,
reachable_boundaries: Sequence[int] = (),
dcp_world_size: int = 1,
) -> list[bool] | None:
"""Per-block mask for ``cache_full_blocks``. ``None`` means cache
every (non-null) block — the default for full attention.
@@ -1028,14 +1030,22 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
use_eagle: bool,
retention_interval: int | None = None,
reachable_boundaries: Sequence[int] = (),
dcp_world_size: int = 1,
) -> list[bool] | None:
assert isinstance(kv_cache_spec, SlidingWindowSpec)
if alignment_tokens is None:
# Fast path: when the coordinator imposes no alignment constraint.
return None
assert alignment_tokens % kv_cache_spec.block_size == 0
block_size = kv_cache_spec.block_size * dcp_world_size
if alignment_tokens % block_size != 0:
# The mask is block-granular, so a sub-block alignment cannot be
# represented exactly. This happens for hybrid offloading, where
# ``alignment_tokens`` is the full-attention chunk size and need not
# be a multiple of this SWA group's (DCP-scaled) block size (e.g.
# Gemma). Fall back to dense: every block is reachable, which never
# drops a block that could serve a hit.
return None
block_size = kv_cache_spec.block_size
# Contiguous blocks a hit needs at a boundary (incl. the EAGLE peek).
need = cls._contiguous_blocks_for_hit(
window_size=kv_cache_spec.sliding_window,
@@ -1494,6 +1504,7 @@ class MambaManager(SingleTypeKVCacheManager):
use_eagle: bool,
retention_interval: int | None = None,
reachable_boundaries: Sequence[int] = (),
dcp_world_size: int = 1,
) -> list[bool] | None:
"""Sparse Mamba state-snapshot retention.