[Feature][SimpleCPU] Load fine-grained hybrid prefix hits (#54736)

Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
This commit is contained in:
YukioZzz
2026-09-10 20:36:37 -05:00
committed by GitHub
parent 828f4f19b4
commit 07b7553465
6 changed files with 982 additions and 110 deletions
+9 -40
View File
@@ -714,17 +714,8 @@ def test_disk_mode_block_removed_relabeled_storage() -> None:
assert ev.medium == MEDIUM_STORAGE
def test_mamba_dcp_metadata_guard() -> None:
"""Mamba+DCP: token_ids guard emits [] when capture and emission sizes differ.
The metadata capture at _prepare_eager_store_specs uses
g_block_size = spec.block_size * cp_world_size = 32 * 2 = 64 to slice
tokens. The event emission uses block_size = spec.block_size * 1 = 32
for Mamba (SSM state is not CP-sharded). The guard in
_process_store_completion detects len(token_ids) != block_size and emits
token_ids=[] instead of misleading data. When #49962 fixes the capture
path, the guard stops triggering and correct token_ids flow through.
"""
def test_mamba_align_skips_positional_event_metadata() -> None:
"""Mamba align emits events only from explicit boundary handoffs."""
vbs = BLOCK_SIZE * 2
kv_cfg = _make_mixed_kv_cache_config(
num_blocks=16, with_mamba=True, mamba_block_size=32
@@ -739,8 +730,6 @@ def test_mamba_dcp_metadata_guard() -> None:
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
# Mamba g_block_size = 32 * 2 = 64, so 2 Mamba blocks need 128 computed
# tokens to be "ready"; FA g_block_size = 16 * 2 = 32 needs only 64.
req = _make_cp_request(num_blocks=4, virtual_block_size=vbs)
fa_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=0)
mamba_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=1)
@@ -759,32 +748,12 @@ def test_mamba_dcp_metadata_guard() -> None:
events = list(sched.take_events())
stored = [e for e in events if isinstance(e, BlockStored)]
assert len(stored) == 4, (
f"expected 4 BlockStored (2 FA + 2 Mamba), got {len(stored)}"
)
by_group: dict[int, list[BlockStored]] = {}
for ev in stored:
by_group.setdefault(ev.group_idx, []).append(ev)
# FA group: capture size (64) == emission size (64) → token_ids present.
fa_events = by_group[0]
assert len(fa_events) == 2
for ev in fa_events:
assert len(stored) == 2
assert {ev.group_idx for ev in stored} == {0}
for idx, ev in enumerate(stored):
assert ev.block_size == vbs
assert len(ev.token_ids) == vbs, (
f"FA token_ids should be {vbs}, got {len(ev.token_ids)}"
assert ev.token_ids == req.prompt_token_ids[idx * vbs : (idx + 1) * vbs]
expected_hash = maybe_convert_block_hash(
get_block_hash(make_block_hash_with_group_id(req.block_hashes[idx], 0))
)
# Mamba group: capture size (64) != emission size (32) → guard emits [].
mamba_events = by_group[1]
assert len(mamba_events) == 2
for ev in mamba_events:
assert ev.block_size == 32
assert ev.token_ids == [], (
f"Mamba token_ids should be [] (guard), got {ev.token_ids}"
)
assert ev.parent_block_hash is None, (
"Mamba parent_block_hash should be None (guard)"
)
assert ev.extra_keys is None, "Mamba extra_keys should be None (guard)"
assert ev.block_hashes == [expected_hash]
@@ -4,6 +4,7 @@
from __future__ import annotations
import logging
from dataclasses import dataclass
import pytest
@@ -18,6 +19,7 @@ from vllm.config import (
SchedulerConfig,
VllmConfig,
)
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
SimpleCPUOffloadConnector,
)
@@ -31,6 +33,7 @@ from vllm.v1.core.kv_cache_utils import (
)
from vllm.v1.core.sched.output import (
CachedRequestData,
KVConnectorBlockState,
NewRequestData,
SchedulerOutput,
)
@@ -42,6 +45,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
MambaSpec,
SlidingWindowSpec,
)
from vllm.v1.outputs import KVConnectorOutput
@@ -2128,3 +2132,671 @@ def test_cp_lazy_target_blocks_scaling(cp_world_size: int) -> None:
f"cp_world_size={cp_world_size}: target_cp={target_cp} should be "
f"less than target_base={target_base}"
)
def _make_hybrid_attention_mamba_scheduler(
*,
num_cpu_blocks: int = 8,
num_gpu_blocks: int = 16,
attention_block_size: int = BLOCK_SIZE,
block_size: int = 4 * BLOCK_SIZE,
scheduler_block_size: int | None = None,
hash_block_size: int | None = None,
dcp_world_size: int = 4,
lazy: bool = False,
mamba_cache_mode: str = "align",
enable_kv_cache_events: bool = False,
) -> SchedulerFixture:
"""Build a scheduler for one attention group plus one Mamba group."""
scheduler_block_size = scheduler_block_size or block_size
hash_block_size = hash_block_size or block_size
attention_spec = FullAttentionSpec(
block_size=attention_block_size,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
)
mamba_spec = MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode=mamba_cache_mode,
)
groups = [
KVCacheGroupSpec(["attention"], attention_spec),
KVCacheGroupSpec(["mamba"], mamba_spec),
]
tensors = [
KVCacheTensor(
size=spec.page_size_bytes * num_gpu_blocks,
layers=group.layer_names,
layer_stride=spec.page_size_bytes * num_gpu_blocks,
block_stride=spec.page_size_bytes,
)
for group, spec in zip(groups, (attention_spec, mamba_spec))
]
kv_cache_config = KVCacheConfig(
num_blocks=num_gpu_blocks,
kv_cache_tensors=tensors,
kv_cache_groups=groups,
)
vllm_config = _make_cp_vllm_config(dcp_world_size=dcp_world_size)
vllm_config.cache_config.prefix_cache_retention_interval = 0
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
if enable_kv_cache_events:
vllm_config.kv_events_config = KVEventsConfig(enable_kv_cache_events=True)
# _derive_cpu_config() scales the requested capacity against
# kv_cache_tensors[0].size, so express it as a multiple of that tensor's
# per-block size to get exactly num_cpu_blocks.
cpu_capacity_bytes = tensors[0].size // num_gpu_blocks * num_cpu_blocks
sched = SimpleCPUOffloadScheduler(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
cpu_capacity_bytes=cpu_capacity_bytes,
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
lazy_offload=lazy,
)
gpu_block_pool = BlockPool(
num_gpu_blocks=num_gpu_blocks,
enable_caching=True,
hash_block_size=hash_block_size,
)
sched.bind_gpu_block_pool(gpu_block_pool)
return SchedulerFixture(
scheduler=sched,
gpu_block_pool=gpu_block_pool,
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
)
def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
"""Replicated groups must not be scaled by the DCP world size.
``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP
size and every other spec 1. Reconstructing a group's geometry as
``spec.block_size * cp_world_size`` over-scales the replicated groups, so
the eager store scan believes far fewer of their blocks are ready and
silently offloads only a fraction of them.
"""
attention_block_size = BLOCK_SIZE
mamba_block_size = 4 * BLOCK_SIZE
dcp_world_size = 2
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=32,
num_gpu_blocks=32,
attention_block_size=attention_block_size,
block_size=mamba_block_size,
# Every prefix-cacheable group's resolved block size must be a multiple
# of the hash block, so it cannot exceed the smallest of them.
hash_block_size=BLOCK_SIZE,
dcp_world_size=dcp_world_size,
mamba_cache_mode="all",
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
attention_size, mamba_size = sched.group_block_sizes
assert attention_size == attention_block_size * dcp_world_size
assert mamba_size == mamba_block_size
confirmed = 2 * sched.block_size
req = _make_cp_request(num_blocks=8, virtual_block_size=BLOCK_SIZE)
attention_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, confirmed // attention_size, attention_size, group_id=0
)
mamba_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, confirmed // mamba_size, mamba_size, group_id=1
)
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
sched.build_connector_meta(
make_scheduler_output(
{req.request_id: confirmed},
new_reqs={req.request_id: kv_blocks.get_block_ids()},
)
)
req.num_computed_tokens = confirmed
meta = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: 1},
cached_req_new_blocks={req.request_id: None},
)
)
# Both groups are positionally stable here, so every confirmed block of
# both must be offloaded. Over-scaling the replicated group halves its
# count.
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [
confirmed // attention_size,
confirmed // mamba_size,
]
assert set(meta.store_gpu_blocks) == {
*(b.block_id for b in attention_blocks),
*(b.block_id for b in mamba_blocks),
}
def test_dcp_mixed_cache_loads_fine_grained_external_hit() -> None:
"""Map a fine-grained external suffix after a nonzero local prefix."""
hash_block_size = BLOCK_SIZE
mamba_block_size = 4 * BLOCK_SIZE
local_tokens = 4 * BLOCK_SIZE
# Span two Mamba blocks and end partially in the second one. This exercises
# both cdiv(external_tokens, mamba_block_size) > 1 and the partial tail.
external_tokens = 6 * BLOCK_SIZE
total_cached_tokens = local_tokens + external_tokens
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=32,
num_gpu_blocks=32,
block_size=mamba_block_size,
hash_block_size=hash_block_size,
dcp_world_size=2,
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
assert external_tokens % mamba_block_size != 0
assert sched.cpu_coordinator.enable_partial_hash_hits
producer = _make_cp_request(num_blocks=10, virtual_block_size=hash_block_size)
attention_blocks = _allocate_cp_gpu_blocks(
gpu_pool,
producer,
num_blocks=5,
virtual_block_size=2 * BLOCK_SIZE,
group_id=0,
)
mamba_blocks = gpu_pool.get_new_blocks(3)
gpu_pool.cache_full_blocks(
request=producer,
blocks=mamba_blocks,
num_cached_blocks=0,
num_full_blocks=2,
block_size=mamba_block_size,
kv_cache_group_id=1,
)
gpu_pool.cache_partial_block(
request=producer,
block=mamba_blocks[2],
num_tokens=total_cached_tokens,
kv_cache_group_id=1,
block_size=mamba_block_size,
)
producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0)
# Positional scanning stores append-only FA blocks. Mamba align blocks use
# exact handoffs for every retained boundary, including the partial tail.
store_output = make_scheduler_output(
{producer.request_id: total_cached_tokens},
new_reqs={producer.request_id: producer_blocks.get_block_ids()},
)
store_output.kv_connector_block_state = KVConnectorBlockState(
req_ids=set(),
resolve_block_ids=lambda _: (),
boundary_state_offloads={
producer.request_id: [
(1, mamba_blocks[0].block_id, mamba_block_size),
(1, mamba_blocks[1].block_id, 2 * mamba_block_size),
(1, mamba_blocks[2].block_id, total_cached_tokens),
]
},
)
store_meta = sched.build_connector_meta(store_output)
# The exact handoffs land in the step they arrive; the append-only FA scan
# picks its blocks up once the scheduler has committed the tokens.
assert set(store_meta.store_gpu_blocks) == {
block.block_id for block in mamba_blocks
}
simulate_store_completion(sched, store_meta.store_event)
producer.num_computed_tokens = total_cached_tokens
fa_meta = sched.build_connector_meta(
make_scheduler_output(
{producer.request_id: 1},
cached_req_new_blocks={producer.request_id: None},
)
)
assert set(fa_meta.store_gpu_blocks) == {
block.block_id for block in attention_blocks
}
simulate_store_completion(sched, fa_meta.store_event)
consumer = Request(
request_id="req-dcp-mixed-fine-load",
prompt_token_ids=producer.prompt_token_ids,
sampling_params=producer.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=producer._block_hasher,
)
hit_tokens, is_async = sched.get_num_new_matched_tokens(
consumer, num_computed_tokens=local_tokens
)
assert hit_tokens == external_tokens
assert is_async is True
pending_blocks, pending_tokens, _ = sched._pending_cpu_hits[consumer.request_id]
assert pending_tokens == external_tokens
assert len(pending_blocks[1]) == 2
assert pending_blocks[1][0].is_null
assert not pending_blocks[1][1].is_null
local_attention = attention_blocks[:2]
local_mamba = mamba_blocks[:1]
load_blocks = KVCacheBlocks(
blocks=(
local_attention + gpu_pool.get_new_blocks(3),
local_mamba + gpu_pool.get_new_blocks(2),
)
)
sched.update_state_after_alloc(
consumer,
load_blocks,
num_external_tokens=hit_tokens,
)
load_meta = sched.build_connector_meta(
make_scheduler_output(
{consumer.request_id: 1},
new_reqs={consumer.request_id: load_blocks.get_block_ids()},
)
)
assert load_meta.load_gpu_blocks == [
*(block.block_id for block in load_blocks.blocks[0][2:5]),
load_blocks.blocks[1][2].block_id,
]
assert len(load_meta.load_cpu_blocks) == 4
def test_finished_eager_store_caches_fa_partial_tail_for_hybrid_hit() -> None:
"""A finished non-LCM prompt needs the FA partial tail in CPU too."""
hash_block_size = BLOCK_SIZE
mamba_block_size = BLOCK_SIZE
attention_block_size = 4 * BLOCK_SIZE
dcp_world_size = 2
scheduler_block_size = attention_block_size * dcp_world_size
total_cached_tokens = 5 * hash_block_size
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=32,
num_gpu_blocks=32,
attention_block_size=attention_block_size,
block_size=mamba_block_size,
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
dcp_world_size=dcp_world_size,
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
assert sched.fa_block_size == scheduler_block_size
assert total_cached_tokens % sched.fa_block_size != 0
producer = _make_cp_request(num_blocks=5, virtual_block_size=hash_block_size)
attention_blocks = gpu_pool.get_new_blocks(1)
gpu_pool.cache_partial_block(
request=producer,
block=attention_blocks[0],
num_tokens=total_cached_tokens,
kv_cache_group_id=0,
block_size=scheduler_block_size,
)
mamba_blocks = gpu_pool.get_new_blocks(5)
gpu_pool.cache_full_blocks(
request=producer,
blocks=mamba_blocks,
num_cached_blocks=0,
num_full_blocks=5,
kv_cache_group_id=1,
block_size=mamba_block_size,
)
producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0)
store_output = make_scheduler_output(
{producer.request_id: total_cached_tokens},
new_reqs={producer.request_id: producer_blocks.get_block_ids()},
)
store_output.kv_connector_block_state = KVConnectorBlockState(
req_ids=set(),
resolve_block_ids=lambda _: (),
boundary_state_offloads={
producer.request_id: [
(1, mamba_blocks[4].block_id, total_cached_tokens),
]
},
)
store_meta = sched.build_connector_meta(store_output)
simulate_store_completion(sched, store_meta.store_event)
producer.num_computed_tokens = total_cached_tokens
sched.request_finished_all_groups(producer, producer_blocks.get_block_ids())
finish_meta = sched.build_connector_meta(make_scheduler_output({}))
assert attention_blocks[0].block_id in finish_meta.store_gpu_blocks
simulate_store_completion(sched, finish_meta.store_event)
consumer = Request(
request_id="req-finished-partial-tail-load",
prompt_token_ids=producer.prompt_token_ids,
sampling_params=producer.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=producer._block_hasher,
)
hit_tokens, is_async = sched.get_num_new_matched_tokens(
consumer, num_computed_tokens=0
)
assert hit_tokens == total_cached_tokens
assert is_async is True
def test_finished_eager_store_does_not_duplicate_completed_partial_tail() -> None:
"""Decode can fill the boundary block; the scan then already covers it."""
hash_block_size = BLOCK_SIZE
mamba_block_size = BLOCK_SIZE
attention_block_size = 4 * BLOCK_SIZE
dcp_world_size = 2
scheduler_block_size = attention_block_size * dcp_world_size
prompt_tokens = 5 * hash_block_size
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=32,
num_gpu_blocks=32,
attention_block_size=attention_block_size,
block_size=mamba_block_size,
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
dcp_world_size=dcp_world_size,
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
assert prompt_tokens % sched.fa_block_size != 0
producer = _make_cp_request(num_blocks=5, virtual_block_size=hash_block_size)
attention_blocks = gpu_pool.get_new_blocks(1)
gpu_pool.cache_partial_block(
request=producer,
block=attention_blocks[0],
num_tokens=prompt_tokens,
kv_cache_group_id=0,
block_size=scheduler_block_size,
)
mamba_blocks = gpu_pool.get_new_blocks(5)
gpu_pool.cache_full_blocks(
request=producer,
blocks=mamba_blocks,
num_cached_blocks=0,
num_full_blocks=5,
kv_cache_group_id=1,
block_size=mamba_block_size,
)
producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0)
# Decode carries the request past the physical attention block boundary, so
# the positional scan now considers the same block the partial tail names.
producer.num_computed_tokens = sched.fa_block_size
sched.request_finished_all_groups(producer, producer_blocks.get_block_ids())
finish_meta = sched.build_connector_meta(make_scheduler_output({}))
stored = finish_meta.store_gpu_blocks
assert attention_blocks[0].block_id in stored
assert len(stored) == len(set(stored))
assert len(finish_meta.store_cpu_blocks) == len(stored)
def test_external_lookup_rejects_misaligned_local_prefix(caplog_vllm) -> None:
scheduler_block_size = 4 * BLOCK_SIZE
fix = _make_hybrid_attention_mamba_scheduler(block_size=scheduler_block_size)
request = _make_cp_request(num_blocks=2, virtual_block_size=scheduler_block_size)
# The ``vllm`` logger sets propagate=False, so plain ``caplog`` can observe
# nothing; ``caplog_vllm`` re-enables propagation for the duration.
with caplog_vllm.at_level(logging.WARNING):
hit_tokens, is_async = fix.scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=BLOCK_SIZE
)
assert (hit_tokens, is_async) == (0, False)
assert "requires scheduler-block-aligned local tokens" in caplog_vllm.text
def test_fine_grained_external_hits_require_eager_offload() -> None:
hash_block_size = BLOCK_SIZE
scheduler_block_size = 4 * BLOCK_SIZE
eager = _make_hybrid_attention_mamba_scheduler(
block_size=scheduler_block_size,
hash_block_size=hash_block_size,
)
lazy = _make_hybrid_attention_mamba_scheduler(
block_size=scheduler_block_size,
hash_block_size=hash_block_size,
lazy=True,
)
assert eager.scheduler.cpu_coordinator.enable_partial_hash_hits
assert not lazy.scheduler.cpu_coordinator.enable_partial_hash_hits
def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None:
"""Mamba stores must use exact handoffs, not historical block positions."""
block_size = 4 * BLOCK_SIZE
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=16, num_gpu_blocks=24, block_size=block_size
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers
assert attention_manager.has_positionally_stable_blocks
assert not mamba_manager.has_positionally_stable_blocks
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
attn_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
)
gpu_blocks = gpu_pool.get_new_blocks(2)
gpu_blocks[1].set_block_hash(make_block_hash_with_group_id(req.block_hashes[1], 1))
kv_blocks = KVCacheBlocks(blocks=(attn_blocks, gpu_blocks))
req.num_computed_tokens = 0
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
output = make_scheduler_output(
{req.request_id: 2 * block_size},
new_reqs={req.request_id: kv_blocks.get_block_ids()},
)
output.kv_connector_block_state = KVConnectorBlockState(
req_ids=set(),
resolve_block_ids=lambda _: (),
boundary_state_offloads={
req.request_id: [(1, gpu_blocks[1].block_id, 2 * block_size)]
},
)
meta1 = sched.build_connector_meta(output)
assert meta1.store_event >= 0
# Only the exact handoff lands this step. build_connector_meta runs before
# the scheduler commits this step's tokens, so the positional scan has
# nothing confirmed yet.
assert set(meta1.store_gpu_blocks) == {gpu_blocks[1].block_id}
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [0, 0]
simulate_store_completion(sched, meta1.store_event)
# Next step the attention blocks are confirmed and scanned positionally. A
# later hash update still must not make the mamba align snapshot a valid
# source; the explicit boundary-handoff tests cover the supported path.
gpu_blocks[0].set_block_hash(make_block_hash_with_group_id(req.block_hashes[0], 1))
req.num_computed_tokens = 2 * block_size
meta2 = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: 1},
cached_req_new_blocks={req.request_id: None},
)
)
assert set(meta2.store_gpu_blocks) == {block.block_id for block in attn_blocks}
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0]
simulate_store_completion(sched, meta2.store_event)
# Nothing further: mamba align is never picked up positionally.
meta3 = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: 1},
cached_req_new_blocks={req.request_id: None},
)
)
assert meta3.store_event == -1
assert meta3.store_gpu_blocks == []
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0]
def test_eager_store_scans_mamba_all_blocks_positionally() -> None:
"""Mamba all-mode retains positional identity and needs no handoff."""
block_size = 4 * BLOCK_SIZE
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=16,
num_gpu_blocks=24,
block_size=block_size,
mamba_cache_mode="all",
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers
assert attention_manager.has_positionally_stable_blocks
assert mamba_manager.has_positionally_stable_blocks
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
attention_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
)
mamba_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, 2, virtual_block_size=block_size, group_id=1
)
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
# build_connector_meta runs before the scheduler commits this step's
# tokens, so the positional scan picks the blocks up on the next step.
first = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: 2 * block_size},
new_reqs={req.request_id: kv_blocks.get_block_ids()},
)
)
assert first.store_gpu_blocks == []
req.num_computed_tokens = 2 * block_size
meta = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: 1},
cached_req_new_blocks={req.request_id: None},
)
)
assert set(meta.store_gpu_blocks) == {
*(block.block_id for block in attention_blocks),
*(block.block_id for block in mamba_blocks),
}
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 2]
@pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"])
def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None:
"""Handoffs for a request leaving this step must not be read.
The scheduler drains boundary handoffs without filtering by request
liveness, so a preempted or finished request can still offer block IDs.
Those blocks are being released and may already back another request, so
copying them would publish unrelated KV under a valid hash.
"""
block_size = 4 * BLOCK_SIZE
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=16, num_gpu_blocks=24, block_size=block_size
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
attn_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
)
mamba_blocks = gpu_pool.get_new_blocks(2)
mamba_blocks[1].set_block_hash(
make_block_hash_with_group_id(req.block_hashes[1], 1)
)
kv_blocks = KVCacheBlocks(blocks=(attn_blocks, mamba_blocks))
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
output = make_scheduler_output(
{req.request_id: 2 * block_size},
new_reqs={req.request_id: kv_blocks.get_block_ids()},
)
output.kv_connector_block_state = KVConnectorBlockState(
req_ids=set(),
resolve_block_ids=lambda _: (),
boundary_state_offloads={
req.request_id: [(1, mamba_blocks[1].block_id, 2 * block_size)]
},
)
if gone_via == "preempted":
output.preempted_req_ids = {req.request_id}
elif gone_via == "finished":
output.finished_req_ids = {req.request_id}
else:
sched._reqs_to_store.pop(req.request_id)
meta = sched.build_connector_meta(output)
# The exact-handoff block is never a store source for a departing request.
assert mamba_blocks[1].block_id not in meta.store_gpu_blocks
stats = sched.get_boundary_store_stats()
assert stats.dropped_request_gone == 1
assert stats.stored == 0
def test_boundary_handoff_keeps_block_meta_index_parallel() -> None:
"""Boundary stores must extend block_meta alongside the block ids.
``_process_store_completion`` indexes ``block_meta`` by position, so a
boundary handoff that appends only block ids shortens the list, trips its
length assertion and shifts every following entry's metadata.
"""
block_size = 4 * BLOCK_SIZE
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=16,
num_gpu_blocks=24,
block_size=block_size,
enable_kv_cache_events=True,
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
assert sched.enable_kv_cache_events
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
attn_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
)
mamba_blocks = gpu_pool.get_new_blocks(2)
mamba_blocks[1].set_block_hash(
make_block_hash_with_group_id(req.block_hashes[1], 1)
)
kv_blocks = KVCacheBlocks(blocks=(attn_blocks, mamba_blocks))
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
# Make the FA blocks positionally eligible in the same event as the
# explicit Mamba boundary handoff.
req.num_computed_tokens = 2 * block_size
output = make_scheduler_output(
{req.request_id: 2 * block_size},
new_reqs={req.request_id: kv_blocks.get_block_ids()},
)
output.kv_connector_block_state = KVConnectorBlockState(
req_ids=set(),
resolve_block_ids=lambda _: (),
boundary_state_offloads={
req.request_id: [(1, mamba_blocks[1].block_id, 2 * block_size)]
},
)
gpu_ids, cpu_ids, req_ids, block_meta = sched.prepare_store_specs(output)
assert mamba_blocks[1].block_id in gpu_ids
assert set(gpu_ids) == {
*(block.block_id for block in attn_blocks),
mamba_blocks[1].block_id,
}
assert req_ids == [req.request_id]
assert block_meta is not None
assert len(block_meta) == len(gpu_ids) == len(cpu_ids)
@@ -19,6 +19,7 @@ from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.simple_kv_offload.manager import (
BoundaryStoreStats,
SimpleCPUOffloadScheduler,
)
from vllm.v1.simple_kv_offload.metadata import (
@@ -300,6 +301,12 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
return self.scheduler_manager.has_pending_stores()
return False
def get_boundary_store_stats(self) -> BoundaryStoreStats | None:
"""Return cumulative boundary-handoff store diagnostics."""
if self.scheduler_manager is not None:
return self.scheduler_manager.get_boundary_store_stats()
return None
def take_events(self) -> Iterable[KVCacheEvent]:
if self.scheduler_manager is not None:
return self.scheduler_manager.take_events()
+12 -4
View File
@@ -155,6 +155,9 @@ class KVCacheCoordinator(ABC):
for manager in self.single_type_managers:
if isinstance(manager, MambaManager):
manager.drop_eagle_checkpoint_block = True
self.group_block_sizes = tuple(
manager.block_size for manager in self.single_type_managers
)
# A positive retention interval must be a multiple of the base hit granularity
# (``scheduler_block_size``) to land on real cache-hit boundaries.
@@ -605,6 +608,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
hash_block_size: int,
metrics_collector: KVCacheMetricsCollector | None = None,
num_prefill_lookahead: int = 0,
allow_partial_hash_hits: bool = True,
):
super().__init__(
kv_cache_config,
@@ -629,7 +633,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
# Only groups that participate in prefix caching must satisfy the
# divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool
# tail, block_size=kpool) are scratch buffers and excluded.
group_block_sizes = [
cacheable_block_sizes = [
manager.block_size
for manager, group in zip(
self.single_type_managers, kv_cache_config.kv_cache_groups
@@ -637,10 +641,10 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
if group.kv_cache_spec.prefix_cacheable
]
assert all(
block_size % hash_block_size == 0 for block_size in group_block_sizes
block_size % hash_block_size == 0 for block_size in cacheable_block_sizes
), (
"Each KV cache group's real block_size must be divisible by "
f"hash_block_size. block_sizes={group_block_sizes}, "
f"hash_block_size. block_sizes={cacheable_block_sizes}, "
f"hash_block_size={hash_block_size}"
)
assert pcp_world_size == 1, "PCP not support hybrid attn now."
@@ -669,7 +673,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
)
for g in kv_cache_config.kv_cache_groups
)
self.enable_partial_hash_hits = has_partial_mamba_group
self.enable_partial_hash_hits = (
allow_partial_hash_hits and has_partial_mamba_group
)
if self.enable_partial_hash_hits:
unsupported_partial_hit_managers = {
type(manager).__name__
@@ -987,6 +993,7 @@ def get_kv_cache_coordinator(
hash_block_size: int,
metrics_collector: KVCacheMetricsCollector | None = None,
num_prefill_lookahead: int = 0,
allow_partial_hash_hits: bool = True,
) -> KVCacheCoordinator:
if not enable_caching:
return KVCacheCoordinatorNoPrefixCache(
@@ -1030,4 +1037,5 @@ def get_kv_cache_coordinator(
hash_block_size=hash_block_size,
metrics_collector=metrics_collector,
num_prefill_lookahead=num_prefill_lookahead,
allow_partial_hash_hits=allow_partial_hash_hits,
)
@@ -48,6 +48,16 @@ class SingleTypeKVCacheManager(ABC):
supports_fine_grained_hash_lookup: ClassVar[bool] = False
@property
def has_positionally_stable_blocks(self) -> bool:
"""Whether positional offload scans can follow this block table.
True means already-scanned indices will not be reused for a different
token range. Managers may still null blocks that left their retention
window, as long as the cursor's positional history remains valid.
"""
return True
def __init__(
self,
kv_cache_spec: KVCacheSpec,
@@ -1423,6 +1433,12 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
class MambaManager(SingleTypeKVCacheManager):
supports_fine_grained_hash_lookup: ClassVar[bool] = True
@property
def has_positionally_stable_blocks(self) -> bool:
# Align-mode Mamba can null interior states and relocate speculative
# blocks in place. Other modes retain positional identity.
return self.mamba_cache_mode != "align"
def __init__(
self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs
) -> None:
+266 -66
View File
@@ -5,6 +5,7 @@
import contextlib
from collections.abc import Iterable
from dataclasses import dataclass, field, replace
from enum import Enum, auto
from typing import TYPE_CHECKING, Any
from vllm.config import VllmConfig
@@ -26,6 +27,7 @@ from vllm.v1.core.kv_cache_coordinator import (
from vllm.v1.core.kv_cache_utils import (
BlockHashWithGroupId,
ExternalBlockHash,
dcp_world_size_for_kv_cache_spec,
generate_block_hash_extra_keys,
get_block_hash,
get_group_id,
@@ -99,6 +101,36 @@ class StoreRequestState:
finished: bool = False
class _StoreAdmission(Enum):
READY = auto()
NULL_BLOCK = auto()
NOT_HASHED = auto()
IN_FLIGHT = auto()
ALREADY_CACHED = auto()
@dataclass
class BoundaryStoreStats:
"""Cumulative counters for boundary hand-off stores.
Every branch that declines a hand-off is silent by design: the boundary is
simply not offloaded and a later request misses. Without counters the only
symptom is a lower cache hit rate with nothing in the logs, so each decline
reason is counted and exposed through
``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by
``reset()``; not otherwise cleared.
"""
published: int = 0
stored: int = 0
dropped_cpu_full: int = 0
dropped_request_gone: int = 0
skipped_already_cached: int = 0
skipped_in_flight: int = 0
dropped_null_block: int = 0
dropped_not_hashed: int = 0
class SimpleCPUOffloadScheduler:
"""Scheduler-side manager for CPU offloading."""
@@ -141,16 +173,6 @@ class SimpleCPUOffloadScheduler:
self.fa_gidx = g_idx
break
assert 0 <= self.fa_gidx < len(self.cpu_kv_cache_config.kv_cache_groups)
# FA group's own block_size; divides scheduler_block_size (the LCM)
# but is NOT assumed to equal it.
self.fa_block_size: int = (
self.cpu_kv_cache_config.kv_cache_groups[
self.fa_gidx
].kv_cache_spec.block_size
* self.cp_world_size
)
assert self.block_size % self.fa_block_size == 0
logger.info(
"SimpleCPUOffloadScheduler: Allocating %d offload blocks "
"(%.2f GB, mode=%s, backend=%s)",
@@ -175,7 +197,13 @@ class SimpleCPUOffloadScheduler:
pcp_world_size=1,
scheduler_block_size=self.block_size,
hash_block_size=self.hash_block_size,
allow_partial_hash_hits=not lazy_offload,
)
self.group_block_sizes = self.cpu_coordinator.group_block_sizes
# FA group's own resolved block_size; divides scheduler_block_size (the
# LCM) but is NOT assumed to equal it.
self.fa_block_size: int = self.group_block_sizes[self.fa_gidx]
assert self.block_size % self.fa_block_size == 0
self.cpu_block_pool: BlockPool = self.cpu_coordinator.block_pool
# GPU block pool reference - bound after scheduler builds kv_cache_manager
self._gpu_block_pool: BlockPool | None = None
@@ -186,10 +214,13 @@ class SimpleCPUOffloadScheduler:
# the worker reports completions by event index, not request id.
self._load_event_to_reqs: dict[int, list[str]] = {}
# Pending (cpu_hit_blocks, hit_length) tuples from find_longest_cache_hit,
# kept pinned via touch() while awaiting update_state_after_alloc().
# Pending (cpu_hit_blocks, hit_length, num_computed_tokens) tuples from
# find_longest_cache_hit, kept pinned via touch() while awaiting
# update_state_after_alloc(). ``num_computed_tokens`` is the local
# prefix the hit was resolved against; the load path needs it to place
# external blocks and must not re-derive it from block metadata.
self._pending_cpu_hits: dict[
str, tuple[tuple[list[KVCacheBlock], ...], int]
str, tuple[tuple[list[KVCacheBlock], ...], int, int]
] = {}
# Store metadata
@@ -216,6 +247,7 @@ class SimpleCPUOffloadScheduler:
# Event counters
self._load_event_counter: int = 0
self._store_event_counter: int = 0
self.boundary_store_stats = BoundaryStoreStats()
# For TP/PP: track partial store completions across steps.
# Events must be reported by all world_size workers before considered complete.
@@ -267,7 +299,11 @@ class SimpleCPUOffloadScheduler:
target = 0
for g in kv_cache_config.kv_cache_groups:
spec = g.kv_cache_spec
block_size = spec.block_size * cp_world_size
# Only full attention is sharded across DCP ranks; replicated specs
# (mamba, sliding window, chunked-local) keep their own block size.
block_size = spec.block_size * dcp_world_size_for_kv_cache_spec(
spec, cp_world_size
)
if isinstance(spec, MambaSpec):
target += 2
elif isinstance(spec, SlidingWindowSpec):
@@ -296,6 +332,20 @@ class SimpleCPUOffloadScheduler:
if request.skip_reading_prefix_cache:
return 0, False
if num_computed_tokens % self.block_size != 0:
# Transfers are whole-block copies, so an external suffix cannot
# start in the middle of a destination block. When the GPU-side
# coordinator also serves fine-grained hits, the local prefix can
# land on a hash boundary that is not a scheduler-block boundary;
# such a request keeps its local hit and skips the external one.
logger.warning_once(
"SimpleCPU external lookup requires scheduler-block-aligned "
"local tokens, got %d tokens with scheduler block size %d.",
num_computed_tokens,
self.block_size,
)
return 0, False
num_skipped_hashes = num_computed_tokens // self.hash_block_size
remaining_hashes = request.block_hashes[num_skipped_hashes:]
@@ -318,6 +368,7 @@ class SimpleCPUOffloadScheduler:
self._pending_cpu_hits[request.request_id] = (
cpu_hit_blocks,
hit_length,
num_computed_tokens,
)
return hit_length, True
return 0, False
@@ -371,20 +422,19 @@ class SimpleCPUOffloadScheduler:
)
return
cpu_hit_blocks_full, _ = pending
cpu_hit_blocks_full, _, num_computed_tokens = pending
# ``num_external_tokens`` is LCM-aligned (checked per-group below),
# so this counts whole scheduler-aligned chunks of incoming tokens.
num_blocks_to_load = num_external_tokens // self.block_size
assert num_blocks_to_load > 0
num_cached_fa_blocks = sum(
blk.block_hash is not None for blk in blocks.blocks[self.fa_gidx]
assert num_computed_tokens % self.block_size == 0, (
"SimpleCPU external loads must start at a scheduler-block boundary"
)
# Fine-grained hits are hash-block aligned, not group-block aligned, so
# the per-group block count below rounds up instead of dividing exactly.
# Hash alignment is still required: it is what makes the accepted
# external suffix start on a boundary every group can address.
assert num_external_tokens % self.hash_block_size == 0, (
f"num_external_tokens={num_external_tokens} is not aligned to "
f"hash_block_size={self.hash_block_size}"
)
num_computed_tokens = num_cached_fa_blocks * self.fa_block_size
# Build transfer pairs across all groups.
total_computed_tokens = num_computed_tokens + num_external_tokens
kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups
# The scheduler may have accepted fewer blocks than
# get_num_new_matched_tokens() reported.
@@ -393,14 +443,8 @@ class SimpleCPUOffloadScheduler:
# the rest will be released along with the temp pin below.
cpu_hit_blocks: list[list[KVCacheBlock]] = []
for g in range(num_groups):
g_block_size = (
kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size
)
assert num_external_tokens % g_block_size == 0, (
f"num_external_tokens={num_external_tokens} not aligned to "
f"group {g} block_size={g_block_size}"
)
n_take_g = num_external_tokens // g_block_size
g_block_size = self.group_block_sizes[g]
n_take_g = cdiv(num_external_tokens, g_block_size)
cpu_hit_blocks.append(cpu_hit_blocks_full[g][:n_take_g])
gpu_block_ids: list[int] = []
@@ -413,14 +457,8 @@ class SimpleCPUOffloadScheduler:
if n_ext_g == 0:
continue
# Number of blocks in the computed range for this group.
g_block_size = (
kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size
)
n_computed_g = cdiv(total_computed_tokens, g_block_size)
# Back-trace: ext blocks sit at the tail of the computed range.
gpu_ext_start = n_computed_g - n_ext_g
g_block_size = self.group_block_sizes[g]
gpu_ext_start = num_computed_tokens // g_block_size
group_gpu_ids = block_ids_by_group[g]
for i, cpu_blk in enumerate(cpu_blocks_g):
@@ -614,6 +652,72 @@ class SimpleCPUOffloadScheduler:
num_groups = len(self.cpu_kv_cache_config.kv_cache_groups)
# Dedup against blocks already scheduled.
in_flight = self._in_flight_store_gpu_blocks
num_free = cpu_block_pool.get_num_free_blocks()
block_state = scheduler_output.kv_connector_block_state
boundary_offloads = (
block_state.boundary_state_offloads if block_state is not None else {}
)
stats = self.boundary_store_stats
preempted_req_ids = scheduler_output.preempted_req_ids or ()
for req_id, entries in boundary_offloads.items():
# The scheduler drains handoffs without filtering by request
# liveness. A request that finished or was preempted in this step
# is giving up its blocks, which may already have been reallocated
# to another request; reading them would publish unrelated KV under
# a valid hash. Drop conservatively, as the mooncake store does.
store_state = self._reqs_to_store.get(req_id)
if (
store_state is None
or store_state.finished
or req_id in preempted_req_ids
or req_id in scheduler_output.finished_req_ids
):
stats.published += len(entries)
stats.dropped_request_gone += len(entries)
continue
scheduled_for_req = False
# ``boundary_tokens`` is not needed here: the cache key comes from
# the handed-off block itself, not from the offered boundary.
for _, gpu_block_id, _ in entries:
stats.published += 1
gpu_block = gpu_block_pool.blocks[gpu_block_id]
admission = self._classify_store_candidate(gpu_block)
if admission is _StoreAdmission.NULL_BLOCK:
stats.dropped_null_block += 1
continue
if admission is _StoreAdmission.NOT_HASHED:
stats.dropped_not_hashed += 1
continue
if admission is _StoreAdmission.IN_FLIGHT:
stats.skipped_in_flight += 1
continue
if admission is _StoreAdmission.ALREADY_CACHED:
stats.skipped_already_cached += 1
continue
if num_free <= 0:
stats.dropped_cpu_full += 1
logger.warning_once(
"SimpleCPU dropped a boundary-state handoff because "
"the CPU block pool is full; this boundary will not be retried."
)
continue
cpu_block = cpu_block_pool.get_new_blocks(1)[0]
merged_gpu_block_ids.append(gpu_block_id)
merged_cpu_block_ids.append(cpu_block.block_id)
if merged_block_meta is not None:
# Keep the metadata list index-parallel with the block ids.
# The hand-off block's own hashes are resolved at completion,
# so there is nothing to capture here; emit the event without
# a payload, as the size-mismatch fallback already does.
merged_block_meta.append({})
in_flight.add(gpu_block_id)
gpu_block_pool.touch([gpu_block])
num_free -= 1
stats.stored += 1
scheduled_for_req = True
if scheduled_for_req:
req_ids.append(req_id)
for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output):
state = self._reqs_to_store.get(req_id)
@@ -641,7 +745,7 @@ class SimpleCPUOffloadScheduler:
self._select_eager_blocks_to_store(state, block_ids_by_group)
)
# --- Phase 2: Batch allocate CPU blocks ---
# Batch allocate the CPU destinations.
n_to_alloc = len(gpu_block_ids)
if n_to_alloc > 0:
cpu_blocks_alloc = cpu_block_pool.get_new_blocks(n_to_alloc)
@@ -674,6 +778,9 @@ class SimpleCPUOffloadScheduler:
for g in range(num_groups):
state.num_stored_blocks[g] += advanced_per_group[g]
# A request can contribute both boundary and positional blocks to the
# same event. Completion tracking is set-based, so keep one request ID.
req_ids = list(dict.fromkeys(req_ids))
return merged_gpu_block_ids, merged_cpu_block_ids, req_ids, merged_block_meta
def _select_eager_blocks_to_store(
@@ -694,19 +801,31 @@ class SimpleCPUOffloadScheduler:
)
request = state.request
confirmed_tokens = request.num_computed_tokens - request.num_output_placeholders
aligned_tokens = confirmed_tokens // self.block_size * self.block_size
# Truncate to the granularity a lookup can actually land on. With
# fine-grained hits that is hash_block_size; otherwise hits only land on
# the scheduler block (the group LCM). Using the LCM unconditionally
# would drop whole blocks that sit between the last LCM boundary and a
# reachable fine-grained boundary, so the other groups would hold that
# boundary and this one would not, and the joint hybrid lookup would
# reconcile to zero.
store_alignment = (
self.hash_block_size
if self.cpu_coordinator.enable_partial_hash_hits
else self.block_size
)
aligned_tokens = confirmed_tokens // store_alignment * store_alignment
num_free = self.cpu_block_pool.get_num_free_blocks()
for g, group_gpu_ids in enumerate(block_ids_by_group):
if len(gpu_block_ids) >= num_free:
break
group_manager = self.cpu_coordinator.single_type_managers[g]
if not group_manager.has_positionally_stable_blocks:
continue
# FIXME (yifan): handle CPU cache eviction, where
# num_stored_blocks can be stale and omit evicted blocks in
# the middle of the request.
group_size = (
self.cpu_kv_cache_config.kv_cache_groups[g].kv_cache_spec.block_size
* self.cp_world_size
)
group_size = self.group_block_sizes[g]
ready = min(len(group_gpu_ids), aligned_tokens // group_size)
resolved_hashes = (
resolve_block_hashes(
@@ -720,20 +839,16 @@ class SimpleCPUOffloadScheduler:
start = state.num_stored_blocks[g]
for i, gpu_block_id in enumerate(group_gpu_ids[start:ready], start=start):
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
if gpu_block.is_null or gpu_block.block_hash is None:
advanced_per_group[g] += 1
continue
if (
gpu_block_id in self._in_flight_store_gpu_blocks
or self.cpu_block_pool.cached_block_hash_to_block.get_one_block(
gpu_block.block_hash
)
is not None
self._classify_store_candidate(gpu_block)
is not _StoreAdmission.READY
):
advanced_per_group[g] += 1
continue
if len(gpu_block_ids) >= num_free:
break
primary_block_hash = gpu_block.block_hash
assert primary_block_hash is not None
gpu_block_ids.append(gpu_block_id)
advanced_per_group[g] += 1
if block_meta is not None:
@@ -750,7 +865,7 @@ class SimpleCPUOffloadScheduler:
request, token_start, token_end, curr_mm_idx
)
meta_by_hash = {
gpu_block.block_hash: BlockStoreMeta(
primary_block_hash: BlockStoreMeta(
token_ids=list(
request.all_token_ids[token_start:token_end]
),
@@ -766,7 +881,7 @@ class SimpleCPUOffloadScheduler:
block_hash = make_block_hash_with_group_id(
request.block_hashes[hash_idx], g
)
if block_hash == gpu_block.block_hash:
if block_hash is None or block_hash == primary_block_hash:
continue
hash_start = hash_idx * self.hash_block_size
hash_end = hash_start + self.hash_block_size
@@ -796,6 +911,25 @@ class SimpleCPUOffloadScheduler:
return gpu_block_ids, advanced_per_group, block_meta
def _classify_store_candidate(self, gpu_block: "KVCacheBlock") -> _StoreAdmission:
if gpu_block.is_null:
return _StoreAdmission.NULL_BLOCK
if gpu_block.block_hash is None:
return _StoreAdmission.NOT_HASHED
if gpu_block.block_id in self._in_flight_store_gpu_blocks:
return _StoreAdmission.IN_FLIGHT
if (
self.cpu_block_pool.cached_block_hash_to_block.get_one_block(
gpu_block.block_hash
)
is not None
):
return _StoreAdmission.ALREADY_CACHED
return _StoreAdmission.READY
def get_boundary_store_stats(self) -> BoundaryStoreStats:
return replace(self.boundary_store_stats)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Handle async transfer completions from worker.
@@ -886,12 +1020,7 @@ class SimpleCPUOffloadScheduler:
if self.enable_kv_cache_events:
meta_by_hash = block_meta[i] if block_meta is not None else {}
primary_group_idx = get_group_id(primary_hash)
primary_spec = self.cpu_kv_cache_config.kv_cache_groups[
primary_group_idx
].kv_cache_spec
primary_block_size = primary_spec.block_size * (
self.cp_world_size if not isinstance(primary_spec, MambaSpec) else 1
)
primary_block_size = self.group_block_sizes[primary_group_idx]
events_to_emit: list[
tuple[BlockHashWithGroupId, BlockStoreMeta | None, int]
] = [
@@ -1028,6 +1157,21 @@ class SimpleCPUOffloadScheduler:
if state is None or gpu_pool is None:
return
gpu_ids, _, block_meta = self._select_eager_blocks_to_store(state, block_ids)
partial_tail = self._find_fa_partial_tail_source(request, block_ids)
if (
partial_tail is not None
# Decode tokens can fill the boundary block, in which case the
# positional scan above already selected it. Appending again would
# allocate two CPU blocks for one GPU block.
and partial_tail not in gpu_ids
and len(gpu_ids) < self.cpu_block_pool.get_num_free_blocks()
):
gpu_ids.append(partial_tail)
if block_meta is not None:
# The block's own hashes are registered at completion, so no
# capture is needed here; emit the event without a payload, as
# the size-mismatch fallback above already does.
block_meta.append({})
if not gpu_ids:
return
cpu_blocks = self.cpu_block_pool.get_new_blocks(len(gpu_ids))
@@ -1037,9 +1181,64 @@ class SimpleCPUOffloadScheduler:
self._in_flight_store_gpu_blocks.update(gpu_ids)
gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids])
def _free_pending_cpu_hit(self, pending: tuple) -> None:
def _find_fa_partial_tail_source(
self, request: "Request", block_ids: tuple[list[int], ...]
) -> int | None:
"""Locate the full-attention prompt-tail block for a fine-grained hit.
Mirrors ``FullAttentionManager._cache_partial_tail_block``: only the
final prompt hash boundary is eligible, and boundaries that land on a
physical block edge are already covered by the positional scan.
Attention block tables are append-only, so the block is located
positionally, as the mooncake store and offloading connector also do;
no new core hand-off is required. The boundary key is registered at
completion along with the block's other hashes, so nothing has to be
captured here.
"""
if not self.cpu_coordinator.enable_partial_hash_hits:
return None
assert self._gpu_block_pool is not None
boundary_tokens = (
request.num_prompt_tokens // self.hash_block_size * self.hash_block_size
)
if boundary_tokens == 0 or boundary_tokens > request.num_computed_tokens:
return None
if boundary_tokens % self.fa_block_size == 0:
return None
block_idx = boundary_tokens // self.fa_block_size
fa_block_ids = block_ids[self.fa_gidx]
if block_idx >= len(fa_block_ids):
return None
gpu_block_id = fa_block_ids[block_idx]
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
if gpu_block.is_null or gpu_block_id in self._in_flight_store_gpu_blocks:
return None
hash_idx = boundary_tokens // self.hash_block_size - 1
if hash_idx >= len(request.block_hashes):
return None
block_hash = make_block_hash_with_group_id(
request.block_hashes[hash_idx], self.fa_gidx
)
# Only worth copying when the boundary key is actually registered on
# the source block; completion replays the block's own hashes.
if (
gpu_block.block_hash != block_hash
and block_hash
not in self._gpu_block_pool.cached_block_hashes_by_block.get(
gpu_block_id, ()
)
):
return None
if self.cpu_block_pool.cached_block_hash_to_block.get_one_block(block_hash):
return None
return gpu_block_id
def _free_pending_cpu_hit(
self, pending: tuple[tuple[list["KVCacheBlock"], ...], int, int]
) -> None:
"""Release the temporary CPU block pin taken in get_num_new_matched_tokens()."""
cpu_hit_blocks, _ = pending
cpu_hit_blocks, _hit_length, _num_computed_tokens = pending
blocks_to_free = [
blk for grp in cpu_hit_blocks for blk in grp if not blk.is_null
]
@@ -1140,6 +1339,7 @@ class SimpleCPUOffloadScheduler:
if event_idx in self._abandoned_store_event_to_blocks
}
self._cursor = None
self.boundary_store_stats = BoundaryStoreStats()
# NOTE: _load_event_counter / _store_event_counter are not
# reset as they are monotonic and must stay ahead of the workers
# high-water marks to avoid event index collisions