[Feature][SimpleCPU] Load fine-grained hybrid prefix hits (#54736)
Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
This commit is contained in:
@@ -714,17 +714,8 @@ def test_disk_mode_block_removed_relabeled_storage() -> None:
|
||||
assert ev.medium == MEDIUM_STORAGE
|
||||
|
||||
|
||||
def test_mamba_dcp_metadata_guard() -> None:
|
||||
"""Mamba+DCP: token_ids guard emits [] when capture and emission sizes differ.
|
||||
|
||||
The metadata capture at _prepare_eager_store_specs uses
|
||||
g_block_size = spec.block_size * cp_world_size = 32 * 2 = 64 to slice
|
||||
tokens. The event emission uses block_size = spec.block_size * 1 = 32
|
||||
for Mamba (SSM state is not CP-sharded). The guard in
|
||||
_process_store_completion detects len(token_ids) != block_size and emits
|
||||
token_ids=[] instead of misleading data. When #49962 fixes the capture
|
||||
path, the guard stops triggering and correct token_ids flow through.
|
||||
"""
|
||||
def test_mamba_align_skips_positional_event_metadata() -> None:
|
||||
"""Mamba align emits events only from explicit boundary handoffs."""
|
||||
vbs = BLOCK_SIZE * 2
|
||||
kv_cfg = _make_mixed_kv_cache_config(
|
||||
num_blocks=16, with_mamba=True, mamba_block_size=32
|
||||
@@ -739,8 +730,6 @@ def test_mamba_dcp_metadata_guard() -> None:
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
|
||||
# Mamba g_block_size = 32 * 2 = 64, so 2 Mamba blocks need 128 computed
|
||||
# tokens to be "ready"; FA g_block_size = 16 * 2 = 32 needs only 64.
|
||||
req = _make_cp_request(num_blocks=4, virtual_block_size=vbs)
|
||||
fa_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=0)
|
||||
mamba_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=1)
|
||||
@@ -759,32 +748,12 @@ def test_mamba_dcp_metadata_guard() -> None:
|
||||
|
||||
events = list(sched.take_events())
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 4, (
|
||||
f"expected 4 BlockStored (2 FA + 2 Mamba), got {len(stored)}"
|
||||
)
|
||||
|
||||
by_group: dict[int, list[BlockStored]] = {}
|
||||
for ev in stored:
|
||||
by_group.setdefault(ev.group_idx, []).append(ev)
|
||||
|
||||
# FA group: capture size (64) == emission size (64) → token_ids present.
|
||||
fa_events = by_group[0]
|
||||
assert len(fa_events) == 2
|
||||
for ev in fa_events:
|
||||
assert len(stored) == 2
|
||||
assert {ev.group_idx for ev in stored} == {0}
|
||||
for idx, ev in enumerate(stored):
|
||||
assert ev.block_size == vbs
|
||||
assert len(ev.token_ids) == vbs, (
|
||||
f"FA token_ids should be {vbs}, got {len(ev.token_ids)}"
|
||||
assert ev.token_ids == req.prompt_token_ids[idx * vbs : (idx + 1) * vbs]
|
||||
expected_hash = maybe_convert_block_hash(
|
||||
get_block_hash(make_block_hash_with_group_id(req.block_hashes[idx], 0))
|
||||
)
|
||||
|
||||
# Mamba group: capture size (64) != emission size (32) → guard emits [].
|
||||
mamba_events = by_group[1]
|
||||
assert len(mamba_events) == 2
|
||||
for ev in mamba_events:
|
||||
assert ev.block_size == 32
|
||||
assert ev.token_ids == [], (
|
||||
f"Mamba token_ids should be [] (guard), got {ev.token_ids}"
|
||||
)
|
||||
assert ev.parent_block_hash is None, (
|
||||
"Mamba parent_block_hash should be None (guard)"
|
||||
)
|
||||
assert ev.extra_keys is None, "Mamba extra_keys should be None (guard)"
|
||||
assert ev.block_hashes == [expected_hash]
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
@@ -18,6 +19,7 @@ from vllm.config import (
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
@@ -31,6 +33,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
)
|
||||
from vllm.v1.core.sched.output import (
|
||||
CachedRequestData,
|
||||
KVConnectorBlockState,
|
||||
NewRequestData,
|
||||
SchedulerOutput,
|
||||
)
|
||||
@@ -42,6 +45,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
@@ -2128,3 +2132,671 @@ def test_cp_lazy_target_blocks_scaling(cp_world_size: int) -> None:
|
||||
f"cp_world_size={cp_world_size}: target_cp={target_cp} should be "
|
||||
f"less than target_base={target_base}"
|
||||
)
|
||||
|
||||
|
||||
def _make_hybrid_attention_mamba_scheduler(
|
||||
*,
|
||||
num_cpu_blocks: int = 8,
|
||||
num_gpu_blocks: int = 16,
|
||||
attention_block_size: int = BLOCK_SIZE,
|
||||
block_size: int = 4 * BLOCK_SIZE,
|
||||
scheduler_block_size: int | None = None,
|
||||
hash_block_size: int | None = None,
|
||||
dcp_world_size: int = 4,
|
||||
lazy: bool = False,
|
||||
mamba_cache_mode: str = "align",
|
||||
enable_kv_cache_events: bool = False,
|
||||
) -> SchedulerFixture:
|
||||
"""Build a scheduler for one attention group plus one Mamba group."""
|
||||
scheduler_block_size = scheduler_block_size or block_size
|
||||
hash_block_size = hash_block_size or block_size
|
||||
attention_spec = FullAttentionSpec(
|
||||
block_size=attention_block_size,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
)
|
||||
groups = [
|
||||
KVCacheGroupSpec(["attention"], attention_spec),
|
||||
KVCacheGroupSpec(["mamba"], mamba_spec),
|
||||
]
|
||||
tensors = [
|
||||
KVCacheTensor(
|
||||
size=spec.page_size_bytes * num_gpu_blocks,
|
||||
layers=group.layer_names,
|
||||
layer_stride=spec.page_size_bytes * num_gpu_blocks,
|
||||
block_stride=spec.page_size_bytes,
|
||||
)
|
||||
for group, spec in zip(groups, (attention_spec, mamba_spec))
|
||||
]
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_gpu_blocks,
|
||||
kv_cache_tensors=tensors,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
vllm_config = _make_cp_vllm_config(dcp_world_size=dcp_world_size)
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = 0
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
if enable_kv_cache_events:
|
||||
vllm_config.kv_events_config = KVEventsConfig(enable_kv_cache_events=True)
|
||||
# _derive_cpu_config() scales the requested capacity against
|
||||
# kv_cache_tensors[0].size, so express it as a multiple of that tensor's
|
||||
# per-block size to get exactly num_cpu_blocks.
|
||||
cpu_capacity_bytes = tensors[0].size // num_gpu_blocks * num_cpu_blocks
|
||||
sched = SimpleCPUOffloadScheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
cpu_capacity_bytes=cpu_capacity_bytes,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
lazy_offload=lazy,
|
||||
)
|
||||
gpu_block_pool = BlockPool(
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
sched.bind_gpu_block_pool(gpu_block_pool)
|
||||
return SchedulerFixture(
|
||||
scheduler=sched,
|
||||
gpu_block_pool=gpu_block_pool,
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
|
||||
"""Replicated groups must not be scaled by the DCP world size.
|
||||
|
||||
``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP
|
||||
size and every other spec 1. Reconstructing a group's geometry as
|
||||
``spec.block_size * cp_world_size`` over-scales the replicated groups, so
|
||||
the eager store scan believes far fewer of their blocks are ready and
|
||||
silently offloads only a fraction of them.
|
||||
"""
|
||||
attention_block_size = BLOCK_SIZE
|
||||
mamba_block_size = 4 * BLOCK_SIZE
|
||||
dcp_world_size = 2
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=32,
|
||||
num_gpu_blocks=32,
|
||||
attention_block_size=attention_block_size,
|
||||
block_size=mamba_block_size,
|
||||
# Every prefix-cacheable group's resolved block size must be a multiple
|
||||
# of the hash block, so it cannot exceed the smallest of them.
|
||||
hash_block_size=BLOCK_SIZE,
|
||||
dcp_world_size=dcp_world_size,
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
attention_size, mamba_size = sched.group_block_sizes
|
||||
assert attention_size == attention_block_size * dcp_world_size
|
||||
assert mamba_size == mamba_block_size
|
||||
|
||||
confirmed = 2 * sched.block_size
|
||||
req = _make_cp_request(num_blocks=8, virtual_block_size=BLOCK_SIZE)
|
||||
attention_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, confirmed // attention_size, attention_size, group_id=0
|
||||
)
|
||||
mamba_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, confirmed // mamba_size, mamba_size, group_id=1
|
||||
)
|
||||
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: confirmed},
|
||||
new_reqs={req.request_id: kv_blocks.get_block_ids()},
|
||||
)
|
||||
)
|
||||
req.num_computed_tokens = confirmed
|
||||
meta = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: 1},
|
||||
cached_req_new_blocks={req.request_id: None},
|
||||
)
|
||||
)
|
||||
|
||||
# Both groups are positionally stable here, so every confirmed block of
|
||||
# both must be offloaded. Over-scaling the replicated group halves its
|
||||
# count.
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [
|
||||
confirmed // attention_size,
|
||||
confirmed // mamba_size,
|
||||
]
|
||||
assert set(meta.store_gpu_blocks) == {
|
||||
*(b.block_id for b in attention_blocks),
|
||||
*(b.block_id for b in mamba_blocks),
|
||||
}
|
||||
|
||||
|
||||
def test_dcp_mixed_cache_loads_fine_grained_external_hit() -> None:
|
||||
"""Map a fine-grained external suffix after a nonzero local prefix."""
|
||||
hash_block_size = BLOCK_SIZE
|
||||
mamba_block_size = 4 * BLOCK_SIZE
|
||||
local_tokens = 4 * BLOCK_SIZE
|
||||
# Span two Mamba blocks and end partially in the second one. This exercises
|
||||
# both cdiv(external_tokens, mamba_block_size) > 1 and the partial tail.
|
||||
external_tokens = 6 * BLOCK_SIZE
|
||||
total_cached_tokens = local_tokens + external_tokens
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=32,
|
||||
num_gpu_blocks=32,
|
||||
block_size=mamba_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
dcp_world_size=2,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
|
||||
assert external_tokens % mamba_block_size != 0
|
||||
assert sched.cpu_coordinator.enable_partial_hash_hits
|
||||
|
||||
producer = _make_cp_request(num_blocks=10, virtual_block_size=hash_block_size)
|
||||
attention_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool,
|
||||
producer,
|
||||
num_blocks=5,
|
||||
virtual_block_size=2 * BLOCK_SIZE,
|
||||
group_id=0,
|
||||
)
|
||||
mamba_blocks = gpu_pool.get_new_blocks(3)
|
||||
gpu_pool.cache_full_blocks(
|
||||
request=producer,
|
||||
blocks=mamba_blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=2,
|
||||
block_size=mamba_block_size,
|
||||
kv_cache_group_id=1,
|
||||
)
|
||||
gpu_pool.cache_partial_block(
|
||||
request=producer,
|
||||
block=mamba_blocks[2],
|
||||
num_tokens=total_cached_tokens,
|
||||
kv_cache_group_id=1,
|
||||
block_size=mamba_block_size,
|
||||
)
|
||||
producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
|
||||
sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0)
|
||||
|
||||
# Positional scanning stores append-only FA blocks. Mamba align blocks use
|
||||
# exact handoffs for every retained boundary, including the partial tail.
|
||||
store_output = make_scheduler_output(
|
||||
{producer.request_id: total_cached_tokens},
|
||||
new_reqs={producer.request_id: producer_blocks.get_block_ids()},
|
||||
)
|
||||
store_output.kv_connector_block_state = KVConnectorBlockState(
|
||||
req_ids=set(),
|
||||
resolve_block_ids=lambda _: (),
|
||||
boundary_state_offloads={
|
||||
producer.request_id: [
|
||||
(1, mamba_blocks[0].block_id, mamba_block_size),
|
||||
(1, mamba_blocks[1].block_id, 2 * mamba_block_size),
|
||||
(1, mamba_blocks[2].block_id, total_cached_tokens),
|
||||
]
|
||||
},
|
||||
)
|
||||
store_meta = sched.build_connector_meta(store_output)
|
||||
# The exact handoffs land in the step they arrive; the append-only FA scan
|
||||
# picks its blocks up once the scheduler has committed the tokens.
|
||||
assert set(store_meta.store_gpu_blocks) == {
|
||||
block.block_id for block in mamba_blocks
|
||||
}
|
||||
simulate_store_completion(sched, store_meta.store_event)
|
||||
producer.num_computed_tokens = total_cached_tokens
|
||||
fa_meta = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{producer.request_id: 1},
|
||||
cached_req_new_blocks={producer.request_id: None},
|
||||
)
|
||||
)
|
||||
assert set(fa_meta.store_gpu_blocks) == {
|
||||
block.block_id for block in attention_blocks
|
||||
}
|
||||
simulate_store_completion(sched, fa_meta.store_event)
|
||||
|
||||
consumer = Request(
|
||||
request_id="req-dcp-mixed-fine-load",
|
||||
prompt_token_ids=producer.prompt_token_ids,
|
||||
sampling_params=producer.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=producer._block_hasher,
|
||||
)
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(
|
||||
consumer, num_computed_tokens=local_tokens
|
||||
)
|
||||
assert hit_tokens == external_tokens
|
||||
assert is_async is True
|
||||
pending_blocks, pending_tokens, _ = sched._pending_cpu_hits[consumer.request_id]
|
||||
assert pending_tokens == external_tokens
|
||||
assert len(pending_blocks[1]) == 2
|
||||
assert pending_blocks[1][0].is_null
|
||||
assert not pending_blocks[1][1].is_null
|
||||
|
||||
local_attention = attention_blocks[:2]
|
||||
local_mamba = mamba_blocks[:1]
|
||||
load_blocks = KVCacheBlocks(
|
||||
blocks=(
|
||||
local_attention + gpu_pool.get_new_blocks(3),
|
||||
local_mamba + gpu_pool.get_new_blocks(2),
|
||||
)
|
||||
)
|
||||
sched.update_state_after_alloc(
|
||||
consumer,
|
||||
load_blocks,
|
||||
num_external_tokens=hit_tokens,
|
||||
)
|
||||
load_meta = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{consumer.request_id: 1},
|
||||
new_reqs={consumer.request_id: load_blocks.get_block_ids()},
|
||||
)
|
||||
)
|
||||
assert load_meta.load_gpu_blocks == [
|
||||
*(block.block_id for block in load_blocks.blocks[0][2:5]),
|
||||
load_blocks.blocks[1][2].block_id,
|
||||
]
|
||||
assert len(load_meta.load_cpu_blocks) == 4
|
||||
|
||||
|
||||
def test_finished_eager_store_caches_fa_partial_tail_for_hybrid_hit() -> None:
|
||||
"""A finished non-LCM prompt needs the FA partial tail in CPU too."""
|
||||
hash_block_size = BLOCK_SIZE
|
||||
mamba_block_size = BLOCK_SIZE
|
||||
attention_block_size = 4 * BLOCK_SIZE
|
||||
dcp_world_size = 2
|
||||
scheduler_block_size = attention_block_size * dcp_world_size
|
||||
total_cached_tokens = 5 * hash_block_size
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=32,
|
||||
num_gpu_blocks=32,
|
||||
attention_block_size=attention_block_size,
|
||||
block_size=mamba_block_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
dcp_world_size=dcp_world_size,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
assert sched.fa_block_size == scheduler_block_size
|
||||
assert total_cached_tokens % sched.fa_block_size != 0
|
||||
|
||||
producer = _make_cp_request(num_blocks=5, virtual_block_size=hash_block_size)
|
||||
attention_blocks = gpu_pool.get_new_blocks(1)
|
||||
gpu_pool.cache_partial_block(
|
||||
request=producer,
|
||||
block=attention_blocks[0],
|
||||
num_tokens=total_cached_tokens,
|
||||
kv_cache_group_id=0,
|
||||
block_size=scheduler_block_size,
|
||||
)
|
||||
mamba_blocks = gpu_pool.get_new_blocks(5)
|
||||
gpu_pool.cache_full_blocks(
|
||||
request=producer,
|
||||
blocks=mamba_blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=5,
|
||||
kv_cache_group_id=1,
|
||||
block_size=mamba_block_size,
|
||||
)
|
||||
producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
|
||||
sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0)
|
||||
|
||||
store_output = make_scheduler_output(
|
||||
{producer.request_id: total_cached_tokens},
|
||||
new_reqs={producer.request_id: producer_blocks.get_block_ids()},
|
||||
)
|
||||
store_output.kv_connector_block_state = KVConnectorBlockState(
|
||||
req_ids=set(),
|
||||
resolve_block_ids=lambda _: (),
|
||||
boundary_state_offloads={
|
||||
producer.request_id: [
|
||||
(1, mamba_blocks[4].block_id, total_cached_tokens),
|
||||
]
|
||||
},
|
||||
)
|
||||
store_meta = sched.build_connector_meta(store_output)
|
||||
simulate_store_completion(sched, store_meta.store_event)
|
||||
|
||||
producer.num_computed_tokens = total_cached_tokens
|
||||
sched.request_finished_all_groups(producer, producer_blocks.get_block_ids())
|
||||
finish_meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
assert attention_blocks[0].block_id in finish_meta.store_gpu_blocks
|
||||
simulate_store_completion(sched, finish_meta.store_event)
|
||||
|
||||
consumer = Request(
|
||||
request_id="req-finished-partial-tail-load",
|
||||
prompt_token_ids=producer.prompt_token_ids,
|
||||
sampling_params=producer.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=producer._block_hasher,
|
||||
)
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(
|
||||
consumer, num_computed_tokens=0
|
||||
)
|
||||
assert hit_tokens == total_cached_tokens
|
||||
assert is_async is True
|
||||
|
||||
|
||||
def test_finished_eager_store_does_not_duplicate_completed_partial_tail() -> None:
|
||||
"""Decode can fill the boundary block; the scan then already covers it."""
|
||||
hash_block_size = BLOCK_SIZE
|
||||
mamba_block_size = BLOCK_SIZE
|
||||
attention_block_size = 4 * BLOCK_SIZE
|
||||
dcp_world_size = 2
|
||||
scheduler_block_size = attention_block_size * dcp_world_size
|
||||
prompt_tokens = 5 * hash_block_size
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=32,
|
||||
num_gpu_blocks=32,
|
||||
attention_block_size=attention_block_size,
|
||||
block_size=mamba_block_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
dcp_world_size=dcp_world_size,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
assert prompt_tokens % sched.fa_block_size != 0
|
||||
|
||||
producer = _make_cp_request(num_blocks=5, virtual_block_size=hash_block_size)
|
||||
attention_blocks = gpu_pool.get_new_blocks(1)
|
||||
gpu_pool.cache_partial_block(
|
||||
request=producer,
|
||||
block=attention_blocks[0],
|
||||
num_tokens=prompt_tokens,
|
||||
kv_cache_group_id=0,
|
||||
block_size=scheduler_block_size,
|
||||
)
|
||||
mamba_blocks = gpu_pool.get_new_blocks(5)
|
||||
gpu_pool.cache_full_blocks(
|
||||
request=producer,
|
||||
blocks=mamba_blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=5,
|
||||
kv_cache_group_id=1,
|
||||
block_size=mamba_block_size,
|
||||
)
|
||||
producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
|
||||
sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0)
|
||||
|
||||
# Decode carries the request past the physical attention block boundary, so
|
||||
# the positional scan now considers the same block the partial tail names.
|
||||
producer.num_computed_tokens = sched.fa_block_size
|
||||
sched.request_finished_all_groups(producer, producer_blocks.get_block_ids())
|
||||
finish_meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
|
||||
stored = finish_meta.store_gpu_blocks
|
||||
assert attention_blocks[0].block_id in stored
|
||||
assert len(stored) == len(set(stored))
|
||||
assert len(finish_meta.store_cpu_blocks) == len(stored)
|
||||
|
||||
|
||||
def test_external_lookup_rejects_misaligned_local_prefix(caplog_vllm) -> None:
|
||||
scheduler_block_size = 4 * BLOCK_SIZE
|
||||
fix = _make_hybrid_attention_mamba_scheduler(block_size=scheduler_block_size)
|
||||
|
||||
request = _make_cp_request(num_blocks=2, virtual_block_size=scheduler_block_size)
|
||||
# The ``vllm`` logger sets propagate=False, so plain ``caplog`` can observe
|
||||
# nothing; ``caplog_vllm`` re-enables propagation for the duration.
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
hit_tokens, is_async = fix.scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens=BLOCK_SIZE
|
||||
)
|
||||
|
||||
assert (hit_tokens, is_async) == (0, False)
|
||||
assert "requires scheduler-block-aligned local tokens" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_fine_grained_external_hits_require_eager_offload() -> None:
|
||||
hash_block_size = BLOCK_SIZE
|
||||
scheduler_block_size = 4 * BLOCK_SIZE
|
||||
|
||||
eager = _make_hybrid_attention_mamba_scheduler(
|
||||
block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
lazy = _make_hybrid_attention_mamba_scheduler(
|
||||
block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
lazy=True,
|
||||
)
|
||||
|
||||
assert eager.scheduler.cpu_coordinator.enable_partial_hash_hits
|
||||
assert not lazy.scheduler.cpu_coordinator.enable_partial_hash_hits
|
||||
|
||||
|
||||
def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None:
|
||||
"""Mamba stores must use exact handoffs, not historical block positions."""
|
||||
block_size = 4 * BLOCK_SIZE
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=16, num_gpu_blocks=24, block_size=block_size
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers
|
||||
assert attention_manager.has_positionally_stable_blocks
|
||||
assert not mamba_manager.has_positionally_stable_blocks
|
||||
|
||||
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
|
||||
attn_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
|
||||
)
|
||||
gpu_blocks = gpu_pool.get_new_blocks(2)
|
||||
gpu_blocks[1].set_block_hash(make_block_hash_with_group_id(req.block_hashes[1], 1))
|
||||
kv_blocks = KVCacheBlocks(blocks=(attn_blocks, gpu_blocks))
|
||||
req.num_computed_tokens = 0
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
|
||||
output = make_scheduler_output(
|
||||
{req.request_id: 2 * block_size},
|
||||
new_reqs={req.request_id: kv_blocks.get_block_ids()},
|
||||
)
|
||||
output.kv_connector_block_state = KVConnectorBlockState(
|
||||
req_ids=set(),
|
||||
resolve_block_ids=lambda _: (),
|
||||
boundary_state_offloads={
|
||||
req.request_id: [(1, gpu_blocks[1].block_id, 2 * block_size)]
|
||||
},
|
||||
)
|
||||
meta1 = sched.build_connector_meta(output)
|
||||
assert meta1.store_event >= 0
|
||||
# Only the exact handoff lands this step. build_connector_meta runs before
|
||||
# the scheduler commits this step's tokens, so the positional scan has
|
||||
# nothing confirmed yet.
|
||||
assert set(meta1.store_gpu_blocks) == {gpu_blocks[1].block_id}
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [0, 0]
|
||||
simulate_store_completion(sched, meta1.store_event)
|
||||
|
||||
# Next step the attention blocks are confirmed and scanned positionally. A
|
||||
# later hash update still must not make the mamba align snapshot a valid
|
||||
# source; the explicit boundary-handoff tests cover the supported path.
|
||||
gpu_blocks[0].set_block_hash(make_block_hash_with_group_id(req.block_hashes[0], 1))
|
||||
req.num_computed_tokens = 2 * block_size
|
||||
meta2 = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: 1},
|
||||
cached_req_new_blocks={req.request_id: None},
|
||||
)
|
||||
)
|
||||
assert set(meta2.store_gpu_blocks) == {block.block_id for block in attn_blocks}
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0]
|
||||
simulate_store_completion(sched, meta2.store_event)
|
||||
|
||||
# Nothing further: mamba align is never picked up positionally.
|
||||
meta3 = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: 1},
|
||||
cached_req_new_blocks={req.request_id: None},
|
||||
)
|
||||
)
|
||||
assert meta3.store_event == -1
|
||||
assert meta3.store_gpu_blocks == []
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0]
|
||||
|
||||
|
||||
def test_eager_store_scans_mamba_all_blocks_positionally() -> None:
|
||||
"""Mamba all-mode retains positional identity and needs no handoff."""
|
||||
block_size = 4 * BLOCK_SIZE
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=16,
|
||||
num_gpu_blocks=24,
|
||||
block_size=block_size,
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers
|
||||
assert attention_manager.has_positionally_stable_blocks
|
||||
assert mamba_manager.has_positionally_stable_blocks
|
||||
|
||||
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
|
||||
attention_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
|
||||
)
|
||||
mamba_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, 2, virtual_block_size=block_size, group_id=1
|
||||
)
|
||||
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
|
||||
# build_connector_meta runs before the scheduler commits this step's
|
||||
# tokens, so the positional scan picks the blocks up on the next step.
|
||||
first = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: 2 * block_size},
|
||||
new_reqs={req.request_id: kv_blocks.get_block_ids()},
|
||||
)
|
||||
)
|
||||
assert first.store_gpu_blocks == []
|
||||
req.num_computed_tokens = 2 * block_size
|
||||
meta = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: 1},
|
||||
cached_req_new_blocks={req.request_id: None},
|
||||
)
|
||||
)
|
||||
|
||||
assert set(meta.store_gpu_blocks) == {
|
||||
*(block.block_id for block in attention_blocks),
|
||||
*(block.block_id for block in mamba_blocks),
|
||||
}
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"])
|
||||
def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None:
|
||||
"""Handoffs for a request leaving this step must not be read.
|
||||
|
||||
The scheduler drains boundary handoffs without filtering by request
|
||||
liveness, so a preempted or finished request can still offer block IDs.
|
||||
Those blocks are being released and may already back another request, so
|
||||
copying them would publish unrelated KV under a valid hash.
|
||||
"""
|
||||
block_size = 4 * BLOCK_SIZE
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=16, num_gpu_blocks=24, block_size=block_size
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
|
||||
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
|
||||
attn_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
|
||||
)
|
||||
mamba_blocks = gpu_pool.get_new_blocks(2)
|
||||
mamba_blocks[1].set_block_hash(
|
||||
make_block_hash_with_group_id(req.block_hashes[1], 1)
|
||||
)
|
||||
kv_blocks = KVCacheBlocks(blocks=(attn_blocks, mamba_blocks))
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
|
||||
output = make_scheduler_output(
|
||||
{req.request_id: 2 * block_size},
|
||||
new_reqs={req.request_id: kv_blocks.get_block_ids()},
|
||||
)
|
||||
output.kv_connector_block_state = KVConnectorBlockState(
|
||||
req_ids=set(),
|
||||
resolve_block_ids=lambda _: (),
|
||||
boundary_state_offloads={
|
||||
req.request_id: [(1, mamba_blocks[1].block_id, 2 * block_size)]
|
||||
},
|
||||
)
|
||||
if gone_via == "preempted":
|
||||
output.preempted_req_ids = {req.request_id}
|
||||
elif gone_via == "finished":
|
||||
output.finished_req_ids = {req.request_id}
|
||||
else:
|
||||
sched._reqs_to_store.pop(req.request_id)
|
||||
|
||||
meta = sched.build_connector_meta(output)
|
||||
|
||||
# The exact-handoff block is never a store source for a departing request.
|
||||
assert mamba_blocks[1].block_id not in meta.store_gpu_blocks
|
||||
stats = sched.get_boundary_store_stats()
|
||||
assert stats.dropped_request_gone == 1
|
||||
assert stats.stored == 0
|
||||
|
||||
|
||||
def test_boundary_handoff_keeps_block_meta_index_parallel() -> None:
|
||||
"""Boundary stores must extend block_meta alongside the block ids.
|
||||
|
||||
``_process_store_completion`` indexes ``block_meta`` by position, so a
|
||||
boundary handoff that appends only block ids shortens the list, trips its
|
||||
length assertion and shifts every following entry's metadata.
|
||||
"""
|
||||
block_size = 4 * BLOCK_SIZE
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=16,
|
||||
num_gpu_blocks=24,
|
||||
block_size=block_size,
|
||||
enable_kv_cache_events=True,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
assert sched.enable_kv_cache_events
|
||||
|
||||
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
|
||||
attn_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
|
||||
)
|
||||
mamba_blocks = gpu_pool.get_new_blocks(2)
|
||||
mamba_blocks[1].set_block_hash(
|
||||
make_block_hash_with_group_id(req.block_hashes[1], 1)
|
||||
)
|
||||
kv_blocks = KVCacheBlocks(blocks=(attn_blocks, mamba_blocks))
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
# Make the FA blocks positionally eligible in the same event as the
|
||||
# explicit Mamba boundary handoff.
|
||||
req.num_computed_tokens = 2 * block_size
|
||||
|
||||
output = make_scheduler_output(
|
||||
{req.request_id: 2 * block_size},
|
||||
new_reqs={req.request_id: kv_blocks.get_block_ids()},
|
||||
)
|
||||
output.kv_connector_block_state = KVConnectorBlockState(
|
||||
req_ids=set(),
|
||||
resolve_block_ids=lambda _: (),
|
||||
boundary_state_offloads={
|
||||
req.request_id: [(1, mamba_blocks[1].block_id, 2 * block_size)]
|
||||
},
|
||||
)
|
||||
|
||||
gpu_ids, cpu_ids, req_ids, block_meta = sched.prepare_store_specs(output)
|
||||
assert mamba_blocks[1].block_id in gpu_ids
|
||||
assert set(gpu_ids) == {
|
||||
*(block.block_id for block in attn_blocks),
|
||||
mamba_blocks[1].block_id,
|
||||
}
|
||||
assert req_ids == [req.request_id]
|
||||
assert block_meta is not None
|
||||
assert len(block_meta) == len(gpu_ids) == len(cpu_ids)
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.simple_kv_offload.manager import (
|
||||
BoundaryStoreStats,
|
||||
SimpleCPUOffloadScheduler,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
@@ -300,6 +301,12 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return self.scheduler_manager.has_pending_stores()
|
||||
return False
|
||||
|
||||
def get_boundary_store_stats(self) -> BoundaryStoreStats | None:
|
||||
"""Return cumulative boundary-handoff store diagnostics."""
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.get_boundary_store_stats()
|
||||
return None
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.take_events()
|
||||
|
||||
@@ -155,6 +155,9 @@ class KVCacheCoordinator(ABC):
|
||||
for manager in self.single_type_managers:
|
||||
if isinstance(manager, MambaManager):
|
||||
manager.drop_eagle_checkpoint_block = True
|
||||
self.group_block_sizes = tuple(
|
||||
manager.block_size for manager in self.single_type_managers
|
||||
)
|
||||
|
||||
# A positive retention interval must be a multiple of the base hit granularity
|
||||
# (``scheduler_block_size``) to land on real cache-hit boundaries.
|
||||
@@ -605,6 +608,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
hash_block_size: int,
|
||||
metrics_collector: KVCacheMetricsCollector | None = None,
|
||||
num_prefill_lookahead: int = 0,
|
||||
allow_partial_hash_hits: bool = True,
|
||||
):
|
||||
super().__init__(
|
||||
kv_cache_config,
|
||||
@@ -629,7 +633,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
# Only groups that participate in prefix caching must satisfy the
|
||||
# divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool
|
||||
# tail, block_size=kpool) are scratch buffers and excluded.
|
||||
group_block_sizes = [
|
||||
cacheable_block_sizes = [
|
||||
manager.block_size
|
||||
for manager, group in zip(
|
||||
self.single_type_managers, kv_cache_config.kv_cache_groups
|
||||
@@ -637,10 +641,10 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
if group.kv_cache_spec.prefix_cacheable
|
||||
]
|
||||
assert all(
|
||||
block_size % hash_block_size == 0 for block_size in group_block_sizes
|
||||
block_size % hash_block_size == 0 for block_size in cacheable_block_sizes
|
||||
), (
|
||||
"Each KV cache group's real block_size must be divisible by "
|
||||
f"hash_block_size. block_sizes={group_block_sizes}, "
|
||||
f"hash_block_size. block_sizes={cacheable_block_sizes}, "
|
||||
f"hash_block_size={hash_block_size}"
|
||||
)
|
||||
assert pcp_world_size == 1, "PCP not support hybrid attn now."
|
||||
@@ -669,7 +673,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
self.enable_partial_hash_hits = has_partial_mamba_group
|
||||
self.enable_partial_hash_hits = (
|
||||
allow_partial_hash_hits and has_partial_mamba_group
|
||||
)
|
||||
if self.enable_partial_hash_hits:
|
||||
unsupported_partial_hit_managers = {
|
||||
type(manager).__name__
|
||||
@@ -987,6 +993,7 @@ def get_kv_cache_coordinator(
|
||||
hash_block_size: int,
|
||||
metrics_collector: KVCacheMetricsCollector | None = None,
|
||||
num_prefill_lookahead: int = 0,
|
||||
allow_partial_hash_hits: bool = True,
|
||||
) -> KVCacheCoordinator:
|
||||
if not enable_caching:
|
||||
return KVCacheCoordinatorNoPrefixCache(
|
||||
@@ -1030,4 +1037,5 @@ def get_kv_cache_coordinator(
|
||||
hash_block_size=hash_block_size,
|
||||
metrics_collector=metrics_collector,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
allow_partial_hash_hits=allow_partial_hash_hits,
|
||||
)
|
||||
|
||||
@@ -48,6 +48,16 @@ class SingleTypeKVCacheManager(ABC):
|
||||
|
||||
supports_fine_grained_hash_lookup: ClassVar[bool] = False
|
||||
|
||||
@property
|
||||
def has_positionally_stable_blocks(self) -> bool:
|
||||
"""Whether positional offload scans can follow this block table.
|
||||
|
||||
True means already-scanned indices will not be reused for a different
|
||||
token range. Managers may still null blocks that left their retention
|
||||
window, as long as the cursor's positional history remains valid.
|
||||
"""
|
||||
return True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
@@ -1423,6 +1433,12 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
class MambaManager(SingleTypeKVCacheManager):
|
||||
supports_fine_grained_hash_lookup: ClassVar[bool] = True
|
||||
|
||||
@property
|
||||
def has_positionally_stable_blocks(self) -> bool:
|
||||
# Align-mode Mamba can null interior states and relocate speculative
|
||||
# blocks in place. Other modes retain positional identity.
|
||||
return self.mamba_cache_mode != "align"
|
||||
|
||||
def __init__(
|
||||
self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs
|
||||
) -> None:
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import contextlib
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
@@ -26,6 +27,7 @@ from vllm.v1.core.kv_cache_coordinator import (
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHashWithGroupId,
|
||||
ExternalBlockHash,
|
||||
dcp_world_size_for_kv_cache_spec,
|
||||
generate_block_hash_extra_keys,
|
||||
get_block_hash,
|
||||
get_group_id,
|
||||
@@ -99,6 +101,36 @@ class StoreRequestState:
|
||||
finished: bool = False
|
||||
|
||||
|
||||
class _StoreAdmission(Enum):
|
||||
READY = auto()
|
||||
NULL_BLOCK = auto()
|
||||
NOT_HASHED = auto()
|
||||
IN_FLIGHT = auto()
|
||||
ALREADY_CACHED = auto()
|
||||
|
||||
|
||||
@dataclass
|
||||
class BoundaryStoreStats:
|
||||
"""Cumulative counters for boundary hand-off stores.
|
||||
|
||||
Every branch that declines a hand-off is silent by design: the boundary is
|
||||
simply not offloaded and a later request misses. Without counters the only
|
||||
symptom is a lower cache hit rate with nothing in the logs, so each decline
|
||||
reason is counted and exposed through
|
||||
``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by
|
||||
``reset()``; not otherwise cleared.
|
||||
"""
|
||||
|
||||
published: int = 0
|
||||
stored: int = 0
|
||||
dropped_cpu_full: int = 0
|
||||
dropped_request_gone: int = 0
|
||||
skipped_already_cached: int = 0
|
||||
skipped_in_flight: int = 0
|
||||
dropped_null_block: int = 0
|
||||
dropped_not_hashed: int = 0
|
||||
|
||||
|
||||
class SimpleCPUOffloadScheduler:
|
||||
"""Scheduler-side manager for CPU offloading."""
|
||||
|
||||
@@ -141,16 +173,6 @@ class SimpleCPUOffloadScheduler:
|
||||
self.fa_gidx = g_idx
|
||||
break
|
||||
assert 0 <= self.fa_gidx < len(self.cpu_kv_cache_config.kv_cache_groups)
|
||||
# FA group's own block_size; divides scheduler_block_size (the LCM)
|
||||
# but is NOT assumed to equal it.
|
||||
self.fa_block_size: int = (
|
||||
self.cpu_kv_cache_config.kv_cache_groups[
|
||||
self.fa_gidx
|
||||
].kv_cache_spec.block_size
|
||||
* self.cp_world_size
|
||||
)
|
||||
assert self.block_size % self.fa_block_size == 0
|
||||
|
||||
logger.info(
|
||||
"SimpleCPUOffloadScheduler: Allocating %d offload blocks "
|
||||
"(%.2f GB, mode=%s, backend=%s)",
|
||||
@@ -175,7 +197,13 @@ class SimpleCPUOffloadScheduler:
|
||||
pcp_world_size=1,
|
||||
scheduler_block_size=self.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
allow_partial_hash_hits=not lazy_offload,
|
||||
)
|
||||
self.group_block_sizes = self.cpu_coordinator.group_block_sizes
|
||||
# FA group's own resolved block_size; divides scheduler_block_size (the
|
||||
# LCM) but is NOT assumed to equal it.
|
||||
self.fa_block_size: int = self.group_block_sizes[self.fa_gidx]
|
||||
assert self.block_size % self.fa_block_size == 0
|
||||
self.cpu_block_pool: BlockPool = self.cpu_coordinator.block_pool
|
||||
# GPU block pool reference - bound after scheduler builds kv_cache_manager
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
@@ -186,10 +214,13 @@ class SimpleCPUOffloadScheduler:
|
||||
# the worker reports completions by event index, not request id.
|
||||
self._load_event_to_reqs: dict[int, list[str]] = {}
|
||||
|
||||
# Pending (cpu_hit_blocks, hit_length) tuples from find_longest_cache_hit,
|
||||
# kept pinned via touch() while awaiting update_state_after_alloc().
|
||||
# Pending (cpu_hit_blocks, hit_length, num_computed_tokens) tuples from
|
||||
# find_longest_cache_hit, kept pinned via touch() while awaiting
|
||||
# update_state_after_alloc(). ``num_computed_tokens`` is the local
|
||||
# prefix the hit was resolved against; the load path needs it to place
|
||||
# external blocks and must not re-derive it from block metadata.
|
||||
self._pending_cpu_hits: dict[
|
||||
str, tuple[tuple[list[KVCacheBlock], ...], int]
|
||||
str, tuple[tuple[list[KVCacheBlock], ...], int, int]
|
||||
] = {}
|
||||
|
||||
# Store metadata
|
||||
@@ -216,6 +247,7 @@ class SimpleCPUOffloadScheduler:
|
||||
# Event counters
|
||||
self._load_event_counter: int = 0
|
||||
self._store_event_counter: int = 0
|
||||
self.boundary_store_stats = BoundaryStoreStats()
|
||||
|
||||
# For TP/PP: track partial store completions across steps.
|
||||
# Events must be reported by all world_size workers before considered complete.
|
||||
@@ -267,7 +299,11 @@ class SimpleCPUOffloadScheduler:
|
||||
target = 0
|
||||
for g in kv_cache_config.kv_cache_groups:
|
||||
spec = g.kv_cache_spec
|
||||
block_size = spec.block_size * cp_world_size
|
||||
# Only full attention is sharded across DCP ranks; replicated specs
|
||||
# (mamba, sliding window, chunked-local) keep their own block size.
|
||||
block_size = spec.block_size * dcp_world_size_for_kv_cache_spec(
|
||||
spec, cp_world_size
|
||||
)
|
||||
if isinstance(spec, MambaSpec):
|
||||
target += 2
|
||||
elif isinstance(spec, SlidingWindowSpec):
|
||||
@@ -296,6 +332,20 @@ class SimpleCPUOffloadScheduler:
|
||||
if request.skip_reading_prefix_cache:
|
||||
return 0, False
|
||||
|
||||
if num_computed_tokens % self.block_size != 0:
|
||||
# Transfers are whole-block copies, so an external suffix cannot
|
||||
# start in the middle of a destination block. When the GPU-side
|
||||
# coordinator also serves fine-grained hits, the local prefix can
|
||||
# land on a hash boundary that is not a scheduler-block boundary;
|
||||
# such a request keeps its local hit and skips the external one.
|
||||
logger.warning_once(
|
||||
"SimpleCPU external lookup requires scheduler-block-aligned "
|
||||
"local tokens, got %d tokens with scheduler block size %d.",
|
||||
num_computed_tokens,
|
||||
self.block_size,
|
||||
)
|
||||
return 0, False
|
||||
|
||||
num_skipped_hashes = num_computed_tokens // self.hash_block_size
|
||||
remaining_hashes = request.block_hashes[num_skipped_hashes:]
|
||||
|
||||
@@ -318,6 +368,7 @@ class SimpleCPUOffloadScheduler:
|
||||
self._pending_cpu_hits[request.request_id] = (
|
||||
cpu_hit_blocks,
|
||||
hit_length,
|
||||
num_computed_tokens,
|
||||
)
|
||||
return hit_length, True
|
||||
return 0, False
|
||||
@@ -371,20 +422,19 @@ class SimpleCPUOffloadScheduler:
|
||||
)
|
||||
return
|
||||
|
||||
cpu_hit_blocks_full, _ = pending
|
||||
cpu_hit_blocks_full, _, num_computed_tokens = pending
|
||||
|
||||
# ``num_external_tokens`` is LCM-aligned (checked per-group below),
|
||||
# so this counts whole scheduler-aligned chunks of incoming tokens.
|
||||
num_blocks_to_load = num_external_tokens // self.block_size
|
||||
assert num_blocks_to_load > 0
|
||||
num_cached_fa_blocks = sum(
|
||||
blk.block_hash is not None for blk in blocks.blocks[self.fa_gidx]
|
||||
assert num_computed_tokens % self.block_size == 0, (
|
||||
"SimpleCPU external loads must start at a scheduler-block boundary"
|
||||
)
|
||||
# Fine-grained hits are hash-block aligned, not group-block aligned, so
|
||||
# the per-group block count below rounds up instead of dividing exactly.
|
||||
# Hash alignment is still required: it is what makes the accepted
|
||||
# external suffix start on a boundary every group can address.
|
||||
assert num_external_tokens % self.hash_block_size == 0, (
|
||||
f"num_external_tokens={num_external_tokens} is not aligned to "
|
||||
f"hash_block_size={self.hash_block_size}"
|
||||
)
|
||||
num_computed_tokens = num_cached_fa_blocks * self.fa_block_size
|
||||
|
||||
# Build transfer pairs across all groups.
|
||||
total_computed_tokens = num_computed_tokens + num_external_tokens
|
||||
kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups
|
||||
|
||||
# The scheduler may have accepted fewer blocks than
|
||||
# get_num_new_matched_tokens() reported.
|
||||
@@ -393,14 +443,8 @@ class SimpleCPUOffloadScheduler:
|
||||
# the rest will be released along with the temp pin below.
|
||||
cpu_hit_blocks: list[list[KVCacheBlock]] = []
|
||||
for g in range(num_groups):
|
||||
g_block_size = (
|
||||
kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size
|
||||
)
|
||||
assert num_external_tokens % g_block_size == 0, (
|
||||
f"num_external_tokens={num_external_tokens} not aligned to "
|
||||
f"group {g} block_size={g_block_size}"
|
||||
)
|
||||
n_take_g = num_external_tokens // g_block_size
|
||||
g_block_size = self.group_block_sizes[g]
|
||||
n_take_g = cdiv(num_external_tokens, g_block_size)
|
||||
cpu_hit_blocks.append(cpu_hit_blocks_full[g][:n_take_g])
|
||||
|
||||
gpu_block_ids: list[int] = []
|
||||
@@ -413,14 +457,8 @@ class SimpleCPUOffloadScheduler:
|
||||
if n_ext_g == 0:
|
||||
continue
|
||||
|
||||
# Number of blocks in the computed range for this group.
|
||||
g_block_size = (
|
||||
kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size
|
||||
)
|
||||
n_computed_g = cdiv(total_computed_tokens, g_block_size)
|
||||
|
||||
# Back-trace: ext blocks sit at the tail of the computed range.
|
||||
gpu_ext_start = n_computed_g - n_ext_g
|
||||
g_block_size = self.group_block_sizes[g]
|
||||
gpu_ext_start = num_computed_tokens // g_block_size
|
||||
group_gpu_ids = block_ids_by_group[g]
|
||||
|
||||
for i, cpu_blk in enumerate(cpu_blocks_g):
|
||||
@@ -614,6 +652,72 @@ class SimpleCPUOffloadScheduler:
|
||||
num_groups = len(self.cpu_kv_cache_config.kv_cache_groups)
|
||||
# Dedup against blocks already scheduled.
|
||||
in_flight = self._in_flight_store_gpu_blocks
|
||||
num_free = cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
block_state = scheduler_output.kv_connector_block_state
|
||||
boundary_offloads = (
|
||||
block_state.boundary_state_offloads if block_state is not None else {}
|
||||
)
|
||||
stats = self.boundary_store_stats
|
||||
preempted_req_ids = scheduler_output.preempted_req_ids or ()
|
||||
for req_id, entries in boundary_offloads.items():
|
||||
# The scheduler drains handoffs without filtering by request
|
||||
# liveness. A request that finished or was preempted in this step
|
||||
# is giving up its blocks, which may already have been reallocated
|
||||
# to another request; reading them would publish unrelated KV under
|
||||
# a valid hash. Drop conservatively, as the mooncake store does.
|
||||
store_state = self._reqs_to_store.get(req_id)
|
||||
if (
|
||||
store_state is None
|
||||
or store_state.finished
|
||||
or req_id in preempted_req_ids
|
||||
or req_id in scheduler_output.finished_req_ids
|
||||
):
|
||||
stats.published += len(entries)
|
||||
stats.dropped_request_gone += len(entries)
|
||||
continue
|
||||
scheduled_for_req = False
|
||||
# ``boundary_tokens`` is not needed here: the cache key comes from
|
||||
# the handed-off block itself, not from the offered boundary.
|
||||
for _, gpu_block_id, _ in entries:
|
||||
stats.published += 1
|
||||
gpu_block = gpu_block_pool.blocks[gpu_block_id]
|
||||
admission = self._classify_store_candidate(gpu_block)
|
||||
if admission is _StoreAdmission.NULL_BLOCK:
|
||||
stats.dropped_null_block += 1
|
||||
continue
|
||||
if admission is _StoreAdmission.NOT_HASHED:
|
||||
stats.dropped_not_hashed += 1
|
||||
continue
|
||||
if admission is _StoreAdmission.IN_FLIGHT:
|
||||
stats.skipped_in_flight += 1
|
||||
continue
|
||||
if admission is _StoreAdmission.ALREADY_CACHED:
|
||||
stats.skipped_already_cached += 1
|
||||
continue
|
||||
if num_free <= 0:
|
||||
stats.dropped_cpu_full += 1
|
||||
logger.warning_once(
|
||||
"SimpleCPU dropped a boundary-state handoff because "
|
||||
"the CPU block pool is full; this boundary will not be retried."
|
||||
)
|
||||
continue
|
||||
cpu_block = cpu_block_pool.get_new_blocks(1)[0]
|
||||
merged_gpu_block_ids.append(gpu_block_id)
|
||||
merged_cpu_block_ids.append(cpu_block.block_id)
|
||||
if merged_block_meta is not None:
|
||||
# Keep the metadata list index-parallel with the block ids.
|
||||
# The hand-off block's own hashes are resolved at completion,
|
||||
# so there is nothing to capture here; emit the event without
|
||||
# a payload, as the size-mismatch fallback already does.
|
||||
merged_block_meta.append({})
|
||||
in_flight.add(gpu_block_id)
|
||||
gpu_block_pool.touch([gpu_block])
|
||||
num_free -= 1
|
||||
stats.stored += 1
|
||||
scheduled_for_req = True
|
||||
if scheduled_for_req:
|
||||
req_ids.append(req_id)
|
||||
|
||||
for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output):
|
||||
state = self._reqs_to_store.get(req_id)
|
||||
@@ -641,7 +745,7 @@ class SimpleCPUOffloadScheduler:
|
||||
self._select_eager_blocks_to_store(state, block_ids_by_group)
|
||||
)
|
||||
|
||||
# --- Phase 2: Batch allocate CPU blocks ---
|
||||
# Batch allocate the CPU destinations.
|
||||
n_to_alloc = len(gpu_block_ids)
|
||||
if n_to_alloc > 0:
|
||||
cpu_blocks_alloc = cpu_block_pool.get_new_blocks(n_to_alloc)
|
||||
@@ -674,6 +778,9 @@ class SimpleCPUOffloadScheduler:
|
||||
for g in range(num_groups):
|
||||
state.num_stored_blocks[g] += advanced_per_group[g]
|
||||
|
||||
# A request can contribute both boundary and positional blocks to the
|
||||
# same event. Completion tracking is set-based, so keep one request ID.
|
||||
req_ids = list(dict.fromkeys(req_ids))
|
||||
return merged_gpu_block_ids, merged_cpu_block_ids, req_ids, merged_block_meta
|
||||
|
||||
def _select_eager_blocks_to_store(
|
||||
@@ -694,19 +801,31 @@ class SimpleCPUOffloadScheduler:
|
||||
)
|
||||
request = state.request
|
||||
confirmed_tokens = request.num_computed_tokens - request.num_output_placeholders
|
||||
aligned_tokens = confirmed_tokens // self.block_size * self.block_size
|
||||
# Truncate to the granularity a lookup can actually land on. With
|
||||
# fine-grained hits that is hash_block_size; otherwise hits only land on
|
||||
# the scheduler block (the group LCM). Using the LCM unconditionally
|
||||
# would drop whole blocks that sit between the last LCM boundary and a
|
||||
# reachable fine-grained boundary, so the other groups would hold that
|
||||
# boundary and this one would not, and the joint hybrid lookup would
|
||||
# reconcile to zero.
|
||||
store_alignment = (
|
||||
self.hash_block_size
|
||||
if self.cpu_coordinator.enable_partial_hash_hits
|
||||
else self.block_size
|
||||
)
|
||||
aligned_tokens = confirmed_tokens // store_alignment * store_alignment
|
||||
num_free = self.cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
for g, group_gpu_ids in enumerate(block_ids_by_group):
|
||||
if len(gpu_block_ids) >= num_free:
|
||||
break
|
||||
group_manager = self.cpu_coordinator.single_type_managers[g]
|
||||
if not group_manager.has_positionally_stable_blocks:
|
||||
continue
|
||||
# FIXME (yifan): handle CPU cache eviction, where
|
||||
# num_stored_blocks can be stale and omit evicted blocks in
|
||||
# the middle of the request.
|
||||
group_size = (
|
||||
self.cpu_kv_cache_config.kv_cache_groups[g].kv_cache_spec.block_size
|
||||
* self.cp_world_size
|
||||
)
|
||||
group_size = self.group_block_sizes[g]
|
||||
ready = min(len(group_gpu_ids), aligned_tokens // group_size)
|
||||
resolved_hashes = (
|
||||
resolve_block_hashes(
|
||||
@@ -720,20 +839,16 @@ class SimpleCPUOffloadScheduler:
|
||||
start = state.num_stored_blocks[g]
|
||||
for i, gpu_block_id in enumerate(group_gpu_ids[start:ready], start=start):
|
||||
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
|
||||
if gpu_block.is_null or gpu_block.block_hash is None:
|
||||
advanced_per_group[g] += 1
|
||||
continue
|
||||
if (
|
||||
gpu_block_id in self._in_flight_store_gpu_blocks
|
||||
or self.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
gpu_block.block_hash
|
||||
)
|
||||
is not None
|
||||
self._classify_store_candidate(gpu_block)
|
||||
is not _StoreAdmission.READY
|
||||
):
|
||||
advanced_per_group[g] += 1
|
||||
continue
|
||||
if len(gpu_block_ids) >= num_free:
|
||||
break
|
||||
primary_block_hash = gpu_block.block_hash
|
||||
assert primary_block_hash is not None
|
||||
gpu_block_ids.append(gpu_block_id)
|
||||
advanced_per_group[g] += 1
|
||||
if block_meta is not None:
|
||||
@@ -750,7 +865,7 @@ class SimpleCPUOffloadScheduler:
|
||||
request, token_start, token_end, curr_mm_idx
|
||||
)
|
||||
meta_by_hash = {
|
||||
gpu_block.block_hash: BlockStoreMeta(
|
||||
primary_block_hash: BlockStoreMeta(
|
||||
token_ids=list(
|
||||
request.all_token_ids[token_start:token_end]
|
||||
),
|
||||
@@ -766,7 +881,7 @@ class SimpleCPUOffloadScheduler:
|
||||
block_hash = make_block_hash_with_group_id(
|
||||
request.block_hashes[hash_idx], g
|
||||
)
|
||||
if block_hash == gpu_block.block_hash:
|
||||
if block_hash is None or block_hash == primary_block_hash:
|
||||
continue
|
||||
hash_start = hash_idx * self.hash_block_size
|
||||
hash_end = hash_start + self.hash_block_size
|
||||
@@ -796,6 +911,25 @@ class SimpleCPUOffloadScheduler:
|
||||
|
||||
return gpu_block_ids, advanced_per_group, block_meta
|
||||
|
||||
def _classify_store_candidate(self, gpu_block: "KVCacheBlock") -> _StoreAdmission:
|
||||
if gpu_block.is_null:
|
||||
return _StoreAdmission.NULL_BLOCK
|
||||
if gpu_block.block_hash is None:
|
||||
return _StoreAdmission.NOT_HASHED
|
||||
if gpu_block.block_id in self._in_flight_store_gpu_blocks:
|
||||
return _StoreAdmission.IN_FLIGHT
|
||||
if (
|
||||
self.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
gpu_block.block_hash
|
||||
)
|
||||
is not None
|
||||
):
|
||||
return _StoreAdmission.ALREADY_CACHED
|
||||
return _StoreAdmission.READY
|
||||
|
||||
def get_boundary_store_stats(self) -> BoundaryStoreStats:
|
||||
return replace(self.boundary_store_stats)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Handle async transfer completions from worker.
|
||||
|
||||
@@ -886,12 +1020,7 @@ class SimpleCPUOffloadScheduler:
|
||||
if self.enable_kv_cache_events:
|
||||
meta_by_hash = block_meta[i] if block_meta is not None else {}
|
||||
primary_group_idx = get_group_id(primary_hash)
|
||||
primary_spec = self.cpu_kv_cache_config.kv_cache_groups[
|
||||
primary_group_idx
|
||||
].kv_cache_spec
|
||||
primary_block_size = primary_spec.block_size * (
|
||||
self.cp_world_size if not isinstance(primary_spec, MambaSpec) else 1
|
||||
)
|
||||
primary_block_size = self.group_block_sizes[primary_group_idx]
|
||||
events_to_emit: list[
|
||||
tuple[BlockHashWithGroupId, BlockStoreMeta | None, int]
|
||||
] = [
|
||||
@@ -1028,6 +1157,21 @@ class SimpleCPUOffloadScheduler:
|
||||
if state is None or gpu_pool is None:
|
||||
return
|
||||
gpu_ids, _, block_meta = self._select_eager_blocks_to_store(state, block_ids)
|
||||
partial_tail = self._find_fa_partial_tail_source(request, block_ids)
|
||||
if (
|
||||
partial_tail is not None
|
||||
# Decode tokens can fill the boundary block, in which case the
|
||||
# positional scan above already selected it. Appending again would
|
||||
# allocate two CPU blocks for one GPU block.
|
||||
and partial_tail not in gpu_ids
|
||||
and len(gpu_ids) < self.cpu_block_pool.get_num_free_blocks()
|
||||
):
|
||||
gpu_ids.append(partial_tail)
|
||||
if block_meta is not None:
|
||||
# The block's own hashes are registered at completion, so no
|
||||
# capture is needed here; emit the event without a payload, as
|
||||
# the size-mismatch fallback above already does.
|
||||
block_meta.append({})
|
||||
if not gpu_ids:
|
||||
return
|
||||
cpu_blocks = self.cpu_block_pool.get_new_blocks(len(gpu_ids))
|
||||
@@ -1037,9 +1181,64 @@ class SimpleCPUOffloadScheduler:
|
||||
self._in_flight_store_gpu_blocks.update(gpu_ids)
|
||||
gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids])
|
||||
|
||||
def _free_pending_cpu_hit(self, pending: tuple) -> None:
|
||||
def _find_fa_partial_tail_source(
|
||||
self, request: "Request", block_ids: tuple[list[int], ...]
|
||||
) -> int | None:
|
||||
"""Locate the full-attention prompt-tail block for a fine-grained hit.
|
||||
|
||||
Mirrors ``FullAttentionManager._cache_partial_tail_block``: only the
|
||||
final prompt hash boundary is eligible, and boundaries that land on a
|
||||
physical block edge are already covered by the positional scan.
|
||||
|
||||
Attention block tables are append-only, so the block is located
|
||||
positionally, as the mooncake store and offloading connector also do;
|
||||
no new core hand-off is required. The boundary key is registered at
|
||||
completion along with the block's other hashes, so nothing has to be
|
||||
captured here.
|
||||
"""
|
||||
if not self.cpu_coordinator.enable_partial_hash_hits:
|
||||
return None
|
||||
assert self._gpu_block_pool is not None
|
||||
boundary_tokens = (
|
||||
request.num_prompt_tokens // self.hash_block_size * self.hash_block_size
|
||||
)
|
||||
if boundary_tokens == 0 or boundary_tokens > request.num_computed_tokens:
|
||||
return None
|
||||
if boundary_tokens % self.fa_block_size == 0:
|
||||
return None
|
||||
block_idx = boundary_tokens // self.fa_block_size
|
||||
fa_block_ids = block_ids[self.fa_gidx]
|
||||
if block_idx >= len(fa_block_ids):
|
||||
return None
|
||||
gpu_block_id = fa_block_ids[block_idx]
|
||||
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
|
||||
if gpu_block.is_null or gpu_block_id in self._in_flight_store_gpu_blocks:
|
||||
return None
|
||||
hash_idx = boundary_tokens // self.hash_block_size - 1
|
||||
if hash_idx >= len(request.block_hashes):
|
||||
return None
|
||||
block_hash = make_block_hash_with_group_id(
|
||||
request.block_hashes[hash_idx], self.fa_gidx
|
||||
)
|
||||
# Only worth copying when the boundary key is actually registered on
|
||||
# the source block; completion replays the block's own hashes.
|
||||
if (
|
||||
gpu_block.block_hash != block_hash
|
||||
and block_hash
|
||||
not in self._gpu_block_pool.cached_block_hashes_by_block.get(
|
||||
gpu_block_id, ()
|
||||
)
|
||||
):
|
||||
return None
|
||||
if self.cpu_block_pool.cached_block_hash_to_block.get_one_block(block_hash):
|
||||
return None
|
||||
return gpu_block_id
|
||||
|
||||
def _free_pending_cpu_hit(
|
||||
self, pending: tuple[tuple[list["KVCacheBlock"], ...], int, int]
|
||||
) -> None:
|
||||
"""Release the temporary CPU block pin taken in get_num_new_matched_tokens()."""
|
||||
cpu_hit_blocks, _ = pending
|
||||
cpu_hit_blocks, _hit_length, _num_computed_tokens = pending
|
||||
blocks_to_free = [
|
||||
blk for grp in cpu_hit_blocks for blk in grp if not blk.is_null
|
||||
]
|
||||
@@ -1140,6 +1339,7 @@ class SimpleCPUOffloadScheduler:
|
||||
if event_idx in self._abandoned_store_event_to_blocks
|
||||
}
|
||||
self._cursor = None
|
||||
self.boundary_store_stats = BoundaryStoreStats()
|
||||
# NOTE: _load_event_counter / _store_event_counter are not
|
||||
# reset as they are monotonic and must stay ahead of the workers
|
||||
# high-water marks to avoid event index collisions
|
||||
|
||||
Reference in New Issue
Block a user