From d74285661064541f12be14e49a16defa27de67c6 Mon Sep 17 00:00:00 2001 From: Dao007forever Date: Sun, 26 Jul 2026 23:22:17 -0700 Subject: [PATCH] [3/N][Core][KV Connector] Support reliable partial-tail KV offload for sub-block prompts (#49502) Signed-off-by: Dao Le Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../test_partial_prefix_cache_hits.py | 293 ++++++++++++++++++ .../unit/test_mooncake_store_coordinator.py | 55 ++++ .../unit/test_mooncake_store_hma_e2e.py | 247 ++++++++++++++- .../unit/test_mooncake_store_scheduler.py | 232 +++++++++++++- .../unit/test_mooncake_store_worker.py | 122 ++++++++ .../v1/mooncake/store/coordinator.py | 66 +++- .../kv_connector/v1/mooncake/store/data.py | 35 ++- .../v1/mooncake/store/scheduler.py | 55 +++- .../kv_connector/v1/mooncake/store/worker.py | 209 ++++++++++++- vllm/v1/core/kv_cache_manager.py | 63 +++- vllm/v1/core/sched/output.py | 6 + vllm/v1/core/sched/scheduler.py | 48 ++- vllm/v1/core/single_type_kv_cache_manager.py | 66 +++- 13 files changed, 1448 insertions(+), 49 deletions(-) diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index a42fc41501..70457d5af9 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -223,6 +223,299 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue(): assert moved[0].block_hash_num_tokens == 6 +def test_take_partial_tail_offloads_returns_cow_target(): + """The connector offload hand-off exposes the mamba CoW *target* block Y + (the durable boundary state), not the overwritten source X, and only at + the CoW step.""" + hash_block_size = 2 + block_size = 2 * hash_block_size + kv_cache_config = KVCacheConfig( + num_blocks=24, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=hash_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ), + ), + ], + ) + manager = make_kv_cache_manager( + kv_cache_config=kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=hash_block_size, + ) + + req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) + assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None + + # Step A registered the partial tail but has not CoW'd yet: no offload. + assert manager.take_partial_tail_offloads() == {} + + partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1] + source_block = manager.block_pool.get_cached_block( + partial_mamba_hash, kv_cache_group_ids=[1] + ) + assert source_block is not None + source_block_id = source_block[0].block_id + + # Step B: the producer continues, triggering the CoW X->Y. + req0.num_computed_tokens = 6 + req0.append_output_token_ids([3]) + assert manager.allocate_slots(req0, 1) is not None + + offloads = manager.take_partial_tail_offloads() + assert list(offloads.keys()) == ["0"] + assert len(offloads["0"]) == 1 + group_id, block_id, boundary_tokens = offloads["0"][0] + assert group_id == 1 # the mamba group + assert boundary_tokens == 6 + copies, _ = manager.take_kv_cache_block_copies() + cow_copy = next(c for c in copies if c.src_block_id == source_block_id) + # The offload points at the durable CoW target Y, not the overwritten X. + assert block_id == cow_copy.dst_block_id + assert block_id != source_block_id + # Draining clears it. + assert manager.take_partial_tail_offloads() == {} + + # The hand-off pinned Y (its CoW retention is released after this step, + # and Y is off the request block table); freeing the request unpins it. + cow_block = manager.block_pool.blocks[block_id] + pinned_ref = cow_block.ref_cnt + assert pinned_ref >= 1 + manager.free(req0) + assert cow_block.ref_cnt == pinned_ref - 1 + + +def test_partial_tail_pin_survives_released_cow_retention(): + """If the CoW retention is released before the hand-off is drained + (immediate-free mode), the drain must rescue the cow block from the free + queue: a raw ref increment would leave a ref>0 block allocatable, and the + next allocation would pop it and assert.""" + hash_block_size = 2 + block_size = 2 * hash_block_size + kv_cache_config = KVCacheConfig( + num_blocks=24, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=hash_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ), + ), + ], + ) + manager = make_kv_cache_manager( + kv_cache_config=kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=hash_block_size, + ) + req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) + assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None + req0.num_computed_tokens = 6 + req0.append_output_token_ids([3]) + assert manager.allocate_slots(req0, 1) is not None + + # Retention released before the drain (defer_block_free=False ordering). + _copies, retained = manager.take_kv_cache_block_copies() + manager.block_pool.free_blocks(retained) + + offloads = manager.take_partial_tail_offloads() + ((_group_id, block_id, boundary_tokens),) = offloads["0"] + assert boundary_tokens == 6 + cow_block = manager.block_pool.blocks[block_id] + assert cow_block.ref_cnt == 1 + + # The pinned block is out of the free queue: draining every free block + # neither trips the allocator's ref_cnt assert nor hands it out. + new_blocks = manager.block_pool.get_new_blocks( + manager.block_pool.get_num_free_blocks() + ) + assert block_id not in {b.block_id for b in new_blocks} + + +def test_partial_tail_offload_dropped_when_request_freed_before_drain(): + """A hand-off recorded in the same scheduling pass as the request's death + must not be drained: its release hook has already run, so draining would + leak a pinned block.""" + hash_block_size = 2 + block_size = 2 * hash_block_size + kv_cache_config = KVCacheConfig( + num_blocks=24, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=hash_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ), + ), + ], + ) + manager = make_kv_cache_manager( + kv_cache_config=kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=hash_block_size, + ) + req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) + assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None + req0.num_computed_tokens = 6 + req0.append_output_token_ids([3]) + assert manager.allocate_slots(req0, 1) is not None + + # The request dies (preempt/abort) before the scheduler drains. + manager.block_pool.free_blocks(manager.pop_blocks_for_free(req0)) + assert manager.take_partial_tail_offloads() == {} + + +def test_take_partial_tail_offloads_empty_without_partial_tail(): + """A prompt ending on a block boundary registers no partial tail, so there + is nothing to offload.""" + hash_block_size = 2 + block_size = 2 * hash_block_size + kv_cache_config = KVCacheConfig( + num_blocks=24, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=hash_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ), + ), + ], + ) + manager = make_kv_cache_manager( + kv_cache_config=kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=hash_block_size, + ) + + # 4-token prompt ends exactly on the mamba block boundary (block_size=4). + req0 = make_request("0", [0, 0, 1, 1], hash_block_size, sha256) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) + assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None + assert manager.take_partial_tail_offloads() == {} + + req0.num_computed_tokens = 4 + req0.append_output_token_ids([2]) + assert manager.allocate_slots(req0, 1) is not None + assert manager.take_partial_tail_offloads() == {} + + +def test_truncate_computed_blocks_preserves_sparse_prefix_positions(): + """truncate_computed_blocks slices each group by its own block size, + keeps null placeholders in the retained prefix, and leaves the original + lookup result untouched (pure view, no refcount changes).""" + hash_block_size = 2 + kv_cache_config = KVCacheConfig( + num_blocks=24, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=hash_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=2 * hash_block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ), + ), + ], + ) + manager = make_kv_cache_manager( + kv_cache_config=kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=hash_block_size, + ) + producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) + blocks, num_computed, _ = manager.get_computed_blocks(producer) + assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None + manager.free(producer) + manager.new_step_starts() + + consumer = make_request( + "consumer", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256 + ) + blocks, num_computed, _ = manager.get_computed_blocks(consumer) + assert num_computed == 6 + assert [len(group) for group in blocks.blocks] == [3, 2] + assert blocks.blocks[1][0].is_null + + truncated = manager.truncate_computed_blocks(blocks, 4) + + assert [len(group) for group in truncated.blocks] == [2, 1] + assert truncated.blocks[1][0].is_null + assert [len(group) for group in blocks.blocks] == [3, 2] + + def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit(): hash_block_size = 2 block_size = 2 * hash_block_size diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py index 6e003798c7..6b56fd2f19 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py @@ -3,6 +3,8 @@ from math import lcm +import torch + from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501 ExternalCachedBlockPool, MooncakeStoreCoordinator, @@ -14,10 +16,20 @@ from vllm.v1.core.kv_cache_utils import BlockHash from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheGroupSpec, + MambaSpec, SlidingWindowSpec, ) +def _mamba_align(block_size=32): + return MambaSpec( + block_size=block_size, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + + def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=None): """Construct a coordinator using the natural LCM of group block sizes as the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for @@ -196,6 +208,49 @@ def test_coordinator_group_block_size_double_hash(): assert hit % 32 == 0 +# ----- Fine-grained partial hits (full attention + mamba "align") ----- + + +def test_coordinator_fine_grained_partial_tail_hit(): + """K3 shape: FA + mamba-align, block_size=32 over hash_block_size=16. When + both groups have the sub-block boundary hash, the reconciled hit lands on + the hash boundary (48), not the block boundary (32).""" + groups = [ + KVCacheGroupSpec(["L0"], _full(32)), + KVCacheGroupSpec(["L1"], _mamba_align(32)), + ] + coord = _make_coord(groups, hash_block_size=16) + assert coord.enable_partial_hash_hits + hs = _hashes(4) # 4 hash units of 16 = 64 tokens; block 0 = [0,32), etc. + # Both groups: full block 0 (key = last sub-hash hs[1]) + partial boundary + # at token 48 (key = hs[2]). No hs[3] -> block 1 is not full. + exists = {(g, bytes(h)) for g in (0, 1) for h in (hs[1], hs[2])} + cmap = ExternalCachedBlockPool(16, exists) + _masks, hit = coord.find_longest_cache_hit( + hs, max_length=64, cached_block_pool=cmap + ) + assert hit == 48 + + +def test_coordinator_fine_grained_clips_when_one_group_missing_tail(): + """If only one group has the sub-block boundary, min-convergence clips the + reconciled hit back to the block boundary (32).""" + groups = [ + KVCacheGroupSpec(["L0"], _full(32)), + KVCacheGroupSpec(["L1"], _mamba_align(32)), + ] + coord = _make_coord(groups, hash_block_size=16) + hs = _hashes(4) + # Full block 0 for both; partial boundary hs[2] only for FA (group 0). + exists = {(g, bytes(hs[1])) for g in (0, 1)} + exists |= {(0, bytes(hs[2]))} + cmap = ExternalCachedBlockPool(16, exists) + _masks, hit = coord.find_longest_cache_hit( + hs, max_length=64, cached_block_pool=cmap + ) + assert hit == 32 + + # ----- store_mask ----- diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py index 2ea776adf6..6c8af7fb73 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -32,6 +32,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, KVCacheTensor, + MambaSpec, SlidingWindowSpec, ) @@ -333,14 +334,14 @@ def test_recv_skips_swa_blocks_before_window(): def test_chunked_token_database_hash_block_size_smaller_than_block_size(): """DSv4-style: hash_block_size=4, group block_size=16 — process_tokens - keys each 16-token chunk by its last fine hash, keeping the Mooncake key - at one digest instead of concatenating all 4 fine hashes.""" + keys each chunk by its ending fine hash, including a partial tail.""" md = KeyMetadata("m", 0, 0, 0, 0, group_id=3) db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4) db.set_kv_caches_base_addr([0]) db.set_block_len([512]) - # 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks. fine_hashes = [BlockHash(bytes([i + 1]) * 4) for i in range(8)] + + # 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks. out = list(db.process_tokens(token_len=32, block_hashes=fine_hashes)) assert len(out) == 2 assert out[0][0] == 0 and out[0][1] == 16 @@ -349,3 +350,243 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size(): # prior three. assert out[0][2].hex() == fine_hashes[3].hex() assert out[1][2].hex() == fine_hashes[7].hex() + + # Sub-block hit: emit the partial chunk under its ending fine hash. + out = list(db.process_tokens(token_len=12, block_hashes=fine_hashes[:3])) + assert [(s, e) for s, e, _ in out] == [(0, 12)] + assert out[0][2].hex() == fine_hashes[2].hex() + + # Cross-block hit: emit both the full chunk and its partial tail. + out = list(db.process_tokens(token_len=28, block_hashes=fine_hashes[:7])) + assert [(s, e) for s, e, _ in out] == [(0, 16), (16, 28)] + assert out[0][2].hex() == fine_hashes[3].hex() + assert out[1][2].hex() == fine_hashes[6].hex() + + +def test_sub_block_partial_tail_offload_reads_cow_block(): + """Sub-block prompt (the 900/128/1536 shape, scaled to 12/4/16): the + partial tail is offloaded for both groups under the boundary sub-hash. The + full-attention block is read from the request block table; the mamba block + is the core-provided CoW target, not block_ids.""" + full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + mamba = MambaSpec( + block_size=16, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + groups = [ + KVCacheGroupSpec(["L0"], full), + KVCacheGroupSpec(["L1"], mamba), + ] + coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4) + assert coord.enable_partial_hash_hits + + class _RecordingStore(_DictStore): + def __init__(self): + super().__init__() + self.puts: dict[str, list[int]] = {} + + def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k): + for key, addr in zip(keys, addrs): + self.puts[key] = addr + return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k) + + store = _RecordingStore() + token_dbs = [] + for g_idx in range(2): + db = ChunkedTokenDatabase( + KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx), + block_size=16, + hash_block_size=4, + ) + db.set_kv_caches_base_addr([g_idx * 10_000]) + db.set_block_len([512]) + token_dbs.append(db) + + send = KVCacheStoreSendingThread( + store=store, + coord=coord, + token_databases=token_dbs, + block_size=16, + tp_rank=0, + group_put_steps=[1, 1], + kv_role="kv_both", + ready_event=threading.Event(), + replicate_config=MagicMock(), + ) + + # The surrounding metadata may describe a longer resumed replay, but the + # handoff identifies the exact state boundary to persist. + hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)] + mamba_cow_block = 7 + req = ReqMeta( + req_id="r0", + token_len_chunk=0, + block_ids=([1], [2]), + block_hashes=hs, + can_save=True, + num_prompt_tokens=20, + partial_tail_offloads=[(1, mamba_cow_block, 12)], + ) + + send._maybe_offload_partial_tail(req) + + # boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2]. + partial_hash = hs[2] + fa_key = token_dbs[0].key_for(partial_hash) + mamba_key = token_dbs[1].key_for(partial_hash) + assert set(store.puts) == {fa_key, mamba_key} + # FA reads block_ids[0][0] = block 1: addr = base(0) + 1 * 512. + assert store.puts[fa_key] == [512] + # Mamba reads the CoW block 7, not block_ids[1][0]=2. + assert store.puts[mamba_key] == [10_000 + mamba_cow_block * 512] + + +def test_offload_syncs_event_before_put(): + """An offload-carrying meta synchronizes its CoW-fence event before the + store put reads the blocks, then completes in one pass and drains the + completion counter.""" + full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + mamba = MambaSpec( + block_size=16, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + groups = [ + KVCacheGroupSpec(["L0"], full), + KVCacheGroupSpec(["L1"], mamba), + ] + coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4) + event = MagicMock() + + class _FencedStore(_DictStore): + def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k): + assert event.synchronize.called, "put must run after the event sync" + return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k) + + store = _FencedStore() + token_dbs = [] + for g_idx in range(2): + db = ChunkedTokenDatabase( + KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx), + block_size=16, + hash_block_size=4, + ) + db.set_kv_caches_base_addr([g_idx * 10_000]) + db.set_block_len([512]) + token_dbs.append(db) + + send = KVCacheStoreSendingThread( + store=store, + coord=coord, + token_databases=token_dbs, + block_size=16, + tp_rank=0, + group_put_steps=[1, 1], + kv_role="kv_both", + ready_event=threading.Event(), + replicate_config=MagicMock(), + ) + + hs = [BlockHash(bytes([i + 1]) * 4) for i in range(3)] + req = ReqMeta( + req_id="r1", + token_len_chunk=0, + block_ids=([1], [2]), + block_hashes=hs, + can_save=True, + num_prompt_tokens=12, + partial_tail_offloads=[(1, 7, 12)], + ) + req.current_event = event + send.add_stored_request("r1") + + send.request_queue.put(req) + send._handle_request(send.request_queue.get()) + assert send.request_queue.qsize() == 0 + assert store._data + assert send.stored_requests["r1"] == 0 + event.synchronize.assert_called_once() + + +def test_sub_block_partial_tail_offload_covers_smaller_group_blocks(): + """The K3-shaped 900/128/1536 scenario scaled to 12/4/16, with a + full-attention group whose block (4) is smaller than the lcm (16): the + offload must persist every FA block up to the boundary — the normal save + floors to the lcm, so those blocks are otherwise never written and the + consumer's per-group lookup would miss. The mamba boundary block still + reads the core-provided CoW target.""" + full = FullAttentionSpec(block_size=4, num_kv_heads=8, head_size=64, dtype=None) + mamba = MambaSpec( + block_size=16, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + groups = [ + KVCacheGroupSpec(["L0"], full), + KVCacheGroupSpec(["L1"], mamba), + ] + coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4) + assert coord.enable_partial_hash_hits + + class _RecordingStore(_DictStore): + def __init__(self): + super().__init__() + self.puts: dict[str, list[int]] = {} + + def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k): + for key, addr in zip(keys, addrs): + self.puts[key] = addr + return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k) + + store = _RecordingStore() + token_dbs = [] + for g_idx, block_size in enumerate([4, 16]): + db = ChunkedTokenDatabase( + KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx), + block_size=block_size, + hash_block_size=4, + ) + db.set_kv_caches_base_addr([g_idx * 10_000]) + db.set_block_len([512]) + token_dbs.append(db) + + send = KVCacheStoreSendingThread( + store=store, + coord=coord, + token_databases=token_dbs, + block_size=16, + tp_rank=0, + group_put_steps=[1, 1], + kv_role="kv_both", + ready_event=threading.Event(), + replicate_config=MagicMock(), + ) + + hs = [BlockHash(bytes([i + 1]) * 4) for i in range(3)] # 3 hash units = 12 tok + mamba_cow_block = 7 + req = ReqMeta( + req_id="r2", + token_len_chunk=0, + block_ids=([1, 2, 3], [4]), + block_hashes=hs, + can_save=True, + num_prompt_tokens=12, + partial_tail_offloads=[(1, mamba_cow_block, 12)], + ) + + send._maybe_offload_partial_tail(req) + + # FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal + # block-end hashes; mamba (block 16): the partial boundary block under + # the boundary sub-hash, read from the CoW target. + expected = { + token_dbs[0].key_for(hs[0]): [1 * 512], + token_dbs[0].key_for(hs[1]): [2 * 512], + token_dbs[0].key_for(hs[2]): [3 * 512], + token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512], + } + assert store.puts == expected diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index e7a0cbc1a7..2709b92e52 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -13,11 +13,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor ) -def _make_bare_scheduler() -> MooncakeStoreScheduler: +def _make_bare_scheduler( + *, hash_block_size: int = 16, enable_partial_hash_hits: bool = False +) -> MooncakeStoreScheduler: scheduler = object.__new__(MooncakeStoreScheduler) scheduler.kv_role = "kv_both" scheduler.lookup_async = False scheduler._block_size = 16 + scheduler._hash_block_size = hash_block_size + scheduler.enable_partial_hash_hits = enable_partial_hash_hits scheduler.load_specs = {} scheduler._unfinished_request_ids = {"req-0"} scheduler._unfinished_requests = {} @@ -136,6 +140,7 @@ def test_preemption_resets_tracker_before_request_finished(): block_hashes=[b"h0", b"h1"], prefill_end_tokens=48, ) + scheduler._request_trackers["req-0"].has_pending_offload = True scheduler.build_connector_meta(_make_preemption_scheduler_output()) @@ -144,6 +149,7 @@ def test_preemption_resets_tracker_before_request_finished(): assert tracker.allocated_block_ids == () assert tracker.num_saved_tokens == 0 assert tracker.token_ids is None + assert tracker.has_pending_offload is False assert tracker.prefill_end_tokens == 0 request = SimpleNamespace(request_id="req-0") assert scheduler.request_finished(request, ([0, 1],)) == (False, None) @@ -536,3 +542,227 @@ def test_full_external_hit_with_full_local_hit_skips_load(): assert need_to_allocate == 0 assert load_async is False assert "req-0" not in scheduler.load_specs + + +def test_partial_hash_hit_block_aligned_local_loads_partial_tail(): + # Fine-grained on (hash=4, block=16): a block-aligned local hit can pull a + # sub-block remote hit (24 = a hash boundary inside block 1). Loads [16, 24). + scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + scheduler.load_async = True + scheduler.client = _StubLookupClient(hit_tokens=24) + + request = SimpleNamespace( + request_id="req-0", + num_tokens=32, + block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5", b"h6", b"h7"], + ) + + need_to_allocate, load_async = scheduler.get_num_new_matched_tokens( + request, num_computed_tokens=16 + ) + + assert need_to_allocate == 8 + assert load_async is True + load_spec = scheduler.load_specs["req-0"] + assert load_spec.vllm_cached_tokens == 16 + assert load_spec.kvpool_cached_tokens == 24 + + +def test_partial_hash_hit_no_remote_gain_skips_load(): + # Core always presents a block-aligned local hit (it floors a sub-block + # tail before calling the connector). When the remote hit does not exceed + # that block-aligned local hit, nothing is loaded. + scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + scheduler.load_async = True + scheduler.client = _StubLookupClient(hit_tokens=16) + + request = SimpleNamespace( + request_id="req-0", + num_tokens=32, + block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5", b"h6", b"h7"], + ) + + need_to_allocate, load_async = scheduler.get_num_new_matched_tokens( + request, num_computed_tokens=16 + ) + + assert need_to_allocate == 0 + assert load_async is False + assert "req-0" not in scheduler.load_specs + + +def test_sub_block_prompt_looks_up_with_fine_grained(): + # A prompt smaller than one block (12 < block 16). With fine-grained partial + # hits the sub-block prefix is worth looking up (floor is the hash unit 4, + # not a full block), so a remote partial hit is loaded. Pre-change the + # block-size floor returned (0, False) for such prompts. + scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + scheduler.load_async = True + scheduler.client = _StubLookupClient(hit_tokens=8) + + request = SimpleNamespace( + request_id="req-0", + num_tokens=12, + block_hashes=[b"h0", b"h1", b"h2"], + ) + + need_to_allocate, load_async = scheduler.get_num_new_matched_tokens( + request, num_computed_tokens=0 + ) + + assert need_to_allocate == 8 + assert load_async is True + assert scheduler.load_specs["req-0"].kvpool_cached_tokens == 8 + + +def test_sub_block_prompt_not_looked_up_without_fine_grained(): + # Without fine-grained partial hits, sub-block prompts still skip the lookup + # (there is no full block, and no sub-block key granularity). + scheduler = _make_bare_scheduler() + scheduler.client = _StubLookupClient(hit_tokens=8) + + request = SimpleNamespace( + request_id="req-0", + num_tokens=12, + block_hashes=[b"h0", b"h1", b"h2"], + ) + + need_to_allocate, load_async = scheduler.get_num_new_matched_tokens( + request, num_computed_tokens=0 + ) + + assert need_to_allocate == 0 + assert load_async is False + assert "req-0" not in scheduler.load_specs + + +def test_pending_partial_tail_emits_offload_only_reqmeta(): + # A sub-block prompt never produces a block-aligned save, so the partial- + # tail offload arriving this step is emitted as an offload-only ReqMeta + # (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so + # the worker skips the normal save). Pending-offload state delays the free + # without advancing the normal-save watermark before the put succeeds. + scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + request = SimpleNamespace( + all_token_ids=list(range(12)), + block_hashes=[b"h0", b"h1", b"h2"], + num_output_placeholders=0, + num_prompt_tokens=12, + ) + scheduler._unfinished_requests["req-0"] = (request, ([0],)) + scheduler._request_trackers["req-0"] = RequestTracker( + req_id="req-0", + token_len=12, + allocated_block_ids=([0],), + num_saved_tokens=0, + token_ids=list(range(12)), + prefill_end_tokens=12, + ) + + out = SimpleNamespace( + finished_req_ids=set(), + preempted_req_ids=set(), + scheduled_new_reqs=[], + scheduled_cached_reqs=SimpleNamespace( + req_ids=[], + new_block_ids=[], + num_computed_tokens=[], + resumed_req_ids=set(), + ), + num_scheduled_tokens={}, + scheduled_spec_decode_tokens={}, + partial_tail_offloads={"req-0": [(1, 7, 12)]}, + ) + + meta = scheduler.build_connector_meta(out) + + assert len(meta.requests) == 1 + req_meta = meta.requests[0] + assert req_meta.req_id == "req-0" + assert req_meta.can_save is True + assert req_meta.token_len_chunk == 0 + assert req_meta.partial_tail_offloads == [(1, 7, 12)] + assert req_meta.num_prompt_tokens == 12 + assert req_meta.block_ids == ([0],) + tracker = scheduler._request_trackers["req-0"] + assert tracker.num_saved_tokens == 0 + assert tracker.has_pending_offload is True + request = SimpleNamespace(request_id="req-0") + assert scheduler.request_finished(request, ([0],)) == (True, None) + + +def test_resumed_partial_tail_uses_handoff_boundary(): + scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + request = SimpleNamespace( + all_token_ids=list(range(20)), + block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4"], + num_output_placeholders=0, + num_prompt_tokens=12, + ) + scheduler._unfinished_requests["req-0"] = (request, ([0, 1],)) + scheduler._request_trackers["req-0"] = RequestTracker( + req_id="req-0", + token_len=20, + allocated_block_ids=([0, 1],), + num_saved_tokens=0, + token_ids=list(range(20)), + # Resumption replays prompt + previously generated tokens. + prefill_end_tokens=20, + ) + + out = SimpleNamespace( + finished_req_ids=set(), + preempted_req_ids=set(), + scheduled_new_reqs=[], + scheduled_cached_reqs=SimpleNamespace( + req_ids=[], + new_block_ids=[], + num_computed_tokens=[], + resumed_req_ids=set(), + ), + num_scheduled_tokens={}, + scheduled_spec_decode_tokens={}, + partial_tail_offloads={"req-0": [(1, 7, 12)]}, + ) + + meta = scheduler.build_connector_meta(out) + + assert len(meta.requests) == 1 + assert meta.requests[0].partial_tail_offloads == [(1, 7, 12)] + # Ordinary metadata retains the full resumed prefill range. + assert meta.requests[0].num_prompt_tokens == 20 + tracker = scheduler._request_trackers["req-0"] + assert tracker.num_saved_tokens == 0 + assert tracker.has_pending_offload is True + + +def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary(): + scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + request = SimpleNamespace( + all_token_ids=list(range(48)), + block_hashes=[b"h0", b"h1", b"h2"], + num_output_placeholders=0, + num_prompt_tokens=36, + ) + scheduler._unfinished_requests["req-0"] = (request, ([0, 1],)) + scheduler._request_trackers["req-0"] = RequestTracker( + req_id="req-0", + token_len=44, + allocated_block_ids=([0, 1],), + num_saved_tokens=32, + token_ids=list(range(44)), + prefill_end_tokens=48, + ) + out = _make_scheduler_output(scheduled_spec_tokens=None) + out.partial_tail_offloads = {"req-0": [(0, 7, 36)]} + + meta = scheduler.build_connector_meta(out) + + assert len(meta.requests) == 1 + assert meta.requests[0].can_save is True + assert meta.requests[0].partial_tail_offloads == [(0, 7, 36)] + assert meta.requests[0].num_prompt_tokens == 48 + # Ordinary saving still covers the full resumed prefill range. + tracker = scheduler._request_trackers["req-0"] + assert tracker.num_saved_tokens == 48 + assert tracker.has_pending_offload is True diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index b9876b9827..54553cd49b 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -599,6 +599,88 @@ def test_store_sending_thread_retries_skipped_range_after_pressure(): assert store.batch_put_from_multi_buffers.call_args.args[0] == keys +def _make_partial_tail_send_thread(store): + coord = SimpleNamespace( + enable_partial_hash_hits=True, + hash_block_size=4, + lcm_block_size=16, + ) + db = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0), + block_size=4, + hash_block_size=4, + ) + db.set_kv_caches_base_addr([0x1000]) + db.set_block_len([256]) + return _make_store_sending_thread( + store, + coord=coord, + token_databases=[db], + ) + + +def _make_partial_tail_req(block_ids: list[int]) -> ReqMeta: + return ReqMeta( + req_id="req-a", + token_len_chunk=0, + block_ids=(block_ids,), + block_hashes=[b"a0", b"a1", b"a2"], + can_save=True, + partial_tail_offloads=[(1, 7, 12)], + ) + + +def test_partial_tail_offload_skips_null_source_blocks(): + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.return_value = [256, 256] + thread = _make_partial_tail_send_thread(store) + + assert thread._maybe_offload_partial_tail(_make_partial_tail_req([0, 2, 3])) + + keys, addrs, _sizes, _replicate_config = ( + store.batch_put_from_multi_buffers.call_args.args + ) + assert keys == [ + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132", + ] + assert addrs == [[0x1000 + 2 * 256], [0x1000 + 3 * 256]] + + +def test_partial_tail_offload_honors_active_pressure_gate(): + store = MagicMock() + thread = _make_partial_tail_send_thread(store) + thread._store_pressure_active = True + thread._skip_store_requests.add("req-a") + thread.add_stored_request("req-a") + + thread._handle_request(_make_partial_tail_req([1, 2, 3])) + + store.batch_is_exist.assert_not_called() + store.batch_put_from_multi_buffers.assert_not_called() + assert thread.stored_requests["req-a"] == 0 + + +def test_partial_tail_put_failure_activates_pressure_gate(): + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.return_value = [256, -200, 256] + thread = _make_partial_tail_send_thread(store) + thread.add_stored_request("req-a") + + thread._handle_request(_make_partial_tail_req([1, 2, 3])) + + assert thread._store_pressure_active is True + assert thread._skip_store_requests == {"req-a"} + assert thread._saved_offset.get("req-a", 0) == 0 + assert thread.stored_requests["req-a"] == 0 + + thread.add_stored_request("req-a") + thread._handle_request(_make_partial_tail_req([1, 2, 3])) + assert store.batch_put_from_multi_buffers.call_count == 1 + + def test_store_sending_thread_delta_start_rank_saves_second_local_chunk(): store = MagicMock() store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) @@ -1939,6 +2021,46 @@ def test_lookup_partial_prefix_returns_first_hit_length(): assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32 +def test_lookup_partial_tail_uses_hash_alignment(): + """A stored sub-block tail can serve a request extending past it.""" + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + MambaSpec, + ) + + worker = _make_bare_worker(block_size=16) + full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + mamba = MambaSpec( + block_size=16, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + worker._kv_cache_groups = [ + KVCacheGroupSpec(["full"], full), + KVCacheGroupSpec(["mamba"], mamba), + ] + worker.hash_block_size = 4 + worker.token_dbs = [ + ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=group_id), + block_size=16, + hash_block_size=4, + ) + for group_id in range(2) + ] + worker.coord = mooncake_store_worker.MooncakeStoreCoordinator( + worker._kv_cache_groups, + scheduler_block_size=16, + hash_block_size=4, + ) + _refresh_group_tp_replication_factors(worker) + worker.store.batch_is_exist.return_value = [0, 0, 1, 0, 0, 1] + + assert worker.lookup(13, [b"h0", b"h1", b"h2"]) == 12 + + def test_lookup_full_hit_reuses_existing_boundary(): """A full hit is re-derived below the request end without another RPC.""" worker = _make_bare_worker(block_size=16) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py index db34079ef8..c70ddf95a2 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py @@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheGroupSpec, KVCacheSpec, + MambaSpec, UniformTypeKVCacheSpecs, ) from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry @@ -83,6 +84,9 @@ class MooncakeStoreCoordinator: self.kv_cache_groups = kv_cache_groups self.hash_block_size = hash_block_size self.lcm_block_size = scheduler_block_size + self.enable_partial_hash_hits = partial_hash_hits_enabled( + kv_cache_groups, hash_block_size + ) self.use_eagle = use_eagle # Mirror vLLM core's KVCacheCoordinator.retention_interval. self.retention_interval = retention_interval @@ -94,7 +98,12 @@ class MooncakeStoreCoordinator: self._verify_and_split_kv_cache_groups() def align_lookup_length(self, length: int) -> int: - return length // self.lcm_block_size * self.lcm_block_size + alignment = ( + self.hash_block_size + if self.enable_partial_hash_hits + else self.lcm_block_size + ) + return length // alignment * alignment def _verify_and_split_kv_cache_groups(self) -> None: """Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but @@ -226,9 +235,14 @@ class MooncakeStoreCoordinator: retention_interval: int | None, num_prompt_tokens: int | None, ) -> tuple[list[bool] | None, ...]: - assert aligned_token_len % self.lcm_block_size == 0, ( + mask_alignment = ( + self.hash_block_size + if self.enable_partial_hash_hits + else self.lcm_block_size + ) + assert aligned_token_len % mask_alignment == 0, ( f"aligned_token_len ({aligned_token_len}) must be a multiple of " - f"lcm_block_size ({self.lcm_block_size})" + f"{mask_alignment}" ) masks: list[list[bool] | None] = [] for g_idx, g in enumerate(self.kv_cache_groups): @@ -278,17 +292,21 @@ class MooncakeStoreCoordinator: one already removed by the lookup. """ eagle_indices = self.eagle_attn_group_indices if apply_eagle else set() + alignment_tokens = ( + self.hash_block_size + if self.enable_partial_hash_hits + else self.lcm_block_size + ) if len(self.attention_groups) == 1: spec, group_ids, manager_cls = self.attention_groups[0] - hashes = self.block_hashes_for_spec(block_hashes, spec) hit_blocks, hit_length = manager_cls.find_longest_cache_hit( - block_hashes=hashes, # type: ignore[arg-type] + block_hashes=block_hashes, # type: ignore[arg-type] max_length=max_length, kv_cache_group_ids=group_ids, block_pool=cast(BlockPool, cached_block_pool), kv_cache_spec=spec, drop_eagle_block=(0 in eagle_indices), - alignment_tokens=spec.block_size, + alignment_tokens=alignment_tokens, ) num_groups = len(self.kv_cache_groups) blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)] @@ -320,17 +338,23 @@ class MooncakeStoreCoordinator: drop_eagle_block = idx in eagle_indices and idx not in eagle_verified _max_length = curr_hit_length - if drop_eagle_block: - _max_length = min(curr_hit_length + spec.block_size, max_length) - hashes = self.block_hashes_for_spec(block_hashes, spec) + if drop_eagle_block and not isinstance(spec, MambaSpec): + eagle_margin = ( + self.hash_block_size + if self.enable_partial_hash_hits + and manager_cls.supports_fine_grained_hash_lookup + and spec.block_size > self.hash_block_size + else spec.block_size + ) + _max_length = min(curr_hit_length + eagle_margin, max_length) hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit( - block_hashes=hashes, # type: ignore[arg-type] + block_hashes=block_hashes, # type: ignore[arg-type] max_length=_max_length, kv_cache_group_ids=group_ids, block_pool=cast(BlockPool, cached_block_pool), kv_cache_spec=spec, drop_eagle_block=drop_eagle_block, - alignment_tokens=self.lcm_block_size, + alignment_tokens=alignment_tokens, ) if drop_eagle_block: eagle_verified.add(idx) @@ -348,10 +372,11 @@ class MooncakeStoreCoordinator: break # Truncate full-attention hit_blocks to final converged length; - # other specs already trim themselves inside their hit logic. + # other specs already trim themselves inside their hit logic. cdiv keeps + # the partial tail block when hit_length is not block-aligned. spec0, group_ids0, _ = self.attention_groups[0] if isinstance(spec0, FullAttentionSpec): - num_blocks = hit_length // spec0.block_size + num_blocks = cdiv(hit_length, spec0.block_size) for gid in group_ids0: full_blks = hit_blocks_by_group[gid] assert full_blks is not None @@ -368,3 +393,18 @@ def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec: if isinstance(spec, UniformTypeKVCacheSpecs): return next(iter(spec.kv_cache_specs.values())) return spec + + +def partial_hash_hits_enabled( + kv_cache_groups: list[KVCacheGroupSpec], hash_block_size: int +) -> bool: + """Mirror of core's ``HybridKVCacheCoordinator.enable_partial_hash_hits`` + (its dcp == 1 clause holds: the connector rejects hybrid + DCP/PCP > 1). + Single copy on purpose — scheduler and coordinator must not disagree. + """ + return any( + isinstance(spec := _unwrap_spec(g.kv_cache_spec), MambaSpec) + and spec.mamba_cache_mode == "align" + and spec.block_size > hash_block_size + for g in kv_cache_groups + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py index 57e6bd8de0..6daa1e82ea 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py @@ -228,16 +228,28 @@ class ChunkedTokenDatabase: n = len(chunks) starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n) spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts - assert not (spans % self.block_size).any() + assert not (spans % self.hash_block_size).any() bids = np.fromiter( (block_ids[i] for i in (starts // self.block_size).tolist()), dtype=np.int64, count=n, ) addrs = base[None, :] + bids[:, None] * blen[None, :] - sizes = blen[None, :] * (spans // self.block_size)[:, None] + block_counts = (spans + self.block_size - 1) // self.block_size + sizes = blen[None, :] * block_counts[:, None] return addrs.tolist(), sizes.tolist(), bids.tolist() + def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]: + """Return addresses and sizes for one physical block slot.""" + addr_list = [] + size_list = [] + length = len(self.block_len) + for index, base_addr in enumerate(self.kv_caches_base_addr): + addr = base_addr + block_id * self.block_len[index % length] + addr_list.append(addr) + size_list.append(self.block_len[index % length]) + return addr_list, size_list + def process_tokens( self, token_len: int, @@ -256,7 +268,8 @@ class ChunkedTokenDatabase: rank regardless of where the processed suffix begins. Args: - token_len: Total number of tokens. + token_len: Total number of tokens. Must be hash-block aligned and + covered by ``block_hashes`` when hashes are present. block_hashes: Block hashes computed at ``hash_block_size`` granularity. When ``block_size > hash_block_size`` each group's ``block_size`` chunk is keyed by its last sub-hash via ``chunk_hashes_for_block_size``. @@ -269,11 +282,10 @@ class ChunkedTokenDatabase: assert put_step > 0 if not block_hashes: return - chunk_hashes: Sequence[BlockHash] = chunk_hashes_for_block_size( - block_hashes, self.hash_block_size, self.block_size - ) + assert token_len % self.hash_block_size == 0 + assert token_len // self.hash_block_size <= len(block_hashes) start_chunk = max(0, cdiv(mask_num, self.block_size)) - max_chunks = min(len(chunk_hashes), cdiv(token_len, self.block_size)) + max_chunks = cdiv(token_len, self.block_size) if chunk_mask is not None: max_chunks = min(max_chunks, start_chunk + len(chunk_mask)) for chunk_id in range(start_chunk, max_chunks): @@ -281,9 +293,9 @@ class ChunkedTokenDatabase: continue if chunk_id % put_step != put_step_rank: continue - h = chunk_hashes[chunk_id] start_idx = chunk_id * self.block_size end_idx = min(start_idx + self.block_size, token_len) + h = block_hashes[end_idx // self.hash_block_size - 1] yield start_idx, end_idx, h @@ -306,6 +318,7 @@ class RequestTracker: allocated_block_ids: tuple[list[int], ...] num_saved_tokens: int = 0 token_ids: list[int] | None = None + has_pending_offload: bool = False # Snapshot of the prefill range length at tracker creation time. # For a fresh request this is len(prompt). For a resumed-from-preemption # request it includes previously-generated tokens, which are re-prefilled. @@ -316,6 +329,7 @@ class RequestTracker: self.allocated_block_ids = () self.num_saved_tokens = 0 self.token_ids = None + self.has_pending_offload = False self.prefill_end_tokens = 0 def update( @@ -352,6 +366,11 @@ class ReqMeta: token_ids: list[int] | None = None num_prompt_tokens: int | None = None + # Core-provided per-mamba-group + # (group_id, cow_block_id, boundary_tokens) for this request's partial tail. + # Present only on the producer's CoW step; drives the connector's offload + # (the FA group's block is derived from block_ids and boundary_tokens). + partial_tail_offloads: list[tuple[int, int, int]] | None = None @staticmethod def from_request_tracker( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index 2fef49a307..a14d02927f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -11,6 +11,9 @@ from vllm.config import VllmConfig from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorMetadata, ) +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501 + partial_hash_hits_enabled, +) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501 LoadSpec, MooncakeStoreConnectorMetadata, @@ -63,6 +66,9 @@ class MooncakeStoreScheduler: self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes( kv_cache_config, vllm_config ) + self.enable_partial_hash_hits = partial_hash_hits_enabled( + kv_cache_config.kv_cache_groups, self._hash_block_size + ) # Per-request state self.load_specs: dict[str, LoadSpec] = {} # to be loaded @@ -80,7 +86,12 @@ class MooncakeStoreScheduler: Returns ``(None, False)`` when an async lookup is still in flight, signaling the scheduler to retry this request on a later step. """ - if request.num_tokens < self._block_size: + # Fine-grained hits may land on a hash boundary inside a block; without + # partial hits, prefixes shorter than one physical block are skipped. + align = ( + self._hash_block_size if self.enable_partial_hash_hits else self._block_size + ) + if request.num_tokens < align: return 0, False num_external_hit_tokens = self.client.lookup( @@ -338,6 +349,44 @@ class MooncakeStoreScheduler: if req_meta is not None: meta.add_request(req_meta) + # Flush partial-tail offloads in the step they arrive: the CoW copy is + # enqueued before the connector event records, so this step's event + # fences the cow block. Ride the request's save meta when present, else + # emit an offload-only ReqMeta (token_len_chunk=0 skips the normal + # save; can_save=True takes the normal enqueue path). + step_partial_tails = getattr(scheduler_output, "partial_tail_offloads", None) + if step_partial_tails and not force_skip_save: + pending = dict(step_partial_tails) + for req_meta in meta.requests: + if req_meta.can_save: + groups = pending.pop(req_meta.req_id, None) + if groups: + req_meta.partial_tail_offloads = groups + tracker = self._request_trackers.get(req_meta.req_id) + if tracker is not None: + tracker.has_pending_offload = True + for req_id, groups in pending.items(): + tracker = self._request_trackers.get(req_id) + req_tuple = self._unfinished_requests.get(req_id) + if tracker is None or req_tuple is None: + # Request finished/preempted within this step; its blocks + # are going away, so the offload is conservatively dropped. + logger.debug("Dropping partial-tail offload for request %s", req_id) + continue + assert len({boundary for _, _, boundary in groups}) == 1 + tracker.has_pending_offload = True + meta.add_request( + ReqMeta( + req_id=req_id, + token_len_chunk=0, + block_ids=tracker.allocated_block_ids, + block_hashes=req_tuple[0].block_hashes, + can_save=True, + num_prompt_tokens=tracker.prefill_end_tokens, + partial_tail_offloads=groups, + ) + ) + return meta def request_finished( @@ -352,7 +401,9 @@ class MooncakeStoreScheduler: # Missing tracker can happen when the request is aborted before the # connector observes the normal finished lifecycle or is preempted # before finishing. - if tracker is None or tracker.num_saved_tokens <= 0: + if tracker is None or ( + tracker.num_saved_tokens <= 0 and not tracker.has_pending_offload + ): return False, None total_blocks = sum(len(g) for g in block_ids) delay_free_blocks = total_blocks > 0 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index cc49a3569d..0fd85eb57d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -61,6 +61,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import from vllm.logger import init_logger from vllm.utils.math_utils import cdiv from vllm.utils.network_utils import get_ip, make_zmq_socket +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID from vllm.v1.core.kv_cache_utils import ( BlockHash, maybe_convert_block_hash, @@ -531,6 +532,164 @@ class KVCacheStoreSendingThread(KVTransferThread): self._skip_store_requests.clear() return True + def _maybe_offload_partial_tail(self, req_meta: ReqMeta) -> bool: + """Offload the request's sub-block partial tail (its last prompt hash + boundary) so a later request can hit the sub-block prefix. + + Covers every block from the normal save's lcm floor to the boundary: + the normal save floors to ``lcm_block_size``, so a smaller-block + group's full blocks in that gap are never persisted elsewhere, and + the consumer's lookup needs every group at every probed boundary. + Full blocks are keyed by their block-end hash, the partial boundary + block by the boundary sub-hash; the mamba "align" boundary block is + the core-provided CoW block. All keys are deduped against the store. + + Returns: + True when no put is needed or every put succeeds, False otherwise. + """ + if not self.coord.enable_partial_hash_hits or not req_meta.block_hashes: + return True + partial_tail_offloads = req_meta.partial_tail_offloads + if not partial_tail_offloads: + return True + hash_block_size = self.coord.hash_block_size + boundaries = {boundary for _, _, boundary in partial_tail_offloads} + if len(boundaries) != 1: + raise ValueError( + "Partial-tail offloads for one request must share a boundary" + ) + boundary = boundaries.pop() + if boundary == 0: + return True + if boundary // hash_block_size - 1 >= len(req_meta.block_hashes): + return True + mamba_offloads = { + group_id: block_id for group_id, block_id, _ in partial_tail_offloads + } + + keys: list[str] = [] + addrs: list[list[int]] = [] + sizes: list[list[int]] = [] + saved = self._saved_offset.get(req_meta.req_id, 0) + for g_idx, db in enumerate(self.token_databases): + group_blocks = req_meta.block_ids[g_idx] + # Distribute across ranks by the same rule as normal chunks. + put_step = self.group_put_steps[g_idx] + put_step_rank = (self.tp_rank + g_idx) % put_step + # Always include the boundary block: its sub-hash key is written + # only here, even if normal saves already advanced past it. + last_block = cdiv(boundary, db.block_size) - 1 + for block_idx in range( + min(saved // db.block_size, last_block), last_block + 1 + ): + if block_idx % put_step != put_step_rank: + continue + valid_end = min((block_idx + 1) * db.block_size, boundary) + key_hash = req_meta.block_hashes[valid_end // hash_block_size - 1] + if ( + g_idx in mamba_offloads + and valid_end == boundary + and boundary % db.block_size != 0 + ): + block_id = mamba_offloads[g_idx] + else: + if block_idx >= len(group_blocks): + continue + block_id = group_blocks[block_idx] + if block_id == NULL_BLOCK_ID: + logger.debug( + "Skipping unavailable partial-tail source block " + "(req=%s, group=%d, block=%d)", + req_meta.req_id, + g_idx, + block_idx, + ) + continue + addr, size = db.prepare_value_for_block(block_id) + keys.append(db.key_for(key_hash)) + addrs.append(addr) + sizes.append(size) + + if not keys: + return True + exists_start = time.perf_counter() + try: + exists = self.store.batch_is_exist(keys) + except Exception as e: + self._record_operation( + "save_exists", + exists_start, + len(keys), + status="error", + num_failed_keys=len(keys), + ) + logger.error( + "Failed to check partial-tail keys for request %s: %s", + req_meta.req_id, + e, + ) + return False + self._record_operation("save_exists", exists_start, len(keys)) + missing = [i for i, e in enumerate(exists) if e != 1] + if not missing: + return True + keys = [keys[i] for i in missing] + addrs = [addrs[i] for i in missing] + sizes = [sizes[i] for i in missing] + if req_meta.current_event is not None: + # Fence the CoW block copy enqueued earlier this step. + req_meta.current_event.synchronize() + batch_bytes = _sum_batch_bytes(sizes) + put_start = time.perf_counter() + try: + res = self.store.batch_put_from_multi_buffers( + keys, addrs, sizes, self.replicate_config + ) + except Exception as e: + self._record_operation( + "save_put", + put_start, + len(keys), + num_bytes=batch_bytes, + status="error", + num_failed_keys=len(keys), + ) + logger.error( + "Failed to put partial-tail keys for request %s: %s", + req_meta.req_id, + e, + ) + return False + + failed = [i for i, value in enumerate(res) if value < 0] + self._record_operation( + "save_put", + put_start, + len(keys), + num_bytes=batch_bytes, + status="partial_failure" if failed else "ok", + num_failed_keys=len(failed), + ) + if failed: + failed_codes = {res[i] for i in failed} + logger.warning( + "Partial-tail put failed for request %s: %d/%d keys failed (codes=%s)", + req_meta.req_id, + len(failed), + len(keys), + failed_codes, + ) + if MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes: + self._mark_request_skipped_for_pressure(req_meta.req_id) + return False + + if self._clear_store_pressure(): + logger.info( + "Mooncake CPU/disk offloading pressure cleared after a " + "successful partial-tail batch" + ) + return True + def _handle_request(self, req_meta: ReqMeta): # Cache hits are always a multiple of ``lcm_block_size`` tokens, which # is also ``store_mask``'s precondition. @@ -548,9 +707,6 @@ class KVCacheStoreSendingThread(KVTransferThread): # so the scheduler can release the GPU blocks it pinned for this # request (via `delay_free_blocks`) even when the store path raises. try: - if token_len == 0: - return - if self._should_skip_request(req_id): logger.debug( "Skipping Mooncake store for request %s while CPU/disk " @@ -559,6 +715,16 @@ class KVCacheStoreSendingThread(KVTransferThread): ) return + # Offload the sub-block partial tail (independent of the normal + # block-aligned save, which may be skipped this step). + if req_meta.partial_tail_offloads is not None and not ( + self._maybe_offload_partial_tail(req_meta) + ): + return + + if token_len == 0: + return + # Resume from where this rank left off; only the new suffix is saved. save_start = self._saved_offset.get(req_id, 0) @@ -1385,7 +1551,7 @@ class MooncakeStoreWorker: self.recv_request_queue.put(request) assert self.load_async, "load_async must be True for better performance." - # Issue stores with CUDA event synchronization + # Issue stores with CUDA event synchronization. if self.kv_role in ["kv_producer", "kv_both"]: current_event = None for request in meta.requests: @@ -1501,21 +1667,32 @@ class MooncakeStoreWorker: # candidate_meta stores the (group, hash_bytes) for key slice. candidate_keys: list[str] = [] candidate_meta: list[tuple[int, bytes]] = [] - lookup_masks = self.coord.lookup_mask(token_len) + fine_grained = self.coord.enable_partial_hash_hits + lookup_masks = None if fine_grained else self.coord.lookup_mask(token_len) for g_idx, db in enumerate(self.token_dbs): spec_block_size = db.block_size - lookup_mask = lookup_masks[g_idx] key_prefixes = self._lookup_key_prefixes[g_idx] - group_hashes = self.coord.block_hashes_for_spec( - block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec - ) - max_chunks = min(len(group_hashes), cdiv(token_len, spec_block_size)) - mask_limit = ( - max_chunks if lookup_mask is None else min(max_chunks, len(lookup_mask)) - ) - for chunk_id in range(mask_limit): - if lookup_mask is not None and not lookup_mask[chunk_id]: - continue + if fine_grained: + max_units = min(len(block_hashes), token_len // self.hash_block_size) + unit_ids: range | list[int] = range(max_units) + group_hashes: Sequence[BlockHash] = block_hashes + else: + lookup_mask = lookup_masks[g_idx] # type: ignore[index] + group_hashes = self.coord.block_hashes_for_spec( + block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec + ) + max_chunks = min(len(group_hashes), cdiv(token_len, spec_block_size)) + mask_limit = ( + max_chunks + if lookup_mask is None + else min(max_chunks, len(lookup_mask)) + ) + unit_ids = [ + chunk_id + for chunk_id in range(mask_limit) + if lookup_mask is None or lookup_mask[chunk_id] + ] + for chunk_id in unit_ids: h = group_hashes[chunk_id] hash_hex = h.hex() for key_prefix in key_prefixes: diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index e93a1ce640..fcb67726a5 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -186,6 +186,10 @@ class KVCacheManager: tuple(() for _ in range(self.num_kv_cache_groups)) ) + # Off-table cow blocks handed to a KV connector for partial-tail + # offload; pinned until the request's blocks are freed. + self._partial_tail_pins: dict[str, list[KVCacheBlock]] = {} + @property def usage(self) -> float: """Get the KV cache usage. @@ -568,6 +572,9 @@ class KVCacheManager: Args: request: The request to free the blocks. """ + pins = self._partial_tail_pins.pop(request.request_id, None) + if pins: + self.block_pool.free_blocks(pins) self.coordinator.free(request.request_id) def remove_skipped_blocks( @@ -600,7 +607,14 @@ class KVCacheManager: Returns: The request's blocks in allocation order. """ - return self.coordinator.pop_blocks_for_free(request.request_id) + blocks = self.coordinator.pop_blocks_for_free(request.request_id) + # Pins ride the same (possibly deferred) free as the request blocks. + # Preemption may release a pin under a still-queued offload — the same + # exposure normal saves of table blocks already have. + pins = self._partial_tail_pins.pop(request.request_id, None) + if pins: + blocks = pins + blocks + return blocks def evict_blocks(self, block_ids: set[int]) -> None: """evict blocks from the prefix cache by their block IDs. @@ -760,6 +774,25 @@ class KVCacheManager: # Only create new KVCacheBlocks for non-empty blocks return KVCacheBlocks(blocks) if any(blocks) else self.empty_kv_cache_blocks + def truncate_computed_blocks( + self, blocks: KVCacheBlocks, num_computed_tokens: int + ) -> KVCacheBlocks: + """Return a lookup-result view truncated at an aligned token endpoint. + + Pure slicing: refcounts are untouched and ``blocks`` is not mutated. + """ + truncated: list[list[KVCacheBlock]] = [] + for group_blocks, manager in zip( + blocks.blocks, + self.coordinator.single_type_managers, + strict=True, + ): + assert num_computed_tokens % manager.block_size == 0 + num_blocks = num_computed_tokens // manager.block_size + assert num_blocks <= len(group_blocks) + truncated.append(list(group_blocks[:num_blocks])) + return self.create_kv_cache_blocks(tuple(truncated)) + def take_new_block_ids(self) -> list[int]: """Drain and return new attention block IDs for zeroing.""" ids: list[int] = [] @@ -812,6 +845,34 @@ class KVCacheManager: retained_blocks = [block for pair in pending_copies for block in pair] return copies, retained_blocks + def take_partial_tail_offloads(self) -> dict[str, list[tuple[int, int, int]]]: + """Drain producer partial-tail offload hand-offs per request. + + Returns ``{request_id: [(group_id, block_id, boundary_tokens), ...]}`` + for the durable boundary blocks of producers' last-prompt-boundary + partial tails. Only mamba "align" groups contribute; empty otherwise. + A KV connector reads the referenced blocks and offloads them so a later + request can hit the sub-block prefix. + + Each handed-off block lives off the request block table, so it is + pinned here and unpinned when the request's blocks are freed — for a + producer with saved tokens, after the connector reports sends done. + """ + offloads: dict[str, list[tuple[int, int, int]]] = {} + for mgr in self.coordinator.single_type_managers: + for ( + req_id, + group_id, + block, + boundary_tokens, + ) in mgr.take_pending_partial_tail_offloads(): + self.block_pool.touch((block,)) + self._partial_tail_pins.setdefault(req_id, []).append(block) + offloads.setdefault(req_id, []).append( + (group_id, block.block_id, boundary_tokens) + ) + return offloads + def new_step_starts(self) -> None: """Notify the coordinator that a new step is starting.""" self.coordinator.new_step_starts() diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index dab86c015e..8782061356 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -258,6 +258,12 @@ class SchedulerOutput: # CoW copies to apply after zeroing new blocks and before forward. kv_cache_block_copies: list[KVCacheBlockCopy] | None = None + # Producer partial-tail offload hand-off for external KV connectors: + # {request_id: [(group_id, block_id, boundary_tokens), ...]} pointing at + # the durable boundary block of a producer's last-prompt-boundary partial + # tail (mamba "align" CoW target). None unless partial hash hits are active. + partial_tail_offloads: dict[str, list[tuple[int, int, int]]] | None = None + # Dynamic speculative decoding: optimal K chosen by scheduler. # Number of spec tokens to schedule for the next step. num_spec_tokens_to_schedule: int = 0 diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 7536f6b060..fff8c15e22 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -740,9 +740,16 @@ class Scheduler(SchedulerInterface): # Get externally-cached tokens if using a KVConnector. if self.connector is not None: + # Present a block-aligned local hit to the connector so + # a strictly longer remote hit can supersede a local + # sub-block tail without racing its copy-on-write. + partial_tail = num_new_local_computed_tokens % self.block_size + block_aligned_local = ( + num_new_local_computed_tokens - partial_tail + ) ext_tokens, load_kv_async = ( self.connector.get_num_new_matched_tokens( - request, num_new_local_computed_tokens + request, block_aligned_local ) ) @@ -754,7 +761,27 @@ class Scheduler(SchedulerInterface): step_skipped_waiting.prepend_request(request) continue - num_external_computed_tokens = ext_tokens + if partial_tail and ext_tokens > partial_tail: + # Remote strictly exceeds the full local hit: drop the + # sub-block tail so no CoW is needed, and let the load + # cover it. Trim the partial block out of the local + # computed blocks so it is not adopted from the cache. + new_computed_blocks = ( + self.kv_cache_manager.truncate_computed_blocks( + new_computed_blocks, block_aligned_local + ) + ) + num_new_local_computed_tokens = block_aligned_local + num_external_computed_tokens = ext_tokens + elif partial_tail: + # Remote does not exceed the full local hit: keep the + # local sub-block tail and load nothing external. + num_external_computed_tokens = 0 + # Nothing to load remotely -> not an async-load step; + # clearing avoids the `load_kv_async` assert below. + load_kv_async = False + else: + num_external_computed_tokens = ext_tokens if hit_diverged and num_external_computed_tokens == 0: # No external tokens back the deeper local hit, so its @@ -1108,6 +1135,22 @@ class Scheduler(SchedulerInterface): self.prev_step_scheduled_req_ids.clear() self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys()) + # Producer partial-tail hand-off for external KV connectors. Drained + # before the CoW retentions are released below, so the pin lands while + # the cow block still holds a retention ref. Without a producer-side + # connector nothing consumes the hand-off, so skip the drain (and its + # pin); the manager drops stale entries when the request's blocks are + # popped for free. + pending_partial_tail_offloads = None + if ( + self.connector is not None + and self.vllm_config.kv_transfer_config is not None + and self.vllm_config.kv_transfer_config.is_kv_producer + ): + pending_partial_tail_offloads = ( + self.kv_cache_manager.take_partial_tail_offloads() or None + ) + kv_cache_block_copies, cow_retained_blocks = ( self.kv_cache_manager.take_kv_cache_block_copies() ) @@ -1153,6 +1196,7 @@ class Scheduler(SchedulerInterface): free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(), new_block_ids_to_zero=self._get_new_block_ids_to_zero(), kv_cache_block_copies=pending_kv_cache_block_copies, + partial_tail_offloads=pending_partial_tail_offloads, num_spec_tokens_to_schedule=num_spec_tokens_to_schedule, ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(), ) diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index f8578c68a3..75cf97f4f4 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -115,6 +115,15 @@ class SingleTypeKVCacheManager(ABC): # managers (full attention, mamba "align"); harmlessly empty elsewhere. self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {} self._pending_cow_copies: list[tuple[KVCacheBlock, KVCacheBlock]] = [] + # Partial-tail offload hand-off for external KV connectors: when a + # producer registers its last-prompt-boundary partial tail and the + # durable boundary block is not on the append-only request block table + # (mamba "align" CoW target), record the request, group, block, and + # exact token boundary so a connector can offload it under the right + # hash. Populated only by mamba "align". + self._pending_partial_tail_offloads: list[ + tuple[str, int, KVCacheBlock, int] + ] = [] @classmethod def _get_num_evictable_blocks(cls, blocks: Sequence[KVCacheBlock]): @@ -378,6 +387,21 @@ class SingleTypeKVCacheManager(ABC): self._pending_cow_copies = [] return pending_copies + def take_pending_partial_tail_offloads( + self, + ) -> list[tuple[str, int, KVCacheBlock, int]]: + """Drain producer partial-tail hand-offs. + + Entries are ``(req_id, group_id, block, boundary_tokens)``. + + Only mamba "align" populates this. The block lives off the request + block table, so the caller must pin it until the connector has read + it — nothing else keeps it alive once the CoW retention is released. + """ + pending = self._pending_partial_tail_offloads + self._pending_partial_tail_offloads = [] + return pending + def _apply_cow( self, request_id: str, @@ -693,7 +717,8 @@ class FullAttentionManager(SingleTypeKVCacheManager): alignment_tokens < block_size and block_size % alignment_tokens == 0 ) if fine_grained: - assert isinstance(block_hashes, list) + # list or lazy BlobBlockHashes view + assert isinstance(block_hashes, Sequence) full_block_hashes: BlockHashList = BlockHashListWithBlockSize( block_hashes, alignment_tokens, block_size ) @@ -716,7 +741,8 @@ class FullAttentionManager(SingleTypeKVCacheManager): # Phase 2 (fine-grained only): extend into the first non-full block by # probing its interior hash boundaries high-to-low (longest hit first). if fine_grained: - assert isinstance(block_hashes, list) + # list or lazy BlobBlockHashes view + assert isinstance(block_hashes, Sequence) scale_factor = block_size // alignment_tokens first_partial_idx = len(computed_blocks[0]) * scale_factor max_partial_idx = min( @@ -1244,6 +1270,11 @@ class MambaManager(SingleTypeKVCacheManager): self.last_state_block_idx: dict[str, int] = {} # The set of the requests that have been allocated blocks self._allocated_block_reqs: set[str] = set() + # Requests that registered their own last-prompt-boundary partial + # tail (producers). On the next step's CoW the boundary state moves + # into a private cow_block; we record that block for connector + # offload (see _pending_partial_tail_offloads). + self._producer_partial_tail_reqs: dict[str, int] = {} @classmethod def find_longest_cache_hit( @@ -1277,7 +1308,8 @@ class MambaManager(SingleTypeKVCacheManager): block_size = kv_cache_spec.block_size if alignment_tokens < block_size and block_size % alignment_tokens == 0: - assert isinstance(block_hashes, list) + # list or lazy BlobBlockHashes view + assert isinstance(block_hashes, Sequence) hash_block_size = alignment_tokens scale_factor = block_size // hash_block_size max_num_partial_units = min( @@ -1590,6 +1622,21 @@ class MambaManager(SingleTypeKVCacheManager): self.block_pool.move_block_hashes(source_block, cow_block) self._pending_cow_copies.append((source_block, cow_block)) source_block.ref_cnt += 1 + boundary_tokens = self._producer_partial_tail_reqs.pop( + request_id, None + ) + if boundary_tokens is not None: + # This CoW preserved a producer's own boundary + # state in cow_block; hand it to the connector for + # partial-tail offload once the copy has run. + self._pending_partial_tail_offloads.append( + ( + request_id, + self.kv_cache_group_id, + cow_block, + boundary_tokens, + ) + ) if cow_block.block_hash is not None: # The moved entry is only filled by this step's # copy, so defer same-step hits on it. @@ -1607,6 +1654,14 @@ class MambaManager(SingleTypeKVCacheManager): if self.mamba_cache_mode == "align": self._allocated_block_reqs.discard(request_id) self.last_state_block_idx.pop(request_id, None) + self._producer_partial_tail_reqs.pop(request_id, None) + # A hand-off whose request died in this same scheduling pass must + # not reach the connector: its unpin hook (free) has already run. + self._pending_partial_tail_offloads = [ + entry + for entry in self._pending_partial_tail_offloads + if entry[0] != request_id + ] return super().pop_blocks_for_free(request_id) def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: @@ -1681,6 +1736,11 @@ class MambaManager(SingleTypeKVCacheManager): if partial_hash is not None: self._partial_hit_reqs[request.request_id] = (block_idx, source_block) self.num_cached_block[request.request_id] = block_idx + # Producer of this partial tail: the boundary state currently lives + # in ``source_block`` but the next step's forward overwrites it. The + # upcoming CoW copies it into a durable cow_block; record the req so + # allocate_new_blocks hands that block to the connector for offload. + self._producer_partial_tail_reqs[request.request_id] = num_tokens return partial_hash