[3/N][Core][KV Connector] Support reliable partial-tail KV offload for sub-block prompts (#49502)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
544cb724c8
commit
d742856610
@@ -223,6 +223,299 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
|
||||
assert moved[0].block_hash_num_tokens == 6
|
||||
|
||||
|
||||
def test_take_partial_tail_offloads_returns_cow_target():
|
||||
"""The connector offload hand-off exposes the mamba CoW *target* block Y
|
||||
(the durable boundary state), not the overwritten source X, and only at
|
||||
the CoW step."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
# Step A registered the partial tail but has not CoW'd yet: no offload.
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
|
||||
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
source_block = manager.block_pool.get_cached_block(
|
||||
partial_mamba_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert source_block is not None
|
||||
source_block_id = source_block[0].block_id
|
||||
|
||||
# Step B: the producer continues, triggering the CoW X->Y.
|
||||
req0.num_computed_tokens = 6
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
|
||||
offloads = manager.take_partial_tail_offloads()
|
||||
assert list(offloads.keys()) == ["0"]
|
||||
assert len(offloads["0"]) == 1
|
||||
group_id, block_id, boundary_tokens = offloads["0"][0]
|
||||
assert group_id == 1 # the mamba group
|
||||
assert boundary_tokens == 6
|
||||
copies, _ = manager.take_kv_cache_block_copies()
|
||||
cow_copy = next(c for c in copies if c.src_block_id == source_block_id)
|
||||
# The offload points at the durable CoW target Y, not the overwritten X.
|
||||
assert block_id == cow_copy.dst_block_id
|
||||
assert block_id != source_block_id
|
||||
# Draining clears it.
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
|
||||
# The hand-off pinned Y (its CoW retention is released after this step,
|
||||
# and Y is off the request block table); freeing the request unpins it.
|
||||
cow_block = manager.block_pool.blocks[block_id]
|
||||
pinned_ref = cow_block.ref_cnt
|
||||
assert pinned_ref >= 1
|
||||
manager.free(req0)
|
||||
assert cow_block.ref_cnt == pinned_ref - 1
|
||||
|
||||
|
||||
def test_partial_tail_pin_survives_released_cow_retention():
|
||||
"""If the CoW retention is released before the hand-off is drained
|
||||
(immediate-free mode), the drain must rescue the cow block from the free
|
||||
queue: a raw ref increment would leave a ref>0 block allocatable, and the
|
||||
next allocation would pop it and assert."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
req0.num_computed_tokens = 6
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
|
||||
# Retention released before the drain (defer_block_free=False ordering).
|
||||
_copies, retained = manager.take_kv_cache_block_copies()
|
||||
manager.block_pool.free_blocks(retained)
|
||||
|
||||
offloads = manager.take_partial_tail_offloads()
|
||||
((_group_id, block_id, boundary_tokens),) = offloads["0"]
|
||||
assert boundary_tokens == 6
|
||||
cow_block = manager.block_pool.blocks[block_id]
|
||||
assert cow_block.ref_cnt == 1
|
||||
|
||||
# The pinned block is out of the free queue: draining every free block
|
||||
# neither trips the allocator's ref_cnt assert nor hands it out.
|
||||
new_blocks = manager.block_pool.get_new_blocks(
|
||||
manager.block_pool.get_num_free_blocks()
|
||||
)
|
||||
assert block_id not in {b.block_id for b in new_blocks}
|
||||
|
||||
|
||||
def test_partial_tail_offload_dropped_when_request_freed_before_drain():
|
||||
"""A hand-off recorded in the same scheduling pass as the request's death
|
||||
must not be drained: its release hook has already run, so draining would
|
||||
leak a pinned block."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
req0.num_computed_tokens = 6
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
|
||||
# The request dies (preempt/abort) before the scheduler drains.
|
||||
manager.block_pool.free_blocks(manager.pop_blocks_for_free(req0))
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
|
||||
|
||||
def test_take_partial_tail_offloads_empty_without_partial_tail():
|
||||
"""A prompt ending on a block boundary registers no partial tail, so there
|
||||
is nothing to offload."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
# 4-token prompt ends exactly on the mamba block boundary (block_size=4).
|
||||
req0 = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
|
||||
req0.num_computed_tokens = 4
|
||||
req0.append_output_token_ids([2])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
|
||||
|
||||
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
"""truncate_computed_blocks slices each group by its own block size,
|
||||
keeps null placeholders in the retained prefix, and leaves the original
|
||||
lookup result untouched (pure view, no refcount changes)."""
|
||||
hash_block_size = 2
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=2 * hash_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
blocks, num_computed, _ = manager.get_computed_blocks(producer)
|
||||
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
|
||||
manager.free(producer)
|
||||
manager.new_step_starts()
|
||||
|
||||
consumer = make_request(
|
||||
"consumer", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256
|
||||
)
|
||||
blocks, num_computed, _ = manager.get_computed_blocks(consumer)
|
||||
assert num_computed == 6
|
||||
assert [len(group) for group in blocks.blocks] == [3, 2]
|
||||
assert blocks.blocks[1][0].is_null
|
||||
|
||||
truncated = manager.truncate_computed_blocks(blocks, 4)
|
||||
|
||||
assert [len(group) for group in truncated.blocks] == [2, 1]
|
||||
assert truncated.blocks[1][0].is_null
|
||||
assert [len(group) for group in blocks.blocks] == [3, 2]
|
||||
|
||||
|
||||
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
from math import lcm
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
ExternalCachedBlockPool,
|
||||
MooncakeStoreCoordinator,
|
||||
@@ -14,10 +16,20 @@ from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
|
||||
def _mamba_align(block_size=32):
|
||||
return MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
|
||||
|
||||
def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=None):
|
||||
"""Construct a coordinator using the natural LCM of group block sizes as
|
||||
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
|
||||
@@ -196,6 +208,49 @@ def test_coordinator_group_block_size_double_hash():
|
||||
assert hit % 32 == 0
|
||||
|
||||
|
||||
# ----- Fine-grained partial hits (full attention + mamba "align") -----
|
||||
|
||||
|
||||
def test_coordinator_fine_grained_partial_tail_hit():
|
||||
"""K3 shape: FA + mamba-align, block_size=32 over hash_block_size=16. When
|
||||
both groups have the sub-block boundary hash, the reconciled hit lands on
|
||||
the hash boundary (48), not the block boundary (32)."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(32)),
|
||||
KVCacheGroupSpec(["L1"], _mamba_align(32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
assert coord.enable_partial_hash_hits
|
||||
hs = _hashes(4) # 4 hash units of 16 = 64 tokens; block 0 = [0,32), etc.
|
||||
# Both groups: full block 0 (key = last sub-hash hs[1]) + partial boundary
|
||||
# at token 48 (key = hs[2]). No hs[3] -> block 1 is not full.
|
||||
exists = {(g, bytes(h)) for g in (0, 1) for h in (hs[1], hs[2])}
|
||||
cmap = ExternalCachedBlockPool(16, exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
assert hit == 48
|
||||
|
||||
|
||||
def test_coordinator_fine_grained_clips_when_one_group_missing_tail():
|
||||
"""If only one group has the sub-block boundary, min-convergence clips the
|
||||
reconciled hit back to the block boundary (32)."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(32)),
|
||||
KVCacheGroupSpec(["L1"], _mamba_align(32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
# Full block 0 for both; partial boundary hs[2] only for FA (group 0).
|
||||
exists = {(g, bytes(hs[1])) for g in (0, 1)}
|
||||
exists |= {(0, bytes(hs[2]))}
|
||||
cmap = ExternalCachedBlockPool(16, exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
assert hit == 32
|
||||
|
||||
|
||||
# ----- store_mask -----
|
||||
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
@@ -333,14 +334,14 @@ def test_recv_skips_swa_blocks_before_window():
|
||||
|
||||
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
|
||||
keys each 16-token chunk by its last fine hash, keeping the Mooncake key
|
||||
at one digest instead of concatenating all 4 fine hashes."""
|
||||
keys each chunk by its ending fine hash, including a partial tail."""
|
||||
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
|
||||
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
|
||||
db.set_kv_caches_base_addr([0])
|
||||
db.set_block_len([512])
|
||||
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
|
||||
fine_hashes = [BlockHash(bytes([i + 1]) * 4) for i in range(8)]
|
||||
|
||||
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
|
||||
out = list(db.process_tokens(token_len=32, block_hashes=fine_hashes))
|
||||
assert len(out) == 2
|
||||
assert out[0][0] == 0 and out[0][1] == 16
|
||||
@@ -349,3 +350,243 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
# prior three.
|
||||
assert out[0][2].hex() == fine_hashes[3].hex()
|
||||
assert out[1][2].hex() == fine_hashes[7].hex()
|
||||
|
||||
# Sub-block hit: emit the partial chunk under its ending fine hash.
|
||||
out = list(db.process_tokens(token_len=12, block_hashes=fine_hashes[:3]))
|
||||
assert [(s, e) for s, e, _ in out] == [(0, 12)]
|
||||
assert out[0][2].hex() == fine_hashes[2].hex()
|
||||
|
||||
# Cross-block hit: emit both the full chunk and its partial tail.
|
||||
out = list(db.process_tokens(token_len=28, block_hashes=fine_hashes[:7]))
|
||||
assert [(s, e) for s, e, _ in out] == [(0, 16), (16, 28)]
|
||||
assert out[0][2].hex() == fine_hashes[3].hex()
|
||||
assert out[1][2].hex() == fine_hashes[6].hex()
|
||||
|
||||
|
||||
def test_sub_block_partial_tail_offload_reads_cow_block():
|
||||
"""Sub-block prompt (the 900/128/1536 shape, scaled to 12/4/16): the
|
||||
partial tail is offloaded for both groups under the boundary sub-hash. The
|
||||
full-attention block is read from the request block table; the mamba block
|
||||
is the core-provided CoW target, not block_ids."""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
]
|
||||
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
|
||||
assert coord.enable_partial_hash_hits
|
||||
|
||||
class _RecordingStore(_DictStore):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.puts: dict[str, list[int]] = {}
|
||||
|
||||
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k):
|
||||
for key, addr in zip(keys, addrs):
|
||||
self.puts[key] = addr
|
||||
return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k)
|
||||
|
||||
store = _RecordingStore()
|
||||
token_dbs = []
|
||||
for g_idx in range(2):
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx),
|
||||
block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
db.set_kv_caches_base_addr([g_idx * 10_000])
|
||||
db.set_block_len([512])
|
||||
token_dbs.append(db)
|
||||
|
||||
send = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
coord=coord,
|
||||
token_databases=token_dbs,
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
group_put_steps=[1, 1],
|
||||
kv_role="kv_both",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=MagicMock(),
|
||||
)
|
||||
|
||||
# The surrounding metadata may describe a longer resumed replay, but the
|
||||
# handoff identifies the exact state boundary to persist.
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)]
|
||||
mamba_cow_block = 7
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1], [2]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
partial_tail_offloads=[(1, mamba_cow_block, 12)],
|
||||
)
|
||||
|
||||
send._maybe_offload_partial_tail(req)
|
||||
|
||||
# boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2].
|
||||
partial_hash = hs[2]
|
||||
fa_key = token_dbs[0].key_for(partial_hash)
|
||||
mamba_key = token_dbs[1].key_for(partial_hash)
|
||||
assert set(store.puts) == {fa_key, mamba_key}
|
||||
# FA reads block_ids[0][0] = block 1: addr = base(0) + 1 * 512.
|
||||
assert store.puts[fa_key] == [512]
|
||||
# Mamba reads the CoW block 7, not block_ids[1][0]=2.
|
||||
assert store.puts[mamba_key] == [10_000 + mamba_cow_block * 512]
|
||||
|
||||
|
||||
def test_offload_syncs_event_before_put():
|
||||
"""An offload-carrying meta synchronizes its CoW-fence event before the
|
||||
store put reads the blocks, then completes in one pass and drains the
|
||||
completion counter."""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
]
|
||||
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
|
||||
event = MagicMock()
|
||||
|
||||
class _FencedStore(_DictStore):
|
||||
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k):
|
||||
assert event.synchronize.called, "put must run after the event sync"
|
||||
return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k)
|
||||
|
||||
store = _FencedStore()
|
||||
token_dbs = []
|
||||
for g_idx in range(2):
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx),
|
||||
block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
db.set_kv_caches_base_addr([g_idx * 10_000])
|
||||
db.set_block_len([512])
|
||||
token_dbs.append(db)
|
||||
|
||||
send = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
coord=coord,
|
||||
token_databases=token_dbs,
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
group_put_steps=[1, 1],
|
||||
kv_role="kv_both",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=MagicMock(),
|
||||
)
|
||||
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(3)]
|
||||
req = ReqMeta(
|
||||
req_id="r1",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1], [2]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
partial_tail_offloads=[(1, 7, 12)],
|
||||
)
|
||||
req.current_event = event
|
||||
send.add_stored_request("r1")
|
||||
|
||||
send.request_queue.put(req)
|
||||
send._handle_request(send.request_queue.get())
|
||||
assert send.request_queue.qsize() == 0
|
||||
assert store._data
|
||||
assert send.stored_requests["r1"] == 0
|
||||
event.synchronize.assert_called_once()
|
||||
|
||||
|
||||
def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
|
||||
"""The K3-shaped 900/128/1536 scenario scaled to 12/4/16, with a
|
||||
full-attention group whose block (4) is smaller than the lcm (16): the
|
||||
offload must persist every FA block up to the boundary — the normal save
|
||||
floors to the lcm, so those blocks are otherwise never written and the
|
||||
consumer's per-group lookup would miss. The mamba boundary block still
|
||||
reads the core-provided CoW target."""
|
||||
full = FullAttentionSpec(block_size=4, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
]
|
||||
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
|
||||
assert coord.enable_partial_hash_hits
|
||||
|
||||
class _RecordingStore(_DictStore):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.puts: dict[str, list[int]] = {}
|
||||
|
||||
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k):
|
||||
for key, addr in zip(keys, addrs):
|
||||
self.puts[key] = addr
|
||||
return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k)
|
||||
|
||||
store = _RecordingStore()
|
||||
token_dbs = []
|
||||
for g_idx, block_size in enumerate([4, 16]):
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx),
|
||||
block_size=block_size,
|
||||
hash_block_size=4,
|
||||
)
|
||||
db.set_kv_caches_base_addr([g_idx * 10_000])
|
||||
db.set_block_len([512])
|
||||
token_dbs.append(db)
|
||||
|
||||
send = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
coord=coord,
|
||||
token_databases=token_dbs,
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
group_put_steps=[1, 1],
|
||||
kv_role="kv_both",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=MagicMock(),
|
||||
)
|
||||
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(3)] # 3 hash units = 12 tok
|
||||
mamba_cow_block = 7
|
||||
req = ReqMeta(
|
||||
req_id="r2",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1, 2, 3], [4]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
partial_tail_offloads=[(1, mamba_cow_block, 12)],
|
||||
)
|
||||
|
||||
send._maybe_offload_partial_tail(req)
|
||||
|
||||
# FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal
|
||||
# block-end hashes; mamba (block 16): the partial boundary block under
|
||||
# the boundary sub-hash, read from the CoW target.
|
||||
expected = {
|
||||
token_dbs[0].key_for(hs[0]): [1 * 512],
|
||||
token_dbs[0].key_for(hs[1]): [2 * 512],
|
||||
token_dbs[0].key_for(hs[2]): [3 * 512],
|
||||
token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512],
|
||||
}
|
||||
assert store.puts == expected
|
||||
|
||||
@@ -13,11 +13,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor
|
||||
)
|
||||
|
||||
|
||||
def _make_bare_scheduler() -> MooncakeStoreScheduler:
|
||||
def _make_bare_scheduler(
|
||||
*, hash_block_size: int = 16, enable_partial_hash_hits: bool = False
|
||||
) -> MooncakeStoreScheduler:
|
||||
scheduler = object.__new__(MooncakeStoreScheduler)
|
||||
scheduler.kv_role = "kv_both"
|
||||
scheduler.lookup_async = False
|
||||
scheduler._block_size = 16
|
||||
scheduler._hash_block_size = hash_block_size
|
||||
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
|
||||
scheduler.load_specs = {}
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
@@ -136,6 +140,7 @@ def test_preemption_resets_tracker_before_request_finished():
|
||||
block_hashes=[b"h0", b"h1"],
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
scheduler._request_trackers["req-0"].has_pending_offload = True
|
||||
|
||||
scheduler.build_connector_meta(_make_preemption_scheduler_output())
|
||||
|
||||
@@ -144,6 +149,7 @@ def test_preemption_resets_tracker_before_request_finished():
|
||||
assert tracker.allocated_block_ids == ()
|
||||
assert tracker.num_saved_tokens == 0
|
||||
assert tracker.token_ids is None
|
||||
assert tracker.has_pending_offload is False
|
||||
assert tracker.prefill_end_tokens == 0
|
||||
request = SimpleNamespace(request_id="req-0")
|
||||
assert scheduler.request_finished(request, ([0, 1],)) == (False, None)
|
||||
@@ -536,3 +542,227 @@ def test_full_external_hit_with_full_local_hit_skips_load():
|
||||
assert need_to_allocate == 0
|
||||
assert load_async is False
|
||||
assert "req-0" not in scheduler.load_specs
|
||||
|
||||
|
||||
def test_partial_hash_hit_block_aligned_local_loads_partial_tail():
|
||||
# Fine-grained on (hash=4, block=16): a block-aligned local hit can pull a
|
||||
# sub-block remote hit (24 = a hash boundary inside block 1). Loads [16, 24).
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler.load_async = True
|
||||
scheduler.client = _StubLookupClient(hit_tokens=24)
|
||||
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
num_tokens=32,
|
||||
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5", b"h6", b"h7"],
|
||||
)
|
||||
|
||||
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens=16
|
||||
)
|
||||
|
||||
assert need_to_allocate == 8
|
||||
assert load_async is True
|
||||
load_spec = scheduler.load_specs["req-0"]
|
||||
assert load_spec.vllm_cached_tokens == 16
|
||||
assert load_spec.kvpool_cached_tokens == 24
|
||||
|
||||
|
||||
def test_partial_hash_hit_no_remote_gain_skips_load():
|
||||
# Core always presents a block-aligned local hit (it floors a sub-block
|
||||
# tail before calling the connector). When the remote hit does not exceed
|
||||
# that block-aligned local hit, nothing is loaded.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler.load_async = True
|
||||
scheduler.client = _StubLookupClient(hit_tokens=16)
|
||||
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
num_tokens=32,
|
||||
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5", b"h6", b"h7"],
|
||||
)
|
||||
|
||||
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens=16
|
||||
)
|
||||
|
||||
assert need_to_allocate == 0
|
||||
assert load_async is False
|
||||
assert "req-0" not in scheduler.load_specs
|
||||
|
||||
|
||||
def test_sub_block_prompt_looks_up_with_fine_grained():
|
||||
# A prompt smaller than one block (12 < block 16). With fine-grained partial
|
||||
# hits the sub-block prefix is worth looking up (floor is the hash unit 4,
|
||||
# not a full block), so a remote partial hit is loaded. Pre-change the
|
||||
# block-size floor returned (0, False) for such prompts.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler.load_async = True
|
||||
scheduler.client = _StubLookupClient(hit_tokens=8)
|
||||
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
num_tokens=12,
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
)
|
||||
|
||||
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens=0
|
||||
)
|
||||
|
||||
assert need_to_allocate == 8
|
||||
assert load_async is True
|
||||
assert scheduler.load_specs["req-0"].kvpool_cached_tokens == 8
|
||||
|
||||
|
||||
def test_sub_block_prompt_not_looked_up_without_fine_grained():
|
||||
# Without fine-grained partial hits, sub-block prompts still skip the lookup
|
||||
# (there is no full block, and no sub-block key granularity).
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.client = _StubLookupClient(hit_tokens=8)
|
||||
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
num_tokens=12,
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
)
|
||||
|
||||
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens=0
|
||||
)
|
||||
|
||||
assert need_to_allocate == 0
|
||||
assert load_async is False
|
||||
assert "req-0" not in scheduler.load_specs
|
||||
|
||||
|
||||
def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
# A sub-block prompt never produces a block-aligned save, so the partial-
|
||||
# tail offload arriving this step is emitted as an offload-only ReqMeta
|
||||
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
|
||||
# the worker skips the normal save). Pending-offload state delays the free
|
||||
# without advancing the normal-save watermark before the put succeeds.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(12)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=12,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=12,
|
||||
allocated_block_ids=([0],),
|
||||
num_saved_tokens=0,
|
||||
token_ids=list(range(12)),
|
||||
prefill_end_tokens=12,
|
||||
)
|
||||
|
||||
out = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
partial_tail_offloads={"req-0": [(1, 7, 12)]},
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
req_meta = meta.requests[0]
|
||||
assert req_meta.req_id == "req-0"
|
||||
assert req_meta.can_save is True
|
||||
assert req_meta.token_len_chunk == 0
|
||||
assert req_meta.partial_tail_offloads == [(1, 7, 12)]
|
||||
assert req_meta.num_prompt_tokens == 12
|
||||
assert req_meta.block_ids == ([0],)
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 0
|
||||
assert tracker.has_pending_offload is True
|
||||
request = SimpleNamespace(request_id="req-0")
|
||||
assert scheduler.request_finished(request, ([0],)) == (True, None)
|
||||
|
||||
|
||||
def test_resumed_partial_tail_uses_handoff_boundary():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(20)),
|
||||
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4"],
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=12,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=20,
|
||||
allocated_block_ids=([0, 1],),
|
||||
num_saved_tokens=0,
|
||||
token_ids=list(range(20)),
|
||||
# Resumption replays prompt + previously generated tokens.
|
||||
prefill_end_tokens=20,
|
||||
)
|
||||
|
||||
out = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
partial_tail_offloads={"req-0": [(1, 7, 12)]},
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
assert meta.requests[0].partial_tail_offloads == [(1, 7, 12)]
|
||||
# Ordinary metadata retains the full resumed prefill range.
|
||||
assert meta.requests[0].num_prompt_tokens == 20
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 0
|
||||
assert tracker.has_pending_offload is True
|
||||
|
||||
|
||||
def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=36,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=44,
|
||||
allocated_block_ids=([0, 1],),
|
||||
num_saved_tokens=32,
|
||||
token_ids=list(range(44)),
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
out = _make_scheduler_output(scheduled_spec_tokens=None)
|
||||
out.partial_tail_offloads = {"req-0": [(0, 7, 36)]}
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
assert meta.requests[0].can_save is True
|
||||
assert meta.requests[0].partial_tail_offloads == [(0, 7, 36)]
|
||||
assert meta.requests[0].num_prompt_tokens == 48
|
||||
# Ordinary saving still covers the full resumed prefill range.
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 48
|
||||
assert tracker.has_pending_offload is True
|
||||
|
||||
@@ -599,6 +599,88 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def _make_partial_tail_send_thread(store):
|
||||
coord = SimpleNamespace(
|
||||
enable_partial_hash_hits=True,
|
||||
hash_block_size=4,
|
||||
lcm_block_size=16,
|
||||
)
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0),
|
||||
block_size=4,
|
||||
hash_block_size=4,
|
||||
)
|
||||
db.set_kv_caches_base_addr([0x1000])
|
||||
db.set_block_len([256])
|
||||
return _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db],
|
||||
)
|
||||
|
||||
|
||||
def _make_partial_tail_req(block_ids: list[int]) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=(block_ids,),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
partial_tail_offloads=[(1, 7, 12)],
|
||||
)
|
||||
|
||||
|
||||
def test_partial_tail_offload_skips_null_source_blocks():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
assert thread._maybe_offload_partial_tail(_make_partial_tail_req([0, 2, 3]))
|
||||
|
||||
keys, addrs, _sizes, _replicate_config = (
|
||||
store.batch_put_from_multi_buffers.call_args.args
|
||||
)
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
assert addrs == [[0x1000 + 2 * 256], [0x1000 + 3 * 256]]
|
||||
|
||||
|
||||
def test_partial_tail_offload_honors_active_pressure_gate():
|
||||
store = MagicMock()
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
thread._store_pressure_active = True
|
||||
thread._skip_store_requests.add("req-a")
|
||||
thread.add_stored_request("req-a")
|
||||
|
||||
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
|
||||
|
||||
def test_partial_tail_put_failure_activates_pressure_gate():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, -200, 256]
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
thread.add_stored_request("req-a")
|
||||
|
||||
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
|
||||
|
||||
assert thread._store_pressure_active is True
|
||||
assert thread._skip_store_requests == {"req-a"}
|
||||
assert thread._saved_offset.get("req-a", 0) == 0
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
@@ -1939,6 +2021,46 @@ def test_lookup_partial_prefix_returns_first_hit_length():
|
||||
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
|
||||
|
||||
|
||||
def test_lookup_partial_tail_uses_hash_alignment():
|
||||
"""A stored sub-block tail can serve a request extending past it."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
worker._kv_cache_groups = [
|
||||
KVCacheGroupSpec(["full"], full),
|
||||
KVCacheGroupSpec(["mamba"], mamba),
|
||||
]
|
||||
worker.hash_block_size = 4
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=group_id),
|
||||
block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
for group_id in range(2)
|
||||
]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 0, 0, 1]
|
||||
|
||||
assert worker.lookup(13, [b"h0", b"h1", b"h2"]) == 12
|
||||
|
||||
|
||||
def test_lookup_full_hit_reuses_existing_boundary():
|
||||
"""A full hit is re-derived below the request end without another RPC."""
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
|
||||
Reference in New Issue
Block a user