[2/N][Core] support partial prefix cache hit for hybrid model (#46384)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
8e981630c9
commit
481e481be7
@@ -0,0 +1,816 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Fine-grained partial prefix-cache hits for hybrid (full attention + mamba
|
||||||
|
"align") models: scheduler chunk splitting, partial tail registration, CoW
|
||||||
|
on partial hits, and same-step deferral."""
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from tests.v1.core.test_prefix_caching import make_kv_cache_manager, make_request
|
||||||
|
from vllm.utils.hashing import sha256
|
||||||
|
from vllm.v1.core.kv_cache_utils import (
|
||||||
|
KVCacheBlockCopy,
|
||||||
|
get_block_hash,
|
||||||
|
get_group_id,
|
||||||
|
init_none_hash,
|
||||||
|
)
|
||||||
|
from vllm.v1.core.sched.scheduler import Scheduler
|
||||||
|
from vllm.v1.kv_cache_interface import (
|
||||||
|
FullAttentionSpec,
|
||||||
|
KVCacheConfig,
|
||||||
|
KVCacheGroupSpec,
|
||||||
|
MambaSpec,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _auto_init_hash_fn():
|
||||||
|
init_none_hash(sha256)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mamba_align_split_partial_tail_schedule():
|
||||||
|
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||||
|
at the prompt's last hash boundary (registering the partial tail), then
|
||||||
|
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
|
||||||
|
0 -> 8192 -> 9728 -> 9984 -> 10000."""
|
||||||
|
block_size = 512
|
||||||
|
hash_block_size = 32
|
||||||
|
mock = SimpleNamespace(
|
||||||
|
cache_config=SimpleNamespace(block_size=block_size),
|
||||||
|
use_eagle=False,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
mamba_partial_cache_hit=True,
|
||||||
|
)
|
||||||
|
split = Scheduler._mamba_block_aligned_split
|
||||||
|
|
||||||
|
req = make_request("0", [0] * 10000, hash_block_size, sha256)
|
||||||
|
req.num_computed_tokens = 0
|
||||||
|
assert split(self=mock, request=req, num_new_tokens=8192) == 8192
|
||||||
|
req.num_computed_tokens = 8192
|
||||||
|
# Stop at the last block boundary (9728).
|
||||||
|
assert split(self=mock, request=req, num_new_tokens=1808) == 1536
|
||||||
|
req.num_computed_tokens = 9728
|
||||||
|
# Extra stop at the prompt's last hash boundary (9984).
|
||||||
|
assert split(self=mock, request=req, num_new_tokens=272) == 256
|
||||||
|
req.num_computed_tokens = 9984
|
||||||
|
# Final 16 tokens run unchanged (no mid-block-resume stop: the next
|
||||||
|
# block boundary is past the last block boundary).
|
||||||
|
assert split(self=mock, request=req, num_new_tokens=16) == 16
|
||||||
|
|
||||||
|
# Partial hits off: no extra stop, the tail runs in one chunk.
|
||||||
|
mock.mamba_partial_cache_hit = False
|
||||||
|
req.num_computed_tokens = 9728
|
||||||
|
assert split(self=mock, request=req, num_new_tokens=272) == 272
|
||||||
|
mock.mamba_partial_cache_hit = True
|
||||||
|
|
||||||
|
# A request resumed mid-block (partial hash hit at 9984): the first chunk
|
||||||
|
# stops at the next block boundary (10240), later chunk ends re-align.
|
||||||
|
req2 = make_request("1", [0] * 12000, hash_block_size, sha256)
|
||||||
|
req2.num_computed_tokens = 9984
|
||||||
|
assert split(self=mock, request=req2, num_new_tokens=2016) == 256
|
||||||
|
req2.num_computed_tokens = 10240
|
||||||
|
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_mamba_align_partial_hash_hit():
|
||||||
|
hash_block_size = 2
|
||||||
|
mamba_block_size = 2 * hash_block_size
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=20,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=hash_block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=mamba_block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
manager = make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||||
|
assert num_computed == 0
|
||||||
|
blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks)
|
||||||
|
assert blocks is not None
|
||||||
|
manager.free(req0)
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||||
|
partial_mamba_block = manager.block_pool.get_cached_block(
|
||||||
|
partial_mamba_hash, kv_cache_group_ids=[1]
|
||||||
|
)
|
||||||
|
assert partial_mamba_block is not None
|
||||||
|
assert partial_mamba_block[0].block_hash_num_tokens == 6
|
||||||
|
|
||||||
|
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 6
|
||||||
|
assert [len(group) for group in computed_blocks.blocks] == [3, 2]
|
||||||
|
|
||||||
|
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
|
||||||
|
assert new_blocks is not None
|
||||||
|
mamba_new_block_ids = new_blocks.get_block_ids()[1]
|
||||||
|
assert len(mamba_new_block_ids) == 1
|
||||||
|
assert mamba_new_block_ids[0] != partial_mamba_block[0].block_id
|
||||||
|
assert manager.get_blocks("1").get_block_ids()[1][1] == mamba_new_block_ids[0]
|
||||||
|
assert partial_mamba_block[0].block_hash is not None
|
||||||
|
assert get_block_hash(partial_mamba_block[0].block_hash) == partial_mamba_hash
|
||||||
|
assert get_group_id(partial_mamba_block[0].block_hash) == 1
|
||||||
|
assert partial_mamba_block[0].block_hash_num_tokens == 6
|
||||||
|
copies, _ = manager.take_kv_cache_block_copies()
|
||||||
|
assert (
|
||||||
|
KVCacheBlockCopy(
|
||||||
|
src_block_id=partial_mamba_block[0].block_id,
|
||||||
|
dst_block_id=mamba_new_block_ids[0],
|
||||||
|
)
|
||||||
|
in copies
|
||||||
|
)
|
||||||
|
assert manager.get_blocks("1").blocks[1][1].block_hash_num_tokens == 8
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 2 * hash_block_size
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=24,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=hash_block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
manager = make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||||
|
assert num_computed == 0
|
||||||
|
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||||
|
|
||||||
|
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||||
|
partial_mamba_block = manager.block_pool.get_cached_block(
|
||||||
|
partial_mamba_hash, kv_cache_group_ids=[1]
|
||||||
|
)
|
||||||
|
assert partial_mamba_block is not None
|
||||||
|
partial_mamba_block_id = partial_mamba_block[0].block_id
|
||||||
|
assert manager.get_blocks("0").get_block_ids()[1][1] == partial_mamba_block_id
|
||||||
|
|
||||||
|
req0.num_computed_tokens = 6
|
||||||
|
req0.append_output_token_ids([3])
|
||||||
|
new_blocks = manager.allocate_slots(req0, 1)
|
||||||
|
assert new_blocks is not None
|
||||||
|
|
||||||
|
# Reversed CoW for the owning request: it keeps its own block (the
|
||||||
|
# worker's block table is append-only), and no new mamba block is handed
|
||||||
|
# to the worker. The prefix-cache entry is moved to a private copy that
|
||||||
|
# the queued block copy fills before the next forward.
|
||||||
|
assert new_blocks.get_block_ids()[1] == []
|
||||||
|
assert manager.get_blocks("0").get_block_ids()[1][1] == partial_mamba_block_id
|
||||||
|
copies, _ = manager.take_kv_cache_block_copies()
|
||||||
|
cow_copy = next(c for c in copies if c.src_block_id == partial_mamba_block_id)
|
||||||
|
assert cow_copy.dst_block_id != partial_mamba_block_id
|
||||||
|
# The source block gave up the hash; the copy target now owns the entry.
|
||||||
|
assert partial_mamba_block[0].block_hash is None
|
||||||
|
moved = manager.block_pool.get_cached_block(
|
||||||
|
partial_mamba_hash, kv_cache_group_ids=[1]
|
||||||
|
)
|
||||||
|
assert moved is not None
|
||||||
|
assert moved[0].block_id == cow_copy.dst_block_id
|
||||||
|
assert get_block_hash(moved[0].block_hash) == partial_mamba_hash
|
||||||
|
assert get_group_id(moved[0].block_hash) == 1
|
||||||
|
assert moved[0].block_hash_num_tokens == 6
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 2 * hash_block_size
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=32,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=hash_block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
manager = make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||||
|
assert num_computed == 0
|
||||||
|
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||||
|
|
||||||
|
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||||
|
partial_mamba_block = manager.block_pool.get_cached_block(
|
||||||
|
partial_mamba_hash, kv_cache_group_ids=[1]
|
||||||
|
)
|
||||||
|
assert partial_mamba_block is not None
|
||||||
|
partial_mamba_block_id = partial_mamba_block[0].block_id
|
||||||
|
|
||||||
|
req0.num_computed_tokens = 6
|
||||||
|
req0.append_output_token_ids([3])
|
||||||
|
assert manager.allocate_slots(req0, 1) is not None
|
||||||
|
# The owner moved the prefix-cache entry to a private copy; capture its id.
|
||||||
|
owner_copies, _ = manager.take_kv_cache_block_copies()
|
||||||
|
cow_copy = next(c for c in owner_copies if c.src_block_id == partial_mamba_block_id)
|
||||||
|
moved_block_id = cow_copy.dst_block_id
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 6
|
||||||
|
# The later request hits the moved (private-copy) entry, not the source.
|
||||||
|
assert computed_blocks.get_block_ids()[1][1] == moved_block_id
|
||||||
|
|
||||||
|
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
|
||||||
|
assert new_blocks is not None
|
||||||
|
mamba_new_block_ids = new_blocks.get_block_ids()[1]
|
||||||
|
assert len(mamba_new_block_ids) == 1
|
||||||
|
assert mamba_new_block_ids[0] != moved_block_id
|
||||||
|
# The hitting request CoWs from the moved entry into its own private block.
|
||||||
|
copies, _ = manager.take_kv_cache_block_copies()
|
||||||
|
assert (
|
||||||
|
KVCacheBlockCopy(
|
||||||
|
src_block_id=moved_block_id,
|
||||||
|
dst_block_id=mamba_new_block_ids[0],
|
||||||
|
)
|
||||||
|
in copies
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
|
||||||
|
"""The owner's move re-arms the same-step guard: the moved entry is
|
||||||
|
filled by this step's copy, and chained same-step copies read stale
|
||||||
|
sources, so a request hitting it in the move step must be deferred."""
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 2 * hash_block_size
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=32,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=hash_block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
manager = make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||||
|
assert num_computed == 0
|
||||||
|
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
# The owning request continues decoding: the partial entry moves to a
|
||||||
|
# private copy in this step.
|
||||||
|
req0.num_computed_tokens = 6
|
||||||
|
req0.append_output_token_ids([3])
|
||||||
|
assert manager.allocate_slots(req0, 1) is not None
|
||||||
|
|
||||||
|
# A request hitting the moved entry in the SAME step must be deferred.
|
||||||
|
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 6
|
||||||
|
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is None
|
||||||
|
|
||||||
|
# Next step the moved entry is consumable.
|
||||||
|
manager.new_step_starts()
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 6
|
||||||
|
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_full_attention_partial_hash_hit_uses_cow():
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 2 * hash_block_size
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=24,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
manager = make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||||
|
assert num_computed == 0
|
||||||
|
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||||
|
manager.free(req0)
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
partial_full_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||||
|
partial_full_block = manager.block_pool.get_cached_block(
|
||||||
|
partial_full_hash, kv_cache_group_ids=[0]
|
||||||
|
)
|
||||||
|
assert partial_full_block is not None
|
||||||
|
|
||||||
|
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 6
|
||||||
|
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
|
||||||
|
|
||||||
|
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
|
||||||
|
assert new_blocks is not None
|
||||||
|
full_new_block_ids = new_blocks.get_block_ids()[0]
|
||||||
|
assert len(full_new_block_ids) == 1
|
||||||
|
assert full_new_block_ids[0] != partial_full_block[0].block_id
|
||||||
|
assert partial_full_block[0].block_hash is not None
|
||||||
|
assert get_block_hash(partial_full_block[0].block_hash) == partial_full_hash
|
||||||
|
assert get_group_id(partial_full_block[0].block_hash) == 0
|
||||||
|
assert partial_full_block[0].block_hash_num_tokens == 6
|
||||||
|
copies, retained = manager.take_kv_cache_block_copies()
|
||||||
|
assert (
|
||||||
|
KVCacheBlockCopy(
|
||||||
|
src_block_id=partial_full_block[0].block_id,
|
||||||
|
dst_block_id=full_new_block_ids[0],
|
||||||
|
)
|
||||||
|
in copies
|
||||||
|
)
|
||||||
|
assert partial_full_block[0].ref_cnt == 1
|
||||||
|
manager.block_pool.free_blocks(retained)
|
||||||
|
assert partial_full_block[0].ref_cnt == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_partial_hit_cow_target_starts_uncached():
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 2 * hash_block_size
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=32,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
manager = make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||||
|
assert num_computed == 0
|
||||||
|
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||||
|
manager.free(req0)
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
partial_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||||
|
partial_full_block = manager.block_pool.get_cached_block(
|
||||||
|
partial_hash, kv_cache_group_ids=[0]
|
||||||
|
)
|
||||||
|
partial_mamba_block = manager.block_pool.get_cached_block(
|
||||||
|
partial_hash, kv_cache_group_ids=[1]
|
||||||
|
)
|
||||||
|
assert partial_full_block is not None
|
||||||
|
assert partial_mamba_block is not None
|
||||||
|
|
||||||
|
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 6
|
||||||
|
|
||||||
|
new_blocks = manager.allocate_slots(
|
||||||
|
req1,
|
||||||
|
2,
|
||||||
|
num_computed,
|
||||||
|
computed_blocks,
|
||||||
|
delay_cache_blocks=True,
|
||||||
|
)
|
||||||
|
assert new_blocks is not None
|
||||||
|
|
||||||
|
full_cow_block = manager.get_blocks("1").blocks[0][1]
|
||||||
|
mamba_cow_block = manager.get_blocks("1").blocks[1][1]
|
||||||
|
assert full_cow_block.block_id != partial_full_block[0].block_id
|
||||||
|
assert mamba_cow_block.block_id != partial_mamba_block[0].block_id
|
||||||
|
assert full_cow_block.block_hash is None
|
||||||
|
assert full_cow_block.block_hash_num_tokens is None
|
||||||
|
assert mamba_cow_block.block_hash is None
|
||||||
|
assert mamba_cow_block.block_hash_num_tokens is None
|
||||||
|
|
||||||
|
assert partial_full_block[0].block_hash is not None
|
||||||
|
assert get_block_hash(partial_full_block[0].block_hash) == partial_hash
|
||||||
|
assert get_group_id(partial_full_block[0].block_hash) == 0
|
||||||
|
assert partial_full_block[0].block_hash_num_tokens == 6
|
||||||
|
assert partial_mamba_block[0].block_hash is not None
|
||||||
|
assert get_block_hash(partial_mamba_block[0].block_hash) == partial_hash
|
||||||
|
assert get_group_id(partial_mamba_block[0].block_hash) == 1
|
||||||
|
assert partial_mamba_block[0].block_hash_num_tokens == 6
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_partial_hash_truncates_full_attention_hit_length():
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 2 * hash_block_size
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=24,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
manager = make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
)
|
||||||
|
pool = manager.block_pool
|
||||||
|
req = make_request(
|
||||||
|
"0",
|
||||||
|
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
|
||||||
|
hash_block_size,
|
||||||
|
sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
full_blocks = pool.get_new_blocks(3)
|
||||||
|
pool.cache_full_blocks(
|
||||||
|
request=req,
|
||||||
|
blocks=full_blocks,
|
||||||
|
num_cached_blocks=0,
|
||||||
|
num_full_blocks=2,
|
||||||
|
block_size=block_size,
|
||||||
|
kv_cache_group_id=0,
|
||||||
|
)
|
||||||
|
pool.cache_partial_block(
|
||||||
|
request=req,
|
||||||
|
block=full_blocks[2],
|
||||||
|
num_tokens=10,
|
||||||
|
kv_cache_group_id=0,
|
||||||
|
block_size=block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
mamba_block = pool.get_new_blocks(1)[0]
|
||||||
|
pool.cache_partial_block(
|
||||||
|
request=req,
|
||||||
|
block=mamba_block,
|
||||||
|
num_tokens=6,
|
||||||
|
kv_cache_group_id=1,
|
||||||
|
block_size=block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req)
|
||||||
|
assert num_computed == 6
|
||||||
|
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
|
||||||
|
|
||||||
|
|
||||||
|
def test_cow_retained_blocks_returned_for_release():
|
||||||
|
"""new_step_starts returns the CoW copy retentions instead of freeing
|
||||||
|
them; the scheduler owns releasing them once the copy has run."""
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 2 * hash_block_size
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=24,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=hash_block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
manager = make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
)
|
||||||
|
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||||
|
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||||
|
|
||||||
|
# The owner's move queues a copy and retains both endpoints.
|
||||||
|
req0.num_computed_tokens = 6
|
||||||
|
req0.append_output_token_ids([3])
|
||||||
|
assert manager.allocate_slots(req0, 1) is not None
|
||||||
|
(cow_copy,), retained = manager.take_kv_cache_block_copies()
|
||||||
|
assert {b.block_id for b in retained} == {
|
||||||
|
cow_copy.src_block_id,
|
||||||
|
cow_copy.dst_block_id,
|
||||||
|
}
|
||||||
|
# Not freed yet: the retention refs are still held.
|
||||||
|
assert all(b.ref_cnt > 0 for b in retained)
|
||||||
|
manager.block_pool.free_blocks(retained)
|
||||||
|
|
||||||
|
|
||||||
|
def test_free_cow_retained_blocks_defers_until_copy_step_processed():
|
||||||
|
"""Scheduler releases CoW retentions immediately when the copy's step has
|
||||||
|
been processed (or deferral is off), and defers them otherwise."""
|
||||||
|
from collections import deque
|
||||||
|
|
||||||
|
freed: list = []
|
||||||
|
blocks = [SimpleNamespace(block_id=7), SimpleNamespace(block_id=9)]
|
||||||
|
mock = SimpleNamespace(
|
||||||
|
kv_cache_manager=SimpleNamespace(
|
||||||
|
block_pool=SimpleNamespace(free_blocks=freed.extend)
|
||||||
|
),
|
||||||
|
deferred_frees=deque(),
|
||||||
|
defer_block_free=True,
|
||||||
|
processed_step_seq=2,
|
||||||
|
)
|
||||||
|
free = Scheduler._free_cow_retained_blocks
|
||||||
|
|
||||||
|
# Copy step still in flight: deferred with its fence.
|
||||||
|
free(mock, list(blocks), fence_seq=3)
|
||||||
|
assert not freed
|
||||||
|
assert mock.deferred_frees == deque([(3, blocks[::-1])])
|
||||||
|
|
||||||
|
# Copy step processed: freed immediately.
|
||||||
|
mock.processed_step_seq = 3
|
||||||
|
free(mock, list(blocks), fence_seq=3)
|
||||||
|
assert freed == blocks
|
||||||
|
|
||||||
|
# Deferral disabled: freed immediately regardless of the fence.
|
||||||
|
freed.clear()
|
||||||
|
mock.deferred_frees.clear()
|
||||||
|
mock.defer_block_free = False
|
||||||
|
mock.processed_step_seq = 0
|
||||||
|
free(mock, list(blocks), fence_seq=3)
|
||||||
|
assert freed == blocks
|
||||||
|
|
||||||
|
|
||||||
|
def test_full_attention_eagle_drops_one_hash_unit():
|
||||||
|
"""With fine-grained partial hits, eagle rewinds the hit by one hash unit
|
||||||
|
instead of a whole cache block: the tail block's KV is append-only, so it
|
||||||
|
still covers the reduced length and stays in the hit as a partial block."""
|
||||||
|
from vllm.v1.core.block_pool import BlockPool
|
||||||
|
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
|
||||||
|
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 4
|
||||||
|
pool = BlockPool(
|
||||||
|
num_gpu_blocks=10, enable_caching=True, hash_block_size=hash_block_size
|
||||||
|
)
|
||||||
|
spec = FullAttentionSpec(
|
||||||
|
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||||
|
)
|
||||||
|
req = make_request("0", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||||
|
|
||||||
|
def find(drop_eagle_block):
|
||||||
|
return FullAttentionManager.find_longest_cache_hit(
|
||||||
|
block_hashes=req.block_hashes,
|
||||||
|
max_length=8,
|
||||||
|
kv_cache_group_ids=[0],
|
||||||
|
block_pool=pool,
|
||||||
|
kv_cache_spec=spec,
|
||||||
|
drop_eagle_block=drop_eagle_block,
|
||||||
|
alignment_tokens=hash_block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Two full cached blocks (hit 8): eagle rewinds to 6, keeping the last
|
||||||
|
# block as a partial hit instead of dropping it to 4.
|
||||||
|
blocks = pool.get_new_blocks(2)
|
||||||
|
pool.cache_full_blocks(
|
||||||
|
request=req,
|
||||||
|
blocks=blocks,
|
||||||
|
num_cached_blocks=0,
|
||||||
|
num_full_blocks=2,
|
||||||
|
block_size=block_size,
|
||||||
|
kv_cache_group_id=0,
|
||||||
|
)
|
||||||
|
hit_blocks, hit_length = find(drop_eagle_block=False)
|
||||||
|
assert (hit_length, len(hit_blocks[0])) == (8, 2)
|
||||||
|
hit_blocks, hit_length = find(drop_eagle_block=True)
|
||||||
|
assert (hit_length, len(hit_blocks[0])) == (6, 2)
|
||||||
|
|
||||||
|
# A partial tail at 6 (block 1 not fully cached): eagle rewinds to the
|
||||||
|
# block boundary and trims the tail block.
|
||||||
|
pool2 = BlockPool(
|
||||||
|
num_gpu_blocks=10, enable_caching=True, hash_block_size=hash_block_size
|
||||||
|
)
|
||||||
|
pool = pool2
|
||||||
|
blocks = pool.get_new_blocks(2)
|
||||||
|
pool.cache_full_blocks(
|
||||||
|
request=req,
|
||||||
|
blocks=blocks[:1],
|
||||||
|
num_cached_blocks=0,
|
||||||
|
num_full_blocks=1,
|
||||||
|
block_size=block_size,
|
||||||
|
kv_cache_group_id=0,
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
pool.cache_partial_block(
|
||||||
|
request=req,
|
||||||
|
block=blocks[1],
|
||||||
|
num_tokens=6,
|
||||||
|
kv_cache_group_id=0,
|
||||||
|
block_size=block_size,
|
||||||
|
)
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
hit_blocks, hit_length = find(drop_eagle_block=False)
|
||||||
|
assert (hit_length, len(hit_blocks[0])) == (6, 2)
|
||||||
|
hit_blocks, hit_length = find(drop_eagle_block=True)
|
||||||
|
assert (hit_length, len(hit_blocks[0])) == (4, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
|
||||||
|
"""Regression: with eagle, the mamba group must not receive the eagle
|
||||||
|
lookup margin — its finder never applies the drop, so it could return a
|
||||||
|
hit past the blocks the (dropped) full-attention group covers, crashing
|
||||||
|
the consumer's CoW with block_idx >= len(req_blocks)."""
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 2 * hash_block_size
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=32,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
manager = make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
use_eagle=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# The owner prefills in scheduler-split style: stop at the block boundary
|
||||||
|
# (4), then at the prompt's last hash boundary (6, partial entries).
|
||||||
|
req0 = make_request("0", [7] * 6, hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||||
|
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
|
||||||
|
req0.num_computed_tokens = 4
|
||||||
|
manager.new_step_starts()
|
||||||
|
assert manager.allocate_slots(req0, 2) is not None
|
||||||
|
req0.num_computed_tokens = 6
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
# A longer request with eagle: full attention drops the partial tail, so
|
||||||
|
# the joint hit must fall back to the block boundary the FA blocks cover.
|
||||||
|
req1 = make_request("1", [7] * 6 + [9] * 2, hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 4
|
||||||
|
assert all(
|
||||||
|
len(group) * block_size >= num_computed for group in computed_blocks.blocks
|
||||||
|
)
|
||||||
|
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
|
||||||
@@ -18,6 +18,7 @@ from unittest.mock import PropertyMock, patch
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
|
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||||
from vllm.v1.core.sched.output import SchedulerOutput
|
from vllm.v1.core.sched.output import SchedulerOutput
|
||||||
from vllm.v1.outputs import ModelRunnerOutput
|
from vllm.v1.outputs import ModelRunnerOutput
|
||||||
from vllm.v1.request import RequestStatus
|
from vllm.v1.request import RequestStatus
|
||||||
@@ -412,3 +413,64 @@ def test_non_async_abort_defers_via_last_sched_seq():
|
|||||||
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
||||||
assert not scheduler.deferred_frees
|
assert not scheduler.deferred_frees
|
||||||
assert pool.get_num_free_blocks() == num_free_initially
|
assert pool.get_num_free_blocks() == num_free_initially
|
||||||
|
|
||||||
|
|
||||||
|
def test_cow_retentions_deferred_until_copy_step_processed():
|
||||||
|
"""The endpoints of a queued KV block copy must stay out of the free
|
||||||
|
pool until the step that runs the copy has been processed. Freed
|
||||||
|
earlier, an endpoint can be reallocated (e.g. as a PD KV-load
|
||||||
|
destination) and overwritten by a transfer that is not ordered against
|
||||||
|
the copy still pending in the in-flight step.
|
||||||
|
"""
|
||||||
|
scheduler = _create_deferring_scheduler()
|
||||||
|
pool = scheduler.kv_cache_manager.block_pool
|
||||||
|
manager = scheduler.kv_cache_manager.coordinator.single_type_managers[0]
|
||||||
|
|
||||||
|
request = create_requests(
|
||||||
|
num_requests=1,
|
||||||
|
num_tokens=NUM_PROMPT_TOKENS,
|
||||||
|
max_tokens=5,
|
||||||
|
stop_token_ids=[STOP_TOKEN_ID],
|
||||||
|
)[0]
|
||||||
|
scheduler.add_request(request)
|
||||||
|
|
||||||
|
# Simulate a partial-hit CoW performed while scheduling step 1, whose
|
||||||
|
# hitting request was freed within the same step: the copy rides out0
|
||||||
|
# and each endpoint stays alive only through its copy retention.
|
||||||
|
src_block, dst_block = pool.get_new_blocks(2)
|
||||||
|
block_copy = KVCacheBlockCopy(
|
||||||
|
src_block_id=src_block.block_id, dst_block_id=dst_block.block_id
|
||||||
|
)
|
||||||
|
manager._pending_cow_copies.append((src_block, dst_block))
|
||||||
|
out0 = scheduler.schedule()
|
||||||
|
assert out0.kv_cache_block_copies == [block_copy]
|
||||||
|
|
||||||
|
# Exhaust the rest of the pool so the copy endpoints are the only blocks
|
||||||
|
# a new request could receive, then add one that fits exactly in them.
|
||||||
|
pool.get_new_blocks(pool.get_num_free_blocks())
|
||||||
|
late_request = create_requests(
|
||||||
|
num_requests=1,
|
||||||
|
num_tokens=2 * scheduler.block_size,
|
||||||
|
max_tokens=5,
|
||||||
|
req_ids=["late"],
|
||||||
|
)[0]
|
||||||
|
scheduler.add_request(late_request)
|
||||||
|
|
||||||
|
# Step 2 is scheduled while step 1 (which runs the copy) is still in
|
||||||
|
# flight: the retentions are released against the copy's fence, so the
|
||||||
|
# endpoints must not reach the free pool -- the late request must not be
|
||||||
|
# scheduled onto them.
|
||||||
|
out1 = scheduler.schedule()
|
||||||
|
assert src_block.ref_cnt == 1
|
||||||
|
assert dst_block.ref_cnt == 1
|
||||||
|
assert scheduler.deferred_frees
|
||||||
|
assert not out1.scheduled_new_reqs
|
||||||
|
|
||||||
|
# Step 1's output is processed: the copy has run, endpoints return to
|
||||||
|
# the pool and the late request can be scheduled onto them safely.
|
||||||
|
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
||||||
|
assert src_block.ref_cnt == 0
|
||||||
|
assert dst_block.ref_cnt == 0
|
||||||
|
assert not scheduler.deferred_frees
|
||||||
|
out2 = scheduler.schedule()
|
||||||
|
assert [r.req_id for r in out2.scheduled_new_reqs] == ["late"]
|
||||||
|
|||||||
@@ -2597,3 +2597,63 @@ def test_hma_not_disabled_when_kv_events_enabled():
|
|||||||
assert vllm_config.scheduler_config.disable_hybrid_kv_cache_manager is False, (
|
assert vllm_config.scheduler_config.disable_hybrid_kv_cache_manager is False, (
|
||||||
"kv_events_config must not force-disable the hybrid KV cache manager."
|
"kv_events_config must not force-disable the hybrid KV cache manager."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_block_hashes_gate():
|
||||||
|
# Resolve symbols through the module so they stay consistent with each other
|
||||||
|
# even after other tests reload ``kv_cache_utils``.
|
||||||
|
resolve_block_hashes = kv_cache_utils.resolve_block_hashes
|
||||||
|
BlockHashListWithBlockSize = kv_cache_utils.BlockHashListWithBlockSize
|
||||||
|
# Raw, hash_block_size-granularity hashes (contents are opaque here).
|
||||||
|
raw = [BlockHash(bytes([i])) for i in range(8)]
|
||||||
|
|
||||||
|
# block_size == hash_block_size: always reuse the raw hashes.
|
||||||
|
assert resolve_block_hashes(raw, 2, 2, alignment_tokens=2) is raw
|
||||||
|
assert (
|
||||||
|
resolve_block_hashes(
|
||||||
|
raw, 2, 2, supports_fine_grained_hash_lookup=True, alignment_tokens=2
|
||||||
|
)
|
||||||
|
is raw
|
||||||
|
)
|
||||||
|
|
||||||
|
# Fine-grained manager, partial hits ON (alignment_tokens == hash_block_size
|
||||||
|
# < block_size): keep raw hashes so the manager can scan at hash granularity.
|
||||||
|
assert (
|
||||||
|
resolve_block_hashes(
|
||||||
|
raw, 2, 4, supports_fine_grained_hash_lookup=True, alignment_tokens=2
|
||||||
|
)
|
||||||
|
is raw
|
||||||
|
)
|
||||||
|
|
||||||
|
# Fine-grained manager, partial hits OFF (alignment_tokens ==
|
||||||
|
# scheduler_block_size >= block_size): must fall back to a block-size view,
|
||||||
|
# exactly like the pre-refactor coordinator did.
|
||||||
|
off = resolve_block_hashes(
|
||||||
|
raw, 2, 4, supports_fine_grained_hash_lookup=True, alignment_tokens=4
|
||||||
|
)
|
||||||
|
assert isinstance(off, BlockHashListWithBlockSize)
|
||||||
|
assert off.scale_factor == 2
|
||||||
|
|
||||||
|
# Non-fine-grained manager (e.g. sliding window): always a block-size view
|
||||||
|
# when block_size != hash_block_size, regardless of alignment_tokens.
|
||||||
|
swa = resolve_block_hashes(
|
||||||
|
raw, 2, 4, supports_fine_grained_hash_lookup=False, alignment_tokens=2
|
||||||
|
)
|
||||||
|
assert isinstance(swa, BlockHashListWithBlockSize)
|
||||||
|
assert swa.scale_factor == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_block_hashes_rejects_mismatched_view():
|
||||||
|
resolve_block_hashes = kv_cache_utils.resolve_block_hashes
|
||||||
|
BlockHashListWithBlockSize = kv_cache_utils.BlockHashListWithBlockSize
|
||||||
|
raw = [BlockHash(bytes([i])) for i in range(8)]
|
||||||
|
|
||||||
|
# A view built at this block_size is returned as-is (idempotent).
|
||||||
|
view = BlockHashListWithBlockSize(raw, 2, 4)
|
||||||
|
assert resolve_block_hashes(view, 2, 4) is view
|
||||||
|
|
||||||
|
# A view built at a different block_size must fail loudly rather than be
|
||||||
|
# silently reinterpreted at the wrong granularity.
|
||||||
|
mismatched = BlockHashListWithBlockSize(raw, 2, 8)
|
||||||
|
with pytest.raises(AssertionError):
|
||||||
|
resolve_block_hashes(mismatched, 2, 4)
|
||||||
|
|||||||
@@ -1072,7 +1072,10 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
|||||||
# Next, validate scheduler logic for num_uncached_common_prefix_tokens > 0
|
# Next, validate scheduler logic for num_uncached_common_prefix_tokens > 0
|
||||||
# Create minimal mock with just the needed attributes
|
# Create minimal mock with just the needed attributes
|
||||||
mock = SimpleNamespace(
|
mock = SimpleNamespace(
|
||||||
cache_config=SimpleNamespace(block_size=block_size), use_eagle=False
|
cache_config=SimpleNamespace(block_size=block_size),
|
||||||
|
use_eagle=False,
|
||||||
|
hash_block_size=block_size,
|
||||||
|
mamba_partial_cache_hit=False,
|
||||||
)
|
)
|
||||||
num_new_tokens_adjusted = Scheduler._mamba_block_aligned_split(
|
num_new_tokens_adjusted = Scheduler._mamba_block_aligned_split(
|
||||||
self=mock,
|
self=mock,
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ def test_chunked_local_attention_possible_cached_prefix():
|
|||||||
kv_cache_spec=chunked_local_attention_spec,
|
kv_cache_spec=chunked_local_attention_spec,
|
||||||
drop_eagle_block=False,
|
drop_eagle_block=False,
|
||||||
alignment_tokens=block_size,
|
alignment_tokens=block_size,
|
||||||
)[0]
|
)[0][0]
|
||||||
assert len(computed_blocks) == expect_length
|
assert len(computed_blocks) == expect_length
|
||||||
|
|
||||||
assert all(
|
assert all(
|
||||||
@@ -164,7 +164,7 @@ def test_sliding_window_possible_cached_prefix():
|
|||||||
kv_cache_spec=sliding_window_spec,
|
kv_cache_spec=sliding_window_spec,
|
||||||
drop_eagle_block=False,
|
drop_eagle_block=False,
|
||||||
alignment_tokens=block_size,
|
alignment_tokens=block_size,
|
||||||
)[0]
|
)[0][0]
|
||||||
assert len(computed_blocks) == expect_length
|
assert len(computed_blocks) == expect_length
|
||||||
|
|
||||||
assert all(
|
assert all(
|
||||||
@@ -398,13 +398,13 @@ def test_get_num_blocks_to_allocate():
|
|||||||
|
|
||||||
assert (
|
assert (
|
||||||
manager.get_num_blocks_to_allocate(
|
manager.get_num_blocks_to_allocate(
|
||||||
"1", 20 * block_size, cached_blocks_1, 0, 20 * block_size
|
"1", 20 * block_size, cached_blocks_1, 0, 0, 20 * block_size
|
||||||
)
|
)
|
||||||
== 20
|
== 20
|
||||||
)
|
)
|
||||||
assert (
|
assert (
|
||||||
manager.get_num_blocks_to_allocate(
|
manager.get_num_blocks_to_allocate(
|
||||||
"2", 20 * block_size, cached_blocks_2, 0, 20 * block_size
|
"2", 20 * block_size, cached_blocks_2, 0, 0, 20 * block_size
|
||||||
)
|
)
|
||||||
== 15
|
== 15
|
||||||
)
|
)
|
||||||
@@ -434,6 +434,7 @@ def test_evictable_cached_blocks_not_double_allocated():
|
|||||||
num_tokens=2 * block_size,
|
num_tokens=2 * block_size,
|
||||||
new_computed_blocks=[evictable_block],
|
new_computed_blocks=[evictable_block],
|
||||||
total_computed_tokens=block_size,
|
total_computed_tokens=block_size,
|
||||||
|
num_local_computed_tokens=block_size,
|
||||||
num_tokens_main_model=2 * block_size,
|
num_tokens_main_model=2 * block_size,
|
||||||
)
|
)
|
||||||
# Free capacity check should count evictable cached blocks, but allocation
|
# Free capacity check should count evictable cached blocks, but allocation
|
||||||
@@ -474,13 +475,13 @@ def test_chunked_local_attention_get_num_blocks_to_allocate():
|
|||||||
|
|
||||||
assert (
|
assert (
|
||||||
manager.get_num_blocks_to_allocate(
|
manager.get_num_blocks_to_allocate(
|
||||||
"1", 20 * block_size, cached_blocks_1, 0, 20 * block_size
|
"1", 20 * block_size, cached_blocks_1, 0, 0, 20 * block_size
|
||||||
)
|
)
|
||||||
== 20
|
== 20
|
||||||
)
|
)
|
||||||
assert (
|
assert (
|
||||||
manager.get_num_blocks_to_allocate(
|
manager.get_num_blocks_to_allocate(
|
||||||
"2", 20 * block_size, cached_blocks_2, 0, 20 * block_size
|
"2", 20 * block_size, cached_blocks_2, 0, 0, 20 * block_size
|
||||||
)
|
)
|
||||||
== 15
|
== 15
|
||||||
)
|
)
|
||||||
@@ -524,6 +525,7 @@ def test_predictor_matches_allocator_blocks_calculation_with_admission_cap():
|
|||||||
num_tokens=num_tokens,
|
num_tokens=num_tokens,
|
||||||
new_computed_blocks=[],
|
new_computed_blocks=[],
|
||||||
total_computed_tokens=total_computed,
|
total_computed_tokens=total_computed,
|
||||||
|
num_local_computed_tokens=0,
|
||||||
num_tokens_main_model=num_tokens,
|
num_tokens_main_model=num_tokens,
|
||||||
)
|
)
|
||||||
new_blocks = manager.allocate_new_blocks(
|
new_blocks = manager.allocate_new_blocks(
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=Non
|
|||||||
|
|
||||||
|
|
||||||
def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
|
def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
|
||||||
cmap = ExternalCachedBlockPool()
|
cmap = ExternalCachedBlockPool(16)
|
||||||
h = BlockHash(b"\xaa" * 4)
|
h = BlockHash(b"\xaa" * 4)
|
||||||
res = cmap.get_cached_block(h, [0, 1])
|
res = cmap.get_cached_block(h, [0, 1])
|
||||||
assert res is not None
|
assert res is not None
|
||||||
@@ -48,7 +48,7 @@ def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
|
|||||||
|
|
||||||
def test_external_cached_block_pool_hit_all_groups():
|
def test_external_cached_block_pool_hit_all_groups():
|
||||||
h = BlockHash(b"\x11\x22\x33\x44")
|
h = BlockHash(b"\x11\x22\x33\x44")
|
||||||
cmap = ExternalCachedBlockPool({(0, bytes(h)), (1, bytes(h))})
|
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)), (1, bytes(h))})
|
||||||
res = cmap.get_cached_block(h, [0, 1])
|
res = cmap.get_cached_block(h, [0, 1])
|
||||||
assert res is not None
|
assert res is not None
|
||||||
assert len(res) == 2
|
assert len(res) == 2
|
||||||
@@ -58,14 +58,14 @@ def test_external_cached_block_pool_hit_all_groups():
|
|||||||
|
|
||||||
def test_external_cached_block_pool_miss_one_group():
|
def test_external_cached_block_pool_miss_one_group():
|
||||||
h = BlockHash(b"\x11\x22\x33\x44")
|
h = BlockHash(b"\x11\x22\x33\x44")
|
||||||
cmap = ExternalCachedBlockPool({(0, bytes(h))})
|
cmap = ExternalCachedBlockPool(16, {(0, bytes(h))})
|
||||||
assert cmap.get_cached_block(h, [0, 1]) is None
|
assert cmap.get_cached_block(h, [0, 1]) is None
|
||||||
|
|
||||||
|
|
||||||
def test_external_cached_block_pool_unknown_hash():
|
def test_external_cached_block_pool_unknown_hash():
|
||||||
h_known = BlockHash(b"\x01" * 4)
|
h_known = BlockHash(b"\x01" * 4)
|
||||||
h_unknown = BlockHash(b"\x02" * 4)
|
h_unknown = BlockHash(b"\x02" * 4)
|
||||||
cmap = ExternalCachedBlockPool({(0, bytes(h_known))})
|
cmap = ExternalCachedBlockPool(16, {(0, bytes(h_known))})
|
||||||
assert cmap.get_cached_block(h_unknown, [0]) is None
|
assert cmap.get_cached_block(h_unknown, [0]) is None
|
||||||
|
|
||||||
|
|
||||||
@@ -103,7 +103,7 @@ def test_coordinator_single_full_attention_all_hits():
|
|||||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||||
coord = _make_coord(groups, hash_block_size=16)
|
coord = _make_coord(groups, hash_block_size=16)
|
||||||
hs = _hashes(4)
|
hs = _hashes(4)
|
||||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
|
||||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||||
assert hit == 64
|
assert hit == 64
|
||||||
assert masks[0] == [True, True, True, True]
|
assert masks[0] == [True, True, True, True]
|
||||||
@@ -113,7 +113,7 @@ def test_coordinator_single_full_attention_partial_prefix():
|
|||||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||||
coord = _make_coord(groups, hash_block_size=16)
|
coord = _make_coord(groups, hash_block_size=16)
|
||||||
hs = _hashes(4)
|
hs = _hashes(4)
|
||||||
cmap = ExternalCachedBlockPool({(0, bytes(hs[0])), (0, bytes(hs[1]))})
|
cmap = ExternalCachedBlockPool(16, {(0, bytes(hs[0])), (0, bytes(hs[1]))})
|
||||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||||
assert hit == 32
|
assert hit == 32
|
||||||
assert masks[0] == [True, True]
|
assert masks[0] == [True, True]
|
||||||
@@ -123,7 +123,7 @@ def test_coordinator_single_full_attention_no_hits():
|
|||||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||||
coord = _make_coord(groups, hash_block_size=16)
|
coord = _make_coord(groups, hash_block_size=16)
|
||||||
hs = _hashes(4)
|
hs = _hashes(4)
|
||||||
cmap = ExternalCachedBlockPool(set())
|
cmap = ExternalCachedBlockPool(16, set())
|
||||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||||
assert hit == 0
|
assert hit == 0
|
||||||
assert masks[0] == []
|
assert masks[0] == []
|
||||||
@@ -135,7 +135,7 @@ def test_coordinator_single_swa_tautological_pool_masks_pre_window():
|
|||||||
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
|
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
|
||||||
coord = _make_coord(groups, hash_block_size=16)
|
coord = _make_coord(groups, hash_block_size=16)
|
||||||
hs = _hashes(4) # 4 chunks * 16 tokens
|
hs = _hashes(4) # 4 chunks * 16 tokens
|
||||||
cmap = ExternalCachedBlockPool()
|
cmap = ExternalCachedBlockPool(16)
|
||||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||||
assert hit == 64
|
assert hit == 64
|
||||||
# ceil((sw-1)/block_size) = ceil(31/16) = 2 tail blocks.
|
# ceil((sw-1)/block_size) = ceil(31/16) = 2 tail blocks.
|
||||||
@@ -153,7 +153,7 @@ def test_coordinator_hybrid_full_plus_swa_all_hit():
|
|||||||
]
|
]
|
||||||
coord = _make_coord(groups, hash_block_size=16)
|
coord = _make_coord(groups, hash_block_size=16)
|
||||||
hs = _hashes(4)
|
hs = _hashes(4)
|
||||||
cmap = ExternalCachedBlockPool({(g, bytes(h)) for g in (0, 1) for h in hs})
|
cmap = ExternalCachedBlockPool(16, {(g, bytes(h)) for g in (0, 1) for h in hs})
|
||||||
_masks, hit = coord.find_longest_cache_hit(
|
_masks, hit = coord.find_longest_cache_hit(
|
||||||
hs, max_length=64, cached_block_pool=cmap
|
hs, max_length=64, cached_block_pool=cmap
|
||||||
)
|
)
|
||||||
@@ -169,7 +169,7 @@ def test_coordinator_hybrid_hole_in_full_clips_both():
|
|||||||
hs = _hashes(4)
|
hs = _hashes(4)
|
||||||
exists = {(0, bytes(hs[0])), (0, bytes(hs[2])), (0, bytes(hs[3]))}
|
exists = {(0, bytes(hs[0])), (0, bytes(hs[2])), (0, bytes(hs[3]))}
|
||||||
exists |= {(1, bytes(h)) for h in hs}
|
exists |= {(1, bytes(h)) for h in hs}
|
||||||
cmap = ExternalCachedBlockPool(exists)
|
cmap = ExternalCachedBlockPool(16, exists)
|
||||||
_masks, hit = coord.find_longest_cache_hit(
|
_masks, hit = coord.find_longest_cache_hit(
|
||||||
hs, max_length=64, cached_block_pool=cmap
|
hs, max_length=64, cached_block_pool=cmap
|
||||||
)
|
)
|
||||||
@@ -188,7 +188,7 @@ def test_coordinator_group_block_size_double_hash():
|
|||||||
big_hashes = list(chunk_hashes_for_block_size(hs, 16, 32))
|
big_hashes = list(chunk_hashes_for_block_size(hs, 16, 32))
|
||||||
exists = {(0, bytes(h)) for h in hs}
|
exists = {(0, bytes(h)) for h in hs}
|
||||||
exists |= {(1, bytes(bh)) for bh in big_hashes}
|
exists |= {(1, bytes(bh)) for bh in big_hashes}
|
||||||
cmap = ExternalCachedBlockPool(exists)
|
cmap = ExternalCachedBlockPool(16, exists)
|
||||||
_masks, hit = coord.find_longest_cache_hit(
|
_masks, hit = coord.find_longest_cache_hit(
|
||||||
hs, max_length=64, cached_block_pool=cmap
|
hs, max_length=64, cached_block_pool=cmap
|
||||||
)
|
)
|
||||||
@@ -405,7 +405,7 @@ def test_lookup_with_eagle_pops_last_full_attention_block():
|
|||||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
||||||
hs = _hashes(4)
|
hs = _hashes(4)
|
||||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
|
||||||
_masks, hit = coord.find_longest_cache_hit(
|
_masks, hit = coord.find_longest_cache_hit(
|
||||||
hs, max_length=64, cached_block_pool=cmap
|
hs, max_length=64, cached_block_pool=cmap
|
||||||
)
|
)
|
||||||
@@ -426,7 +426,7 @@ def test_load_mask_with_eagle_does_not_double_prune_full_attention():
|
|||||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
||||||
hs = _hashes(4)
|
hs = _hashes(4)
|
||||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
|
||||||
_masks, hit = coord.find_longest_cache_hit(
|
_masks, hit = coord.find_longest_cache_hit(
|
||||||
hs, max_length=64, cached_block_pool=cmap
|
hs, max_length=64, cached_block_pool=cmap
|
||||||
)
|
)
|
||||||
@@ -450,7 +450,7 @@ def test_load_mask_with_eagle_hybrid_full_plus_swa():
|
|||||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
||||||
hs = _hashes(4)
|
hs = _hashes(4)
|
||||||
exists = {(g, bytes(h)) for g in (0, 1) for h in hs}
|
exists = {(g, bytes(h)) for g in (0, 1) for h in hs}
|
||||||
cmap = ExternalCachedBlockPool(exists)
|
cmap = ExternalCachedBlockPool(16, exists)
|
||||||
_masks, hit = coord.find_longest_cache_hit(
|
_masks, hit = coord.find_longest_cache_hit(
|
||||||
hs, max_length=64, cached_block_pool=cmap
|
hs, max_length=64, cached_block_pool=cmap
|
||||||
)
|
)
|
||||||
@@ -469,7 +469,7 @@ def test_load_mask_without_eagle_unchanged():
|
|||||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=False)
|
coord = _make_coord(groups, hash_block_size=16, use_eagle=False)
|
||||||
hs = _hashes(4)
|
hs = _hashes(4)
|
||||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
|
||||||
_masks, hit = coord.find_longest_cache_hit(
|
_masks, hit = coord.find_longest_cache_hit(
|
||||||
hs, max_length=64, cached_block_pool=cmap
|
hs, max_length=64, cached_block_pool=cmap
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ def _minimal_vllm_config(cache_block_size=16):
|
|||||||
cfg.cache_config.block_size = cache_block_size
|
cfg.cache_config.block_size = cache_block_size
|
||||||
cfg.cache_config.num_gpu_blocks = 4
|
cfg.cache_config.num_gpu_blocks = 4
|
||||||
cfg.cache_config.hash_block_size = None
|
cfg.cache_config.hash_block_size = None
|
||||||
|
cfg.cache_config.prefix_match_unit = None
|
||||||
cfg.cache_config.enable_prefix_caching = True
|
cfg.cache_config.enable_prefix_caching = True
|
||||||
cfg.parallel_config.prefill_context_parallel_size = 1
|
cfg.parallel_config.prefill_context_parallel_size = 1
|
||||||
cfg.parallel_config.decode_context_parallel_size = 1
|
cfg.parallel_config.decode_context_parallel_size = 1
|
||||||
|
|||||||
+10
-10
@@ -53,17 +53,17 @@ class CacheConfig:
|
|||||||
"""Whether block_size was explicitly provided. Derived automatically."""
|
"""Whether block_size was explicitly provided. Derived automatically."""
|
||||||
user_specified_mamba_block_size: bool = field(default=False, init=False)
|
user_specified_mamba_block_size: bool = field(default=False, init=False)
|
||||||
"""Whether mamba_block_size was explicitly provided. Derived automatically."""
|
"""Whether mamba_block_size was explicitly provided. Derived automatically."""
|
||||||
hash_block_size: int | None = Field(default=None, gt=0)
|
prefix_match_unit: int | None = Field(default=None, gt=0)
|
||||||
"""Block size (in tokens) used for computing Request's block_hashes.
|
"""The finest token boundary (in tokens) a prefix-cache hit can land on.
|
||||||
|
|
||||||
This can be set to a finer granularity than the physical KV cache block
|
Prefix-cache keys are computed every `prefix_match_unit` tokens. It can
|
||||||
sizes (e.g. 8) as long as every KV cache group's `block_size` is divisible
|
be set finer than the physical KV cache block sizes (e.g. 32 vs a
|
||||||
by it. This enables prefix-caching keys to be computed at the finest common
|
1024-token hybrid-model block) as long as every KV cache group's
|
||||||
granularity and then merged for larger physical block sizes.
|
`block_size` is divisible by it, enabling cache hits at boundaries
|
||||||
|
inside a physical block. It controls matching granularity only, not how
|
||||||
|
often states are stored.
|
||||||
|
|
||||||
This config is not static default. If left unspecified, vLLM will choose a
|
This equals to the `hash_block_size` used throughout the KV cache code.
|
||||||
default based on the resolved KV cache groups (typically the smallest KV
|
|
||||||
cache block size when there are multiple groups).
|
|
||||||
"""
|
"""
|
||||||
gpu_memory_utilization: float = Field(default=0.92, gt=0, le=1)
|
gpu_memory_utilization: float = Field(default=0.92, gt=0, le=1)
|
||||||
"""The fraction of GPU memory to be used for the model executor, which can
|
"""The fraction of GPU memory to be used for the model executor, which can
|
||||||
@@ -209,7 +209,7 @@ class CacheConfig:
|
|||||||
"enable_prefix_caching",
|
"enable_prefix_caching",
|
||||||
"prefix_caching_hash_algo",
|
"prefix_caching_hash_algo",
|
||||||
# Prefix-caching implementation detail (doesn't affect compiled graph).
|
# Prefix-caching implementation detail (doesn't affect compiled graph).
|
||||||
"hash_block_size",
|
"prefix_match_unit",
|
||||||
"mamba_page_size_padded",
|
"mamba_page_size_padded",
|
||||||
"skip_page_size_padded",
|
"skip_page_size_padded",
|
||||||
"user_specified_block_size",
|
"user_specified_block_size",
|
||||||
|
|||||||
@@ -29,10 +29,15 @@ from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
|||||||
class ExternalCachedBlockPool:
|
class ExternalCachedBlockPool:
|
||||||
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
|
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
|
||||||
|
|
||||||
def __init__(self, exists: set[tuple[int, bytes]] | None = None) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
hash_block_size: int,
|
||||||
|
exists: set[tuple[int, bytes]] | None = None,
|
||||||
|
) -> None:
|
||||||
# ``exists=None`` is used on the recv side where hit_length is already
|
# ``exists=None`` is used on the recv side where hit_length is already
|
||||||
# determined and we just want each spec's manager to apply its own mask.
|
# determined and we just want each spec's manager to apply its own mask.
|
||||||
self._exists = exists
|
self._exists = exists
|
||||||
|
self.hash_block_size = hash_block_size
|
||||||
self.null_block = KVCacheBlock(block_id=0)
|
self.null_block = KVCacheBlock(block_id=0)
|
||||||
# Dummy ID 1 for present block for duck-typing.
|
# Dummy ID 1 for present block for duck-typing.
|
||||||
self._present_block = KVCacheBlock(block_id=1)
|
self._present_block = KVCacheBlock(block_id=1)
|
||||||
@@ -166,7 +171,7 @@ class MooncakeStoreCoordinator:
|
|||||||
masks, _ = self.find_longest_cache_hit(
|
masks, _ = self.find_longest_cache_hit(
|
||||||
block_hashes,
|
block_hashes,
|
||||||
token_len,
|
token_len,
|
||||||
ExternalCachedBlockPool(),
|
ExternalCachedBlockPool(self.hash_block_size),
|
||||||
apply_eagle=False,
|
apply_eagle=False,
|
||||||
)
|
)
|
||||||
return masks
|
return masks
|
||||||
@@ -270,7 +275,7 @@ class MooncakeStoreCoordinator:
|
|||||||
if len(self.attention_groups) == 1:
|
if len(self.attention_groups) == 1:
|
||||||
spec, group_ids, manager_cls = self.attention_groups[0]
|
spec, group_ids, manager_cls = self.attention_groups[0]
|
||||||
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
||||||
hit_blocks = manager_cls.find_longest_cache_hit(
|
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
|
||||||
block_hashes=hashes, # type: ignore[arg-type]
|
block_hashes=hashes, # type: ignore[arg-type]
|
||||||
max_length=max_length,
|
max_length=max_length,
|
||||||
kv_cache_group_ids=group_ids,
|
kv_cache_group_ids=group_ids,
|
||||||
@@ -283,11 +288,12 @@ class MooncakeStoreCoordinator:
|
|||||||
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
||||||
for gid, blks in zip(group_ids, hit_blocks, strict=True):
|
for gid, blks in zip(group_ids, hit_blocks, strict=True):
|
||||||
blocks_by_group[gid] = blks
|
blocks_by_group[gid] = blks
|
||||||
return tuple(blocks_by_group), len(hit_blocks[0]) * spec.block_size
|
return tuple(blocks_by_group), hit_length
|
||||||
|
|
||||||
num_groups = len(self.kv_cache_groups)
|
num_groups = len(self.kv_cache_groups)
|
||||||
hit_length = max_length
|
hit_length = max_length
|
||||||
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
|
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
|
||||||
|
hit_length_by_group: list[int] = [0] * num_groups
|
||||||
|
|
||||||
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
|
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
|
||||||
self.attention_groups[0][0], FullAttentionSpec
|
self.attention_groups[0][0], FullAttentionSpec
|
||||||
@@ -298,10 +304,11 @@ class MooncakeStoreCoordinator:
|
|||||||
curr_hit_length = hit_length
|
curr_hit_length = hit_length
|
||||||
|
|
||||||
for idx, (spec, group_ids, manager_cls) in enumerate(self.attention_groups):
|
for idx, (spec, group_ids, manager_cls) in enumerate(self.attention_groups):
|
||||||
cached = hit_blocks_by_group[group_ids[0]]
|
first_group_id = group_ids[0]
|
||||||
|
cached = hit_blocks_by_group[first_group_id]
|
||||||
if isinstance(spec, FullAttentionSpec) and cached is not None:
|
if isinstance(spec, FullAttentionSpec) and cached is not None:
|
||||||
curr_hit_length = (
|
curr_hit_length = min(
|
||||||
curr_hit_length // spec.block_size * spec.block_size
|
curr_hit_length, hit_length_by_group[first_group_id]
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -310,7 +317,7 @@ class MooncakeStoreCoordinator:
|
|||||||
if drop_eagle_block:
|
if drop_eagle_block:
|
||||||
_max_length = min(curr_hit_length + spec.block_size, max_length)
|
_max_length = min(curr_hit_length + spec.block_size, max_length)
|
||||||
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
||||||
hit_blocks = manager_cls.find_longest_cache_hit(
|
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
|
||||||
block_hashes=hashes, # type: ignore[arg-type]
|
block_hashes=hashes, # type: ignore[arg-type]
|
||||||
max_length=_max_length,
|
max_length=_max_length,
|
||||||
kv_cache_group_ids=group_ids,
|
kv_cache_group_ids=group_ids,
|
||||||
@@ -319,7 +326,6 @@ class MooncakeStoreCoordinator:
|
|||||||
drop_eagle_block=drop_eagle_block,
|
drop_eagle_block=drop_eagle_block,
|
||||||
alignment_tokens=self.lcm_block_size,
|
alignment_tokens=self.lcm_block_size,
|
||||||
)
|
)
|
||||||
_new_hit_length = len(hit_blocks[0]) * spec.block_size
|
|
||||||
if drop_eagle_block:
|
if drop_eagle_block:
|
||||||
eagle_verified.add(idx)
|
eagle_verified.add(idx)
|
||||||
elif _new_hit_length < curr_hit_length:
|
elif _new_hit_length < curr_hit_length:
|
||||||
@@ -327,6 +333,7 @@ class MooncakeStoreCoordinator:
|
|||||||
curr_hit_length = _new_hit_length
|
curr_hit_length = _new_hit_length
|
||||||
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
|
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
|
||||||
hit_blocks_by_group[gid] = blocks
|
hit_blocks_by_group[gid] = blocks
|
||||||
|
hit_length_by_group[gid] = _new_hit_length
|
||||||
|
|
||||||
if curr_hit_length >= hit_length:
|
if curr_hit_length >= hit_length:
|
||||||
break
|
break
|
||||||
@@ -343,6 +350,7 @@ class MooncakeStoreCoordinator:
|
|||||||
full_blks = hit_blocks_by_group[gid]
|
full_blks = hit_blocks_by_group[gid]
|
||||||
assert full_blks is not None
|
assert full_blks is not None
|
||||||
del full_blks[num_blocks:]
|
del full_blks[num_blocks:]
|
||||||
|
hit_length_by_group[gid] = hit_length
|
||||||
|
|
||||||
return (
|
return (
|
||||||
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
|
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
|
||||||
|
|||||||
@@ -1523,7 +1523,9 @@ class MooncakeStoreWorker:
|
|||||||
}
|
}
|
||||||
|
|
||||||
_masks, hit_length = self.coord.find_longest_cache_hit(
|
_masks, hit_length = self.coord.find_longest_cache_hit(
|
||||||
block_hashes, token_len, ExternalCachedBlockPool(exists_set)
|
block_hashes,
|
||||||
|
token_len,
|
||||||
|
ExternalCachedBlockPool(self.hash_block_size, exists_set),
|
||||||
)
|
)
|
||||||
return hit_length
|
return hit_length
|
||||||
|
|
||||||
|
|||||||
@@ -693,6 +693,7 @@ class EngineArgs:
|
|||||||
mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype
|
mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype
|
||||||
mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
|
mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
|
||||||
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
|
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
|
||||||
|
prefix_match_unit: int | None = get_field(CacheConfig, "prefix_match_unit")
|
||||||
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
|
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
|
||||||
|
|
||||||
mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON
|
mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON
|
||||||
@@ -1194,6 +1195,9 @@ class EngineArgs:
|
|||||||
cache_group.add_argument(
|
cache_group.add_argument(
|
||||||
"--mamba-block-size", **cache_kwargs["mamba_block_size"]
|
"--mamba-block-size", **cache_kwargs["mamba_block_size"]
|
||||||
)
|
)
|
||||||
|
cache_group.add_argument(
|
||||||
|
"--prefix-match-unit", **cache_kwargs["prefix_match_unit"]
|
||||||
|
)
|
||||||
cache_group.add_argument(
|
cache_group.add_argument(
|
||||||
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
|
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
|
||||||
)
|
)
|
||||||
@@ -1899,6 +1903,7 @@ class EngineArgs:
|
|||||||
mamba_cache_dtype=self.mamba_cache_dtype,
|
mamba_cache_dtype=self.mamba_cache_dtype,
|
||||||
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
|
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
|
||||||
mamba_block_size=self.mamba_block_size,
|
mamba_block_size=self.mamba_block_size,
|
||||||
|
prefix_match_unit=self.prefix_match_unit,
|
||||||
mamba_cache_mode=self.mamba_cache_mode,
|
mamba_cache_mode=self.mamba_cache_mode,
|
||||||
kv_offloading_size=self.kv_offloading_size,
|
kv_offloading_size=self.kv_offloading_size,
|
||||||
kv_offloading_backend=self.kv_offloading_backend,
|
kv_offloading_backend=self.kv_offloading_backend,
|
||||||
|
|||||||
@@ -260,7 +260,9 @@ class BlockPool:
|
|||||||
return
|
return
|
||||||
new_full_blocks = blocks[num_cached_blocks:num_full_blocks]
|
new_full_blocks = blocks[num_cached_blocks:num_full_blocks]
|
||||||
assert block_mask is None or len(block_mask) == len(new_full_blocks)
|
assert block_mask is None or len(block_mask) == len(new_full_blocks)
|
||||||
block_hashes = resolve_block_hashes(request, self.hash_block_size, block_size)
|
block_hashes = resolve_block_hashes(
|
||||||
|
request.block_hashes, self.hash_block_size, block_size
|
||||||
|
)
|
||||||
|
|
||||||
new_block_hashes = block_hashes[num_cached_blocks:]
|
new_block_hashes = block_hashes[num_cached_blocks:]
|
||||||
new_hashes: list[ExternalBlockHash] | None = (
|
new_hashes: list[ExternalBlockHash] | None = (
|
||||||
@@ -390,7 +392,9 @@ class BlockPool:
|
|||||||
if not self.enable_kv_cache_events or num_cached_blocks == 0:
|
if not self.enable_kv_cache_events or num_cached_blocks == 0:
|
||||||
return
|
return
|
||||||
|
|
||||||
block_hashes = resolve_block_hashes(request, self.hash_block_size, block_size)
|
block_hashes = resolve_block_hashes(
|
||||||
|
request.block_hashes, self.hash_block_size, block_size
|
||||||
|
)
|
||||||
|
|
||||||
# Collect external hashes and extra_keys for cached blocks.
|
# Collect external hashes and extra_keys for cached blocks.
|
||||||
cached_hashes: list[ExternalBlockHash] = []
|
cached_hashes: list[ExternalBlockHash] = []
|
||||||
@@ -622,6 +626,24 @@ class BlockPool:
|
|||||||
)
|
)
|
||||||
self.cached_block_hash_to_block.insert(block_hash_with_group_id, block)
|
self.cached_block_hash_to_block.insert(block_hash_with_group_id, block)
|
||||||
|
|
||||||
|
def move_block_hashes(
|
||||||
|
self,
|
||||||
|
src_block: KVCacheBlock,
|
||||||
|
dst_block: KVCacheBlock,
|
||||||
|
) -> None:
|
||||||
|
"""Re-point ``src_block``'s prefix-cache entries to ``dst_block``.
|
||||||
|
|
||||||
|
Used when the request owning ``src_block`` keeps writing into it
|
||||||
|
: the prefix cache holds a private copy (``dst_block``)
|
||||||
|
under the same hashes instead. Entries stay live; no events emitted.
|
||||||
|
"""
|
||||||
|
assert dst_block.block_hash is None
|
||||||
|
assert dst_block.block_id not in self.cached_block_hashes_by_block
|
||||||
|
num_tokens = src_block.block_hash_num_tokens
|
||||||
|
for block_hash in self._remove_cached_block_hashes(src_block):
|
||||||
|
# `num_tokens` only applies to the first (primary) insertion.
|
||||||
|
self._insert_block_hash(block_hash, dst_block, num_tokens=num_tokens)
|
||||||
|
|
||||||
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
|
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
|
||||||
"""Get new blocks from the free block pool.
|
"""Get new blocks from the free block pool.
|
||||||
|
|
||||||
|
|||||||
@@ -5,12 +5,11 @@ from collections.abc import Sequence
|
|||||||
from typing import NamedTuple
|
from typing import NamedTuple
|
||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
|
from vllm.utils.math_utils import cdiv
|
||||||
from vllm.v1.core.block_pool import BlockPool
|
from vllm.v1.core.block_pool import BlockPool
|
||||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||||
from vllm.v1.core.kv_cache_utils import (
|
from vllm.v1.core.kv_cache_utils import (
|
||||||
BlockHash,
|
BlockHash,
|
||||||
BlockHashList,
|
|
||||||
BlockHashListWithBlockSize,
|
|
||||||
KVCacheBlock,
|
KVCacheBlock,
|
||||||
)
|
)
|
||||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||||
@@ -135,6 +134,7 @@ class KVCacheCoordinator(ABC):
|
|||||||
new_computed_blocks: tuple[Sequence[KVCacheBlock], ...],
|
new_computed_blocks: tuple[Sequence[KVCacheBlock], ...],
|
||||||
num_encoder_tokens: int,
|
num_encoder_tokens: int,
|
||||||
total_computed_tokens: int,
|
total_computed_tokens: int,
|
||||||
|
num_local_computed_tokens: int,
|
||||||
num_tokens_main_model: int,
|
num_tokens_main_model: int,
|
||||||
apply_admission_cap: bool = False,
|
apply_admission_cap: bool = False,
|
||||||
) -> int:
|
) -> int:
|
||||||
@@ -150,6 +150,8 @@ class KVCacheCoordinator(ABC):
|
|||||||
num_encoder_tokens: The number of encoder tokens for allocating
|
num_encoder_tokens: The number of encoder tokens for allocating
|
||||||
blocks for cross-attention.
|
blocks for cross-attention.
|
||||||
total_computed_tokens: Include both local and external tokens.
|
total_computed_tokens: Include both local and external tokens.
|
||||||
|
num_local_computed_tokens: The number of local prefix-cache computed
|
||||||
|
tokens.
|
||||||
num_tokens_main_model: The number of tokens for the main model (aka target
|
num_tokens_main_model: The number of tokens for the main model (aka target
|
||||||
model in spec decode). w/o spec decode, it is num_tokens;
|
model in spec decode). w/o spec decode, it is num_tokens;
|
||||||
with spec decode, it is num_tokens - num_lookahead_tokens.
|
with spec decode, it is num_tokens - num_lookahead_tokens.
|
||||||
@@ -171,6 +173,7 @@ class KVCacheCoordinator(ABC):
|
|||||||
num_encoder_tokens,
|
num_encoder_tokens,
|
||||||
[],
|
[],
|
||||||
0,
|
0,
|
||||||
|
0,
|
||||||
num_encoder_tokens,
|
num_encoder_tokens,
|
||||||
apply_admission_cap=apply_admission_cap,
|
apply_admission_cap=apply_admission_cap,
|
||||||
)
|
)
|
||||||
@@ -180,6 +183,7 @@ class KVCacheCoordinator(ABC):
|
|||||||
num_tokens,
|
num_tokens,
|
||||||
new_computed_blocks[i],
|
new_computed_blocks[i],
|
||||||
total_computed_tokens,
|
total_computed_tokens,
|
||||||
|
num_local_computed_tokens,
|
||||||
num_tokens_main_model,
|
num_tokens_main_model,
|
||||||
apply_admission_cap=apply_admission_cap,
|
apply_admission_cap=apply_admission_cap,
|
||||||
)
|
)
|
||||||
@@ -370,7 +374,7 @@ class KVCacheCoordinator(ABC):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
def new_step_starts(self) -> None:
|
def new_step_starts(self) -> None:
|
||||||
"""Called when a new step is started."""
|
"""Notify each manager that a new step is starting."""
|
||||||
for manager in self.single_type_managers:
|
for manager in self.single_type_managers:
|
||||||
manager.new_step_starts()
|
manager.new_step_starts()
|
||||||
|
|
||||||
@@ -483,7 +487,7 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
block_hashes: list[BlockHash],
|
block_hashes: list[BlockHash],
|
||||||
max_cache_hit_length: int,
|
max_cache_hit_length: int,
|
||||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||||
hit_blocks = self.single_type_managers[0].find_longest_cache_hit(
|
hit_blocks, hit_length = self.single_type_managers[0].find_longest_cache_hit(
|
||||||
block_hashes=block_hashes,
|
block_hashes=block_hashes,
|
||||||
max_length=max_cache_hit_length,
|
max_length=max_cache_hit_length,
|
||||||
kv_cache_group_ids=[0],
|
kv_cache_group_ids=[0],
|
||||||
@@ -494,7 +498,7 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
dcp_world_size=self.dcp_world_size,
|
dcp_world_size=self.dcp_world_size,
|
||||||
pcp_world_size=self.pcp_world_size,
|
pcp_world_size=self.pcp_world_size,
|
||||||
)
|
)
|
||||||
return hit_blocks, len(hit_blocks[0]) * self.block_size
|
return hit_blocks, hit_length
|
||||||
|
|
||||||
|
|
||||||
class SpecGroup(NamedTuple):
|
class SpecGroup(NamedTuple):
|
||||||
@@ -572,8 +576,26 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
"full-attention and Mamba groups, got: "
|
"full-attention and Mamba groups, got: "
|
||||||
f"{type(g.kv_cache_spec).__name__}."
|
f"{type(g.kv_cache_spec).__name__}."
|
||||||
)
|
)
|
||||||
|
# Partial hash hits are limited to full-attention + mamba ("align")
|
||||||
|
# without context parallelism.
|
||||||
|
self.enable_partial_hash_hits = dcp_world_size == 1 and any(
|
||||||
|
isinstance(g.kv_cache_spec, MambaSpec)
|
||||||
|
and g.kv_cache_spec.mamba_cache_mode == "align"
|
||||||
|
and g.kv_cache_spec.block_size > hash_block_size
|
||||||
|
for g in kv_cache_config.kv_cache_groups
|
||||||
|
)
|
||||||
self.verify_and_split_kv_cache_groups()
|
self.verify_and_split_kv_cache_groups()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _cache_hit_alignment_tokens(self) -> int:
|
||||||
|
# Fine-grained partial hits may return hash-block-aligned lengths;
|
||||||
|
# otherwise it must stay scheduler-block-aligned.
|
||||||
|
return (
|
||||||
|
self.hash_block_size
|
||||||
|
if self.enable_partial_hash_hits
|
||||||
|
else self.scheduler_block_size
|
||||||
|
)
|
||||||
|
|
||||||
def verify_and_split_kv_cache_groups(self) -> None:
|
def verify_and_split_kv_cache_groups(self) -> None:
|
||||||
"""
|
"""
|
||||||
Groups KV cache groups by their spec type for efficient batch processing
|
Groups KV cache groups by their spec type for efficient batch processing
|
||||||
@@ -617,14 +639,19 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
self.single_type_managers[gid].use_eagle = True
|
self.single_type_managers[gid].use_eagle = True
|
||||||
|
|
||||||
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
|
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
|
||||||
# Cache hits in this coordinator are always a multiple of
|
if self.enable_partial_hash_hits:
|
||||||
# ``scheduler_block_size`` tokens (see ``find_longest_cache_hit``).
|
aligned_num_computed_tokens = num_computed_tokens
|
||||||
# Within an aligned region, SWA groups may only consult a subset of blocks
|
else:
|
||||||
# per ``scheduler_block_size``-segment so the unused blocks also stay
|
# Cache hits in this coordinator are always a multiple of
|
||||||
# out of the prefix-cache hash map.
|
# ``scheduler_block_size`` tokens (see ``find_longest_cache_hit``).
|
||||||
aligned_num_computed_tokens = (
|
# Within an aligned region, SWA groups may only consult a subset of
|
||||||
num_computed_tokens // self.scheduler_block_size * self.scheduler_block_size
|
# blocks per ``scheduler_block_size``-segment so the unused blocks
|
||||||
)
|
# also stay out of the prefix-cache hash map.
|
||||||
|
aligned_num_computed_tokens = (
|
||||||
|
num_computed_tokens
|
||||||
|
// self.scheduler_block_size
|
||||||
|
* self.scheduler_block_size
|
||||||
|
)
|
||||||
for manager in self.single_type_managers:
|
for manager in self.single_type_managers:
|
||||||
num_tokens_to_cache = aligned_num_computed_tokens
|
num_tokens_to_cache = aligned_num_computed_tokens
|
||||||
# EAGLE groups match one block past each aligned boundary and drop
|
# EAGLE groups match one block past each aligned boundary and drop
|
||||||
@@ -667,17 +694,11 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
- The number of tokens of the longest cache hit.
|
- The number of tokens of the longest cache hit.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def _get_block_hashes(block_size: int) -> BlockHashList:
|
|
||||||
if block_size == self.hash_block_size:
|
|
||||||
return block_hashes
|
|
||||||
return BlockHashListWithBlockSize(
|
|
||||||
block_hashes, self.hash_block_size, block_size
|
|
||||||
)
|
|
||||||
|
|
||||||
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
||||||
hit_length = max_cache_hit_length
|
hit_length = max_cache_hit_length
|
||||||
longest_hit_length = 0
|
longest_hit_length = 0
|
||||||
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
|
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
|
||||||
|
hit_length_by_group: list[int] = [0] * num_groups
|
||||||
|
|
||||||
# Simple hybrid (1 full attn + 1 other): one iteration suffices.
|
# Simple hybrid (1 full attn + 1 other): one iteration suffices.
|
||||||
# Full attn is always first if it exists.
|
# Full attn is always first if it exists.
|
||||||
@@ -696,40 +717,53 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
for idx, (spec, group_ids, manager_cls, use_eagle) in enumerate(
|
for idx, (spec, group_ids, manager_cls, use_eagle) in enumerate(
|
||||||
self.attention_groups
|
self.attention_groups
|
||||||
):
|
):
|
||||||
group_block_size = self.single_type_managers[group_ids[0]].block_size
|
first_group_id = group_ids[0]
|
||||||
cached_blocks = hit_blocks_by_group[group_ids[0]]
|
# DCP/PCP shard each block's KV across ranks, so the manager's
|
||||||
|
# effective block size may exceed the spec's.
|
||||||
|
group_block_size = self.single_type_managers[first_group_id].block_size
|
||||||
|
cached_blocks = hit_blocks_by_group[first_group_id]
|
||||||
if isinstance(spec, FullAttentionSpec) and cached_blocks is not None:
|
if isinstance(spec, FullAttentionSpec) and cached_blocks is not None:
|
||||||
# Full attention is downward-closed: we only need to look
|
# Full attention is downward-closed: we only need to look
|
||||||
# up cached blocks once; on subsequent iterations just trim
|
# up cached blocks once; on subsequent iterations just trim
|
||||||
# to the (reduced) current hit length.
|
# to the (reduced) current hit length.
|
||||||
curr_hit_length = (
|
curr_hit_length = min(
|
||||||
curr_hit_length // group_block_size * group_block_size
|
curr_hit_length, hit_length_by_group[first_group_id]
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
drop_eagle_block = use_eagle and idx not in eagle_verified
|
drop_eagle_block = use_eagle and idx not in eagle_verified
|
||||||
|
|
||||||
_max_length = curr_hit_length
|
_max_length = curr_hit_length
|
||||||
if drop_eagle_block:
|
# Eagle matches one extra drop unit (one hash unit for
|
||||||
# Eagle needs to match one more block and then pop the last.
|
# fine-grained managers, else one cache block) and then drops
|
||||||
_max_length = min(
|
# it, landing back at the candidate length. No margin for
|
||||||
curr_hit_length + group_block_size, max_cache_hit_length
|
# mamba: its finder never drops (draft models have no mamba
|
||||||
|
# layers), so the hit would grow past the candidate.
|
||||||
|
if drop_eagle_block and not isinstance(spec, MambaSpec):
|
||||||
|
eagle_margin = (
|
||||||
|
self.hash_block_size
|
||||||
|
if self.enable_partial_hash_hits
|
||||||
|
and manager_cls.supports_fine_grained_hash_lookup
|
||||||
|
and group_block_size > self.hash_block_size
|
||||||
|
else group_block_size
|
||||||
)
|
)
|
||||||
hit_blocks = manager_cls.find_longest_cache_hit(
|
_max_length = min(
|
||||||
block_hashes=_get_block_hashes(group_block_size),
|
curr_hit_length + eagle_margin, max_cache_hit_length
|
||||||
|
)
|
||||||
|
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
|
||||||
|
block_hashes=block_hashes,
|
||||||
max_length=_max_length,
|
max_length=_max_length,
|
||||||
kv_cache_group_ids=group_ids,
|
kv_cache_group_ids=group_ids,
|
||||||
block_pool=self.block_pool,
|
block_pool=self.block_pool,
|
||||||
kv_cache_spec=spec,
|
kv_cache_spec=spec,
|
||||||
drop_eagle_block=drop_eagle_block,
|
drop_eagle_block=drop_eagle_block,
|
||||||
alignment_tokens=self.scheduler_block_size,
|
alignment_tokens=self._cache_hit_alignment_tokens,
|
||||||
dcp_world_size=(
|
dcp_world_size=(
|
||||||
self.dcp_world_size
|
self.dcp_world_size
|
||||||
if isinstance(spec, FullAttentionSpec)
|
if isinstance(spec, FullAttentionSpec)
|
||||||
else 1
|
else 1
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
_new_hit_length = len(hit_blocks[0]) * group_block_size
|
|
||||||
if drop_eagle_block:
|
if drop_eagle_block:
|
||||||
eagle_verified.add(idx)
|
eagle_verified.add(idx)
|
||||||
elif _new_hit_length < curr_hit_length:
|
elif _new_hit_length < curr_hit_length:
|
||||||
@@ -738,6 +772,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
curr_hit_length = _new_hit_length
|
curr_hit_length = _new_hit_length
|
||||||
for group_id, blocks in zip(group_ids, hit_blocks):
|
for group_id, blocks in zip(group_ids, hit_blocks):
|
||||||
hit_blocks_by_group[group_id] = blocks
|
hit_blocks_by_group[group_id] = blocks
|
||||||
|
hit_length_by_group[group_id] = _new_hit_length
|
||||||
|
|
||||||
longest_hit_length = max(longest_hit_length, curr_hit_length)
|
longest_hit_length = max(longest_hit_length, curr_hit_length)
|
||||||
|
|
||||||
@@ -753,10 +788,11 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
group_block_size = self.single_type_managers[
|
group_block_size = self.single_type_managers[
|
||||||
first_group.group_ids[0]
|
first_group.group_ids[0]
|
||||||
].block_size
|
].block_size
|
||||||
num_blocks = hit_length // group_block_size
|
num_blocks = cdiv(hit_length, group_block_size)
|
||||||
for group_id in first_group.group_ids:
|
for group_id in first_group.group_ids:
|
||||||
if (blks := hit_blocks_by_group[group_id]) is not None:
|
if (blks := hit_blocks_by_group[group_id]) is not None:
|
||||||
del blks[num_blocks:]
|
del blks[num_blocks:]
|
||||||
|
hit_length_by_group[group_id] = hit_length
|
||||||
|
|
||||||
# Uncached shared prefix detection: If any attn. group cached a longer prefix
|
# Uncached shared prefix detection: If any attn. group cached a longer prefix
|
||||||
# than the current prefix, it is an uncached common prefix across requests:
|
# than the current prefix, it is an uncached common prefix across requests:
|
||||||
@@ -776,28 +812,20 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
(blocks_per_group, hit_lengths_per_group)
|
(blocks_per_group, hit_lengths_per_group)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def _get_block_hashes(kv_cache_spec: KVCacheSpec) -> BlockHashList:
|
|
||||||
if kv_cache_spec.block_size == self.hash_block_size:
|
|
||||||
return block_hashes
|
|
||||||
return BlockHashListWithBlockSize(
|
|
||||||
block_hashes, self.hash_block_size, kv_cache_spec.block_size
|
|
||||||
)
|
|
||||||
|
|
||||||
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
||||||
hit_blocks: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
hit_blocks: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
||||||
hit_lengths: list[int] = [0] * num_groups
|
hit_lengths: list[int] = [0] * num_groups
|
||||||
|
|
||||||
for spec, group_ids, manager_cls, use_eagle in self.attention_groups:
|
for spec, group_ids, manager_cls, use_eagle in self.attention_groups:
|
||||||
blocks = manager_cls.find_longest_cache_hit(
|
blocks, group_hit = manager_cls.find_longest_cache_hit(
|
||||||
block_hashes=_get_block_hashes(spec),
|
block_hashes=block_hashes,
|
||||||
max_length=max_cache_hit_length,
|
max_length=max_cache_hit_length,
|
||||||
kv_cache_group_ids=group_ids,
|
kv_cache_group_ids=group_ids,
|
||||||
block_pool=self.block_pool,
|
block_pool=self.block_pool,
|
||||||
kv_cache_spec=spec,
|
kv_cache_spec=spec,
|
||||||
drop_eagle_block=use_eagle,
|
drop_eagle_block=use_eagle,
|
||||||
alignment_tokens=self.scheduler_block_size,
|
alignment_tokens=self._cache_hit_alignment_tokens,
|
||||||
)
|
)
|
||||||
group_hit = len(blocks[0]) * spec.block_size
|
|
||||||
for gid, blks in zip(group_ids, blocks):
|
for gid, blks in zip(group_ids, blocks):
|
||||||
hit_blocks[gid] = blks
|
hit_blocks[gid] = blks
|
||||||
hit_lengths[gid] = group_hit
|
hit_lengths[gid] = group_hit
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from vllm.logger import init_logger
|
|||||||
from vllm.utils.math_utils import cdiv
|
from vllm.utils.math_utils import cdiv
|
||||||
from vllm.v1.core.kv_cache_coordinator import get_kv_cache_coordinator
|
from vllm.v1.core.kv_cache_coordinator import get_kv_cache_coordinator
|
||||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy
|
||||||
from vllm.v1.kv_cache_interface import (
|
from vllm.v1.kv_cache_interface import (
|
||||||
AttentionSpec,
|
AttentionSpec,
|
||||||
CrossAttentionSpec,
|
CrossAttentionSpec,
|
||||||
@@ -404,6 +404,7 @@ class KVCacheManager:
|
|||||||
new_computed_blocks=new_computed_block_list,
|
new_computed_blocks=new_computed_block_list,
|
||||||
num_encoder_tokens=num_encoder_tokens,
|
num_encoder_tokens=num_encoder_tokens,
|
||||||
total_computed_tokens=total_computed_tokens,
|
total_computed_tokens=total_computed_tokens,
|
||||||
|
num_local_computed_tokens=num_local_computed_tokens,
|
||||||
num_tokens_main_model=full_num_tokens,
|
num_tokens_main_model=full_num_tokens,
|
||||||
apply_admission_cap=True,
|
apply_admission_cap=True,
|
||||||
)
|
)
|
||||||
@@ -438,6 +439,7 @@ class KVCacheManager:
|
|||||||
num_encoder_tokens=num_encoder_tokens,
|
num_encoder_tokens=num_encoder_tokens,
|
||||||
total_computed_tokens=num_local_computed_tokens
|
total_computed_tokens=num_local_computed_tokens
|
||||||
+ num_external_computed_tokens,
|
+ num_external_computed_tokens,
|
||||||
|
num_local_computed_tokens=num_local_computed_tokens,
|
||||||
num_tokens_main_model=num_tokens_main_model,
|
num_tokens_main_model=num_tokens_main_model,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -665,6 +667,23 @@ class KVCacheManager:
|
|||||||
ids.extend(mgr.take_new_block_ids())
|
ids.extend(mgr.take_new_block_ids())
|
||||||
return ids
|
return ids
|
||||||
|
|
||||||
|
def take_kv_cache_block_copies(
|
||||||
|
self,
|
||||||
|
) -> tuple[list[KVCacheBlockCopy], list[KVCacheBlock]]:
|
||||||
|
"""Drain pending copies and return their retained endpoints."""
|
||||||
|
pending_copies: list[tuple[KVCacheBlock, KVCacheBlock]] = []
|
||||||
|
for mgr in self.coordinator.single_type_managers:
|
||||||
|
pending_copies.extend(mgr.take_pending_cow_copies())
|
||||||
|
copies = [
|
||||||
|
KVCacheBlockCopy(
|
||||||
|
src_block_id=source_block.block_id,
|
||||||
|
dst_block_id=cow_block.block_id,
|
||||||
|
)
|
||||||
|
for source_block, cow_block in pending_copies
|
||||||
|
]
|
||||||
|
retained_blocks = [block for pair in pending_copies for block in pair]
|
||||||
|
return copies, retained_blocks
|
||||||
|
|
||||||
def new_step_starts(self) -> None:
|
def new_step_starts(self) -> None:
|
||||||
"""Called when a new step is started."""
|
"""Notify the coordinator that a new step is starting."""
|
||||||
self.coordinator.new_step_starts()
|
self.coordinator.new_step_starts()
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from collections import defaultdict
|
|||||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||||
from dataclasses import dataclass, replace
|
from dataclasses import dataclass, replace
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Any, NewType, TypeAlias, cast, overload
|
from typing import Any, NamedTuple, NewType, TypeAlias, cast, overload
|
||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
@@ -126,7 +126,7 @@ class KVCacheBlock:
|
|||||||
# when the block is full and cached.
|
# when the block is full and cached.
|
||||||
_block_hash: BlockHashWithGroupId | None = None
|
_block_hash: BlockHashWithGroupId | None = None
|
||||||
# Number of prefix tokens covered by _block_hash. For full blocks this is
|
# Number of prefix tokens covered by _block_hash. For full blocks this is
|
||||||
# the full block boundary; partial aliases can end inside a cache block.
|
# the full block boundary; partial entries can end inside a cache block.
|
||||||
_block_hash_num_tokens: int | None = None
|
_block_hash_num_tokens: int | None = None
|
||||||
|
|
||||||
# Used to construct a doubly linked list for free blocks.
|
# Used to construct a doubly linked list for free blocks.
|
||||||
@@ -176,6 +176,11 @@ class KVCacheBlock:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class KVCacheBlockCopy(NamedTuple):
|
||||||
|
src_block_id: int
|
||||||
|
dst_block_id: int
|
||||||
|
|
||||||
|
|
||||||
class FreeKVCacheBlockQueue:
|
class FreeKVCacheBlockQueue:
|
||||||
"""This class organizes a list of KVCacheBlock objects to a doubly linked
|
"""This class organizes a list of KVCacheBlock objects to a doubly linked
|
||||||
list of free blocks. We implement this class instead of using Python
|
list of free blocks. We implement this class instead of using Python
|
||||||
@@ -631,11 +636,11 @@ def resolve_kv_cache_block_sizes(
|
|||||||
Mamba groups keep their full per-rank state and are not scaled.
|
Mamba groups keep their full per-rank state and are not scaled.
|
||||||
- ``hash_block_size`` is the granularity at which ``Request.block_hashes``
|
- ``hash_block_size`` is the granularity at which ``Request.block_hashes``
|
||||||
is computed. Single group: equals scheduler block size. Multiple groups:
|
is computed. Single group: equals scheduler block size. Multiple groups:
|
||||||
``cache_config.hash_block_size`` override if set, else the GCD of group
|
``cache_config.prefix_match_unit`` override if set, else the GCD of
|
||||||
block sizes; every group's block size must be divisible by it. Returns
|
group block sizes; every group's block size must be divisible by it.
|
||||||
the scheduler block size (i.e. disables finer hashing) if block hashing
|
Returns the scheduler block size (i.e. disables finer hashing) if block
|
||||||
is inactive or a mamba group's block size diverges from the cache
|
hashing is inactive or a mamba group's block size diverges from the
|
||||||
block size (mamba_cache_mode != "align").
|
cache block size (mamba_cache_mode != "align").
|
||||||
"""
|
"""
|
||||||
cache_config = vllm_config.cache_config
|
cache_config = vllm_config.cache_config
|
||||||
dcp = vllm_config.parallel_config.decode_context_parallel_size
|
dcp = vllm_config.parallel_config.decode_context_parallel_size
|
||||||
@@ -671,14 +676,14 @@ def resolve_kv_cache_block_sizes(
|
|||||||
):
|
):
|
||||||
return scheduler_block_size, scheduler_block_size
|
return scheduler_block_size, scheduler_block_size
|
||||||
|
|
||||||
requested = cache_config.hash_block_size
|
requested = cache_config.prefix_match_unit
|
||||||
hash_block_size = (
|
hash_block_size = (
|
||||||
requested if requested is not None else math.gcd(*group_block_sizes)
|
requested if requested is not None else math.gcd(*group_block_sizes)
|
||||||
)
|
)
|
||||||
if any(bs % hash_block_size != 0 for bs in group_block_sizes):
|
if any(bs % hash_block_size != 0 for bs in group_block_sizes):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Invalid hash_block_size={hash_block_size}; all KV cache group "
|
f"Invalid prefix_match_unit={hash_block_size}; all KV cache group "
|
||||||
f"block sizes must be divisible by hash_block_size. "
|
f"block sizes must be divisible by prefix_match_unit. "
|
||||||
f"Got group block sizes={group_block_sizes}."
|
f"Got group block sizes={group_block_sizes}."
|
||||||
)
|
)
|
||||||
return scheduler_block_size, hash_block_size
|
return scheduler_block_size, hash_block_size
|
||||||
@@ -2252,19 +2257,33 @@ BlockHashList = list[BlockHash] | BlockHashListWithBlockSize
|
|||||||
|
|
||||||
|
|
||||||
def resolve_block_hashes(
|
def resolve_block_hashes(
|
||||||
request: Request,
|
block_hashes: BlockHashList,
|
||||||
hash_block_size: int,
|
hash_block_size: int,
|
||||||
block_size: int,
|
block_size: int,
|
||||||
|
*,
|
||||||
|
supports_fine_grained_hash_lookup: bool = False,
|
||||||
|
alignment_tokens: int | None = None,
|
||||||
) -> BlockHashList:
|
) -> BlockHashList:
|
||||||
"""Resolve the block-hash view for ``request`` at ``block_size``.
|
"""Resolve the block-hash view at ``block_size``.
|
||||||
|
|
||||||
When ``block_size`` equals ``hash_block_size``, reuse the request's
|
When ``block_size`` equals ``hash_block_size``, reuse the precomputed block
|
||||||
precomputed ``block_hashes`` directly; otherwise recalculate at
|
hashes directly; otherwise view them at ``block_size`` granularity.
|
||||||
``block_size`` granularity (``block_size`` must be a multiple of
|
Fine-grained lookup keeps the original hashes for partial cache hits.
|
||||||
``hash_block_size``, which happens when KV cache groups differ in
|
|
||||||
block size).
|
|
||||||
"""
|
"""
|
||||||
if block_size == hash_block_size:
|
if block_size == hash_block_size:
|
||||||
return request.block_hashes
|
return block_hashes
|
||||||
|
if isinstance(block_hashes, BlockHashListWithBlockSize):
|
||||||
|
# Already a block-size view
|
||||||
|
assert block_hashes.scale_factor == block_size // hash_block_size
|
||||||
|
return block_hashes
|
||||||
|
# Fine-grained partial hits keep the raw hashes. The caller passes
|
||||||
|
# alignment_tokens = hash_block_size to enable them, else >= block_size.
|
||||||
|
if (
|
||||||
|
supports_fine_grained_hash_lookup
|
||||||
|
and alignment_tokens is not None
|
||||||
|
and alignment_tokens < block_size
|
||||||
|
and block_size % alignment_tokens == 0
|
||||||
|
):
|
||||||
|
return block_hashes
|
||||||
assert block_size % hash_block_size == 0
|
assert block_size % hash_block_size == 0
|
||||||
return BlockHashListWithBlockSize(request.block_hashes, hash_block_size, block_size)
|
return BlockHashListWithBlockSize(block_hashes, hash_block_size, block_size)
|
||||||
|
|||||||
@@ -16,10 +16,12 @@ if TYPE_CHECKING:
|
|||||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||||
from vllm.pooling_params import PoolingParams
|
from vllm.pooling_params import PoolingParams
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||||
from vllm.v1.request import Request
|
from vllm.v1.request import Request
|
||||||
else:
|
else:
|
||||||
ECConnectorMetadata = object
|
ECConnectorMetadata = object
|
||||||
KVConnectorMetadata = object
|
KVConnectorMetadata = object
|
||||||
|
KVCacheBlockCopy = object
|
||||||
LoRARequest = object
|
LoRARequest = object
|
||||||
MultiModalFeatureSpec = object
|
MultiModalFeatureSpec = object
|
||||||
PoolingParams = object
|
PoolingParams = object
|
||||||
@@ -240,6 +242,9 @@ class SchedulerOutput:
|
|||||||
# preventing stale NaN/data from corrupting attention or SSM computation.
|
# preventing stale NaN/data from corrupting attention or SSM computation.
|
||||||
new_block_ids_to_zero: list[int] | None = None
|
new_block_ids_to_zero: list[int] | None = None
|
||||||
|
|
||||||
|
# CoW copies to apply after zeroing new blocks and before forward.
|
||||||
|
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
|
||||||
|
|
||||||
# Dynamic speculative decoding: optimal K chosen by scheduler.
|
# Dynamic speculative decoding: optimal K chosen by scheduler.
|
||||||
# Number of spec tokens to schedule for the next step.
|
# Number of spec tokens to schedule for the next step.
|
||||||
num_spec_tokens_to_schedule: int = 0
|
num_spec_tokens_to_schedule: int = 0
|
||||||
|
|||||||
@@ -259,6 +259,7 @@ class Scheduler(SchedulerInterface):
|
|||||||
# Create the KV cache manager.
|
# Create the KV cache manager.
|
||||||
if hash_block_size is None:
|
if hash_block_size is None:
|
||||||
hash_block_size = block_size
|
hash_block_size = block_size
|
||||||
|
self.hash_block_size = hash_block_size
|
||||||
self.kv_cache_manager = KVCacheManager(
|
self.kv_cache_manager = KVCacheManager(
|
||||||
kv_cache_config=kv_cache_config,
|
kv_cache_config=kv_cache_config,
|
||||||
max_model_len=self.max_model_len,
|
max_model_len=self.max_model_len,
|
||||||
@@ -297,6 +298,13 @@ class Scheduler(SchedulerInterface):
|
|||||||
self.need_mamba_block_aligned_split = (
|
self.need_mamba_block_aligned_split = (
|
||||||
self.has_mamba_layers and self.cache_config.mamba_cache_mode == "align"
|
self.has_mamba_layers and self.cache_config.mamba_cache_mode == "align"
|
||||||
)
|
)
|
||||||
|
# A finer prefix_match_unit is configured: a mamba partial tail entry
|
||||||
|
# can only be registered by a step ending exactly at the prompt's last
|
||||||
|
# hash boundary, so the split adds that stop.
|
||||||
|
self.mamba_partial_cache_hit = (
|
||||||
|
self.need_mamba_block_aligned_split
|
||||||
|
and self.hash_block_size < self.block_size
|
||||||
|
)
|
||||||
|
|
||||||
# Counts of non-empty steps scheduled / processed. update_from_output
|
# Counts of non-empty steps scheduled / processed. update_from_output
|
||||||
# is called once per scheduled step in FIFO order, so these stay in sync.
|
# is called once per scheduled step in FIFO order, so these stay in sync.
|
||||||
@@ -368,9 +376,23 @@ class Scheduler(SchedulerInterface):
|
|||||||
if self.use_eagle:
|
if self.use_eagle:
|
||||||
last_cache_position = max(last_cache_position - block_size, 0)
|
last_cache_position = max(last_cache_position - block_size, 0)
|
||||||
num_computed_tokens_after_sched = num_computed_tokens + num_new_tokens
|
num_computed_tokens_after_sched = num_computed_tokens + num_new_tokens
|
||||||
if num_computed_tokens_after_sched < last_cache_position:
|
next_boundary = (num_computed_tokens // block_size + 1) * block_size
|
||||||
# align to block_size
|
if (
|
||||||
num_new_tokens = num_new_tokens // block_size * block_size
|
num_computed_tokens % block_size != 0
|
||||||
|
and next_boundary <= last_cache_position
|
||||||
|
and num_computed_tokens_after_sched > next_boundary
|
||||||
|
):
|
||||||
|
# Resumed mid-block (partial hash hit): stop the first chunk
|
||||||
|
# at the next block boundary so chunk ends re-align to blocks
|
||||||
|
# and the boundary state is materialized before running past.
|
||||||
|
num_new_tokens = next_boundary - num_computed_tokens
|
||||||
|
elif num_computed_tokens_after_sched < last_cache_position:
|
||||||
|
# Align the chunk END (not its length) to block_size;
|
||||||
|
# identical to flooring the length when the start is
|
||||||
|
# block-aligned. May yield 0 (insufficient budget to reach
|
||||||
|
# the next boundary); the caller then skips the request.
|
||||||
|
aligned_end = num_computed_tokens_after_sched // block_size * block_size
|
||||||
|
num_new_tokens = max(aligned_end - num_computed_tokens, 0)
|
||||||
elif (
|
elif (
|
||||||
num_computed_tokens
|
num_computed_tokens
|
||||||
< last_cache_position
|
< last_cache_position
|
||||||
@@ -378,9 +400,23 @@ class Scheduler(SchedulerInterface):
|
|||||||
):
|
):
|
||||||
# force to cache the last chunk
|
# force to cache the last chunk
|
||||||
num_new_tokens = last_cache_position - num_computed_tokens
|
num_new_tokens = last_cache_position - num_computed_tokens
|
||||||
else:
|
elif self.mamba_partial_cache_hit:
|
||||||
# prefill the last few tokens
|
# Prefill of the final partial block: stop once at the
|
||||||
pass
|
# prompt's last hash boundary so the mamba partial tail entry
|
||||||
|
# can be registered.
|
||||||
|
tail_boundary = (
|
||||||
|
request.num_prompt_tokens
|
||||||
|
// self.hash_block_size
|
||||||
|
* self.hash_block_size
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
num_computed_tokens
|
||||||
|
< tail_boundary
|
||||||
|
< num_computed_tokens_after_sched
|
||||||
|
and tail_boundary < request.num_prompt_tokens
|
||||||
|
and tail_boundary > last_cache_position
|
||||||
|
):
|
||||||
|
num_new_tokens = tail_boundary - num_computed_tokens
|
||||||
|
|
||||||
# Marconi cache admission optimization:
|
# Marconi cache admission optimization:
|
||||||
# cache common prefixes by scheduling num_new_tokens = common prefix length
|
# cache common prefixes by scheduling num_new_tokens = common prefix length
|
||||||
@@ -1084,6 +1120,16 @@ class Scheduler(SchedulerInterface):
|
|||||||
new_block_ids_to_zero = (
|
new_block_ids_to_zero = (
|
||||||
(new_attn_block_ids or None) if self.needs_kv_cache_zeroing else None
|
(new_attn_block_ids or None) if self.needs_kv_cache_zeroing else None
|
||||||
)
|
)
|
||||||
|
kv_cache_block_copies, cow_retained_blocks = (
|
||||||
|
self.kv_cache_manager.take_kv_cache_block_copies()
|
||||||
|
)
|
||||||
|
if kv_cache_block_copies:
|
||||||
|
# The copies run with this step's execution; the first non-empty
|
||||||
|
# step at or after it gets seq `sched_step_seq + 1` (0-token steps
|
||||||
|
# do not advance the seq), and its completion implies the copies
|
||||||
|
# have run.
|
||||||
|
self._free_cow_retained_blocks(cow_retained_blocks, self.sched_step_seq + 1)
|
||||||
|
pending_kv_cache_block_copies = kv_cache_block_copies or None
|
||||||
|
|
||||||
# Dynamic speculative decoding: compute optimal K
|
# Dynamic speculative decoding: compute optimal K
|
||||||
num_spec_tokens_to_schedule = self.num_spec_tokens
|
num_spec_tokens_to_schedule = self.num_spec_tokens
|
||||||
@@ -1108,6 +1154,7 @@ class Scheduler(SchedulerInterface):
|
|||||||
finished_req_ids=self.finished_req_ids,
|
finished_req_ids=self.finished_req_ids,
|
||||||
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
|
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
|
||||||
new_block_ids_to_zero=new_block_ids_to_zero,
|
new_block_ids_to_zero=new_block_ids_to_zero,
|
||||||
|
kv_cache_block_copies=pending_kv_cache_block_copies,
|
||||||
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
|
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2148,11 +2195,23 @@ class Scheduler(SchedulerInterface):
|
|||||||
if blocks:
|
if blocks:
|
||||||
self.deferred_frees.append((self.sched_step_seq, blocks))
|
self.deferred_frees.append((self.sched_step_seq, blocks))
|
||||||
|
|
||||||
|
def _free_cow_retained_blocks(
|
||||||
|
self, blocks: list[KVCacheBlock], fence_seq: int
|
||||||
|
) -> None:
|
||||||
|
"""Release CoW copy retentions, deferring their return to the block
|
||||||
|
pool while the step that runs the copy may still be in flight.
|
||||||
|
"""
|
||||||
|
if not self.defer_block_free or fence_seq <= self.processed_step_seq:
|
||||||
|
self.kv_cache_manager.block_pool.free_blocks(blocks)
|
||||||
|
return
|
||||||
|
self.deferred_frees.append((fence_seq, blocks[::-1]))
|
||||||
|
|
||||||
def _drain_deferred_frees(self):
|
def _drain_deferred_frees(self):
|
||||||
"""Return deferred blocks whose fence step has completed.
|
"""Return deferred blocks whose fence step has completed.
|
||||||
|
|
||||||
Entries are appended with monotonically non-decreasing fences, so
|
Fences are appended in near-monotonic order (a CoW retention fence
|
||||||
stop at the first one that is still pending.
|
can lead request-free fences by one step), so stop at the first
|
||||||
|
pending one; any satisfied entry behind it is merely freed later.
|
||||||
"""
|
"""
|
||||||
while self.deferred_frees:
|
while self.deferred_frees:
|
||||||
fence, _ = self.deferred_frees[0]
|
fence, _ = self.deferred_frees[0]
|
||||||
@@ -2401,6 +2460,7 @@ class Scheduler(SchedulerInterface):
|
|||||||
new_computed_blocks=self.kv_cache_manager.empty_kv_cache_blocks.blocks,
|
new_computed_blocks=self.kv_cache_manager.empty_kv_cache_blocks.blocks,
|
||||||
num_encoder_tokens=0,
|
num_encoder_tokens=0,
|
||||||
total_computed_tokens=request.num_computed_tokens,
|
total_computed_tokens=request.num_computed_tokens,
|
||||||
|
num_local_computed_tokens=request.num_computed_tokens,
|
||||||
num_tokens_main_model=full_num_tokens,
|
num_tokens_main_model=full_num_tokens,
|
||||||
apply_admission_cap=True,
|
apply_admission_cap=True,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -4,13 +4,16 @@ import itertools
|
|||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
from typing import ClassVar
|
||||||
|
|
||||||
from vllm.utils.math_utils import cdiv
|
from vllm.utils.math_utils import cdiv
|
||||||
from vllm.v1.core.block_pool import BlockPool
|
from vllm.v1.core.block_pool import BlockPool
|
||||||
from vllm.v1.core.kv_cache_utils import (
|
from vllm.v1.core.kv_cache_utils import (
|
||||||
BlockHashList,
|
BlockHashList,
|
||||||
|
BlockHashListWithBlockSize,
|
||||||
BlockHashWithGroupId,
|
BlockHashWithGroupId,
|
||||||
KVCacheBlock,
|
KVCacheBlock,
|
||||||
|
resolve_block_hashes,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_cache_interface import (
|
from vllm.v1.kv_cache_interface import (
|
||||||
ChunkedLocalAttentionSpec,
|
ChunkedLocalAttentionSpec,
|
||||||
@@ -36,6 +39,8 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
logic of one specific type of attention layer.
|
logic of one specific type of attention layer.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
supports_fine_grained_hash_lookup: ClassVar[bool] = False
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
kv_cache_spec: KVCacheSpec,
|
kv_cache_spec: KVCacheSpec,
|
||||||
@@ -106,16 +111,34 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
# determining the attention groups.
|
# determining the attention groups.
|
||||||
self.use_eagle = False
|
self.use_eagle = False
|
||||||
|
|
||||||
|
# Partial-hit copy-on-write bookkeeping. Populated only by fine-grained
|
||||||
|
# managers (full attention, mamba "align"); harmlessly empty elsewhere.
|
||||||
|
self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {}
|
||||||
|
self._pending_cow_copies: list[tuple[KVCacheBlock, KVCacheBlock]] = []
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _get_num_evictable_blocks(cls, blocks: Sequence[KVCacheBlock]):
|
def _get_num_evictable_blocks(cls, blocks: Sequence[KVCacheBlock]):
|
||||||
return sum(blk.ref_cnt == 0 and not blk.is_null for blk in blocks)
|
return sum(blk.ref_cnt == 0 and not blk.is_null for blk in blocks)
|
||||||
|
|
||||||
|
def _has_partial_local_hit(
|
||||||
|
self,
|
||||||
|
new_computed_blocks: Sequence[KVCacheBlock],
|
||||||
|
num_local_computed_tokens: int,
|
||||||
|
) -> bool:
|
||||||
|
# The local prefix-cache hit ends inside one of this manager's
|
||||||
|
# blocks: the shared tail block needs CoW.
|
||||||
|
return (
|
||||||
|
len(new_computed_blocks) > 0
|
||||||
|
and num_local_computed_tokens % self.block_size != 0
|
||||||
|
)
|
||||||
|
|
||||||
def get_num_blocks_to_allocate(
|
def get_num_blocks_to_allocate(
|
||||||
self,
|
self,
|
||||||
request_id: str,
|
request_id: str,
|
||||||
num_tokens: int,
|
num_tokens: int,
|
||||||
new_computed_blocks: Sequence[KVCacheBlock],
|
new_computed_blocks: Sequence[KVCacheBlock],
|
||||||
total_computed_tokens: int,
|
total_computed_tokens: int,
|
||||||
|
num_local_computed_tokens: int,
|
||||||
num_tokens_main_model: int,
|
num_tokens_main_model: int,
|
||||||
apply_admission_cap: bool = False,
|
apply_admission_cap: bool = False,
|
||||||
) -> int:
|
) -> int:
|
||||||
@@ -130,6 +153,8 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
prefix caching.
|
prefix caching.
|
||||||
total_computed_tokens: Include both local and external computed
|
total_computed_tokens: Include both local and external computed
|
||||||
tokens.
|
tokens.
|
||||||
|
num_local_computed_tokens: The number of local prefix-cache computed
|
||||||
|
tokens.
|
||||||
num_tokens_main_model: The number of tokens for the main model (aka target
|
num_tokens_main_model: The number of tokens for the main model (aka target
|
||||||
model in spec decode). w/o spec decode, it is num_tokens;
|
model in spec decode). w/o spec decode, it is num_tokens;
|
||||||
with spec decode, it is num_tokens - num_lookahead_tokens.
|
with spec decode, it is num_tokens - num_lookahead_tokens.
|
||||||
@@ -189,6 +214,10 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
num_evictable_blocks = self._get_num_evictable_blocks(
|
num_evictable_blocks = self._get_num_evictable_blocks(
|
||||||
new_computed_blocks[num_skipped_new_computed_blocks:]
|
new_computed_blocks[num_skipped_new_computed_blocks:]
|
||||||
)
|
)
|
||||||
|
if self._has_partial_local_hit(new_computed_blocks, num_local_computed_tokens):
|
||||||
|
# Reserve the extra block that allocate_new_blocks pulls for the
|
||||||
|
# partial-hit CoW redirect.
|
||||||
|
num_new_blocks += 1
|
||||||
return num_new_blocks + num_evictable_blocks
|
return num_new_blocks + num_evictable_blocks
|
||||||
|
|
||||||
def add_local_computed_blocks(
|
def add_local_computed_blocks(
|
||||||
@@ -242,6 +271,13 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
# them so cache_blocks() will not try to re-cache blocks that already
|
# them so cache_blocks() will not try to re-cache blocks that already
|
||||||
# have a block_hash set.
|
# have a block_hash set.
|
||||||
self.num_cached_block[request_id] = len(req_blocks)
|
self.num_cached_block[request_id] = len(req_blocks)
|
||||||
|
if self._has_partial_local_hit(new_computed_blocks, num_local_computed_tokens):
|
||||||
|
# Record the partial tail for the CoW redirect in
|
||||||
|
# allocate_new_blocks; cap the cached count at the full blocks so
|
||||||
|
# cache_blocks() re-caches the private copy once full.
|
||||||
|
block_idx = num_local_computed_tokens // self.block_size
|
||||||
|
self._partial_hit_reqs[request_id] = (block_idx, new_computed_blocks[-1])
|
||||||
|
self.num_cached_block[request_id] = block_idx
|
||||||
|
|
||||||
def allocate_external_computed_blocks(
|
def allocate_external_computed_blocks(
|
||||||
self,
|
self,
|
||||||
@@ -299,17 +335,29 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
Returns:
|
Returns:
|
||||||
The new allocated blocks.
|
The new allocated blocks.
|
||||||
"""
|
"""
|
||||||
|
cow_blocks: list[KVCacheBlock] = []
|
||||||
|
if request_id in self._partial_hit_reqs:
|
||||||
|
# Partial hit: redirect the shared tail to a private CoW block.
|
||||||
|
# Replacing in place keeps the length-based allocation below
|
||||||
|
# correct; the extra block was reserved by
|
||||||
|
# get_num_blocks_to_allocate.
|
||||||
|
block_idx, source_block = self._partial_hit_reqs.pop(request_id)
|
||||||
|
cow_block = self.block_pool.get_new_blocks(1)[0]
|
||||||
|
self._apply_cow(request_id, block_idx, source_block, cow_block)
|
||||||
|
self.new_block_ids.append(cow_block.block_id)
|
||||||
|
cow_blocks.append(cow_block)
|
||||||
|
|
||||||
req_blocks = self.req_to_blocks[request_id]
|
req_blocks = self.req_to_blocks[request_id]
|
||||||
num_required_blocks = cdiv(num_tokens, self.block_size)
|
num_required_blocks = cdiv(num_tokens, self.block_size)
|
||||||
num_new_blocks = num_required_blocks - len(req_blocks)
|
num_new_blocks = num_required_blocks - len(req_blocks)
|
||||||
if num_new_blocks <= 0:
|
if num_new_blocks <= 0:
|
||||||
return []
|
return cow_blocks
|
||||||
else:
|
else:
|
||||||
new_blocks = self.block_pool.get_new_blocks(num_new_blocks)
|
new_blocks = self.block_pool.get_new_blocks(num_new_blocks)
|
||||||
req_blocks.extend(new_blocks)
|
req_blocks.extend(new_blocks)
|
||||||
if self._record_new_block_ids:
|
if self._record_new_block_ids:
|
||||||
self.new_block_ids.extend(b.block_id for b in new_blocks)
|
self.new_block_ids.extend(b.block_id for b in new_blocks)
|
||||||
return new_blocks
|
return cow_blocks + new_blocks
|
||||||
|
|
||||||
def take_new_block_ids(self) -> list[int]:
|
def take_new_block_ids(self) -> list[int]:
|
||||||
"""Drain and return block IDs allocated since the last call."""
|
"""Drain and return block IDs allocated since the last call."""
|
||||||
@@ -317,6 +365,36 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
self.new_block_ids = []
|
self.new_block_ids = []
|
||||||
return ids
|
return ids
|
||||||
|
|
||||||
|
def take_pending_cow_copies(
|
||||||
|
self,
|
||||||
|
) -> list[tuple[KVCacheBlock, KVCacheBlock]]:
|
||||||
|
"""Drain pending CoW source and destination block pairs."""
|
||||||
|
pending_copies = self._pending_cow_copies
|
||||||
|
self._pending_cow_copies = []
|
||||||
|
return pending_copies
|
||||||
|
|
||||||
|
def _apply_cow(
|
||||||
|
self,
|
||||||
|
request_id: str,
|
||||||
|
block_idx: int,
|
||||||
|
source_block: KVCacheBlock,
|
||||||
|
cow_block: KVCacheBlock,
|
||||||
|
) -> None:
|
||||||
|
"""Redirect a partial prefix-cache hit to a private CoW block.
|
||||||
|
|
||||||
|
Both copy endpoints stay retained until the copy has run on the worker,
|
||||||
|
so a same-step free cannot recycle them: ``source_block`` keeps its
|
||||||
|
hit-ref, ``cow_block`` takes an extra ref beyond the one handed to the
|
||||||
|
request.
|
||||||
|
"""
|
||||||
|
req_blocks = self.req_to_blocks[request_id]
|
||||||
|
assert block_idx < len(req_blocks)
|
||||||
|
assert req_blocks[block_idx] is source_block
|
||||||
|
assert not source_block.is_null and source_block.ref_cnt > 0
|
||||||
|
req_blocks[block_idx] = cow_block
|
||||||
|
self._pending_cow_copies.append((source_block, cow_block))
|
||||||
|
cow_block.ref_cnt += 1
|
||||||
|
|
||||||
def cache_blocks(
|
def cache_blocks(
|
||||||
self,
|
self,
|
||||||
request: Request,
|
request: Request,
|
||||||
@@ -398,6 +476,7 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
# Default to [] in case a request is freed (aborted) before alloc.
|
# Default to [] in case a request is freed (aborted) before alloc.
|
||||||
req_blocks = self.req_to_blocks.pop(request_id, [])
|
req_blocks = self.req_to_blocks.pop(request_id, [])
|
||||||
self.num_cached_block.pop(request_id, None)
|
self.num_cached_block.pop(request_id, None)
|
||||||
|
self._partial_hit_reqs.pop(request_id, None)
|
||||||
return req_blocks
|
return req_blocks
|
||||||
|
|
||||||
def free(self, request_id: str) -> None:
|
def free(self, request_id: str) -> None:
|
||||||
@@ -439,7 +518,7 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
alignment_tokens: int,
|
alignment_tokens: int,
|
||||||
dcp_world_size: int = 1,
|
dcp_world_size: int = 1,
|
||||||
pcp_world_size: int = 1,
|
pcp_world_size: int = 1,
|
||||||
) -> tuple[list[KVCacheBlock], ...]:
|
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||||
"""
|
"""
|
||||||
Get the longest cache hit prefix of the blocks that is not longer than
|
Get the longest cache hit prefix of the blocks that is not longer than
|
||||||
`max_length`. The prefix should be a common prefix hit for all the
|
`max_length`. The prefix should be a common prefix hit for all the
|
||||||
@@ -466,11 +545,9 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
pcp_world_size: The world size of prefill context parallelism.
|
pcp_world_size: The world size of prefill context parallelism.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A list of cached blocks with skipped blocks replaced by null block
|
A tuple containing cached blocks and the exact cache-hit length in
|
||||||
for each kv cache group in `kv_cache_group_ids`.
|
tokens. The cached block tuple has skipped blocks replaced by null
|
||||||
Return a list of length `len(kv_cache_group_ids)`, where the i-th
|
blocks for each kv cache group in `kv_cache_group_ids`.
|
||||||
element is a list of cached blocks for the i-th kv cache group
|
|
||||||
in `kv_cache_group_ids`.
|
|
||||||
For example, sliding window manager should return a list like
|
For example, sliding window manager should return a list like
|
||||||
([NULL, NULL, KVCacheBlock(7), KVCacheBlock(8)]) for block size 4
|
([NULL, NULL, KVCacheBlock(7), KVCacheBlock(8)]) for block size 4
|
||||||
and sliding window 8 and len(kv_cache_group_ids) = 1.
|
and sliding window 8 and len(kv_cache_group_ids) = 1.
|
||||||
@@ -558,11 +635,12 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
def new_step_starts(self) -> None:
|
def new_step_starts(self) -> None:
|
||||||
# do nothing by default
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
class FullAttentionManager(SingleTypeKVCacheManager):
|
class FullAttentionManager(SingleTypeKVCacheManager):
|
||||||
|
supports_fine_grained_hash_lookup: ClassVar[bool] = True
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def find_longest_cache_hit(
|
def find_longest_cache_hit(
|
||||||
cls,
|
cls,
|
||||||
@@ -575,42 +653,131 @@ class FullAttentionManager(SingleTypeKVCacheManager):
|
|||||||
alignment_tokens: int,
|
alignment_tokens: int,
|
||||||
dcp_world_size: int = 1,
|
dcp_world_size: int = 1,
|
||||||
pcp_world_size: int = 1,
|
pcp_world_size: int = 1,
|
||||||
) -> tuple[list[KVCacheBlock], ...]:
|
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||||
assert isinstance(
|
assert isinstance(
|
||||||
kv_cache_spec, FullAttentionSpec | ChunkedLocalAttentionSpec
|
kv_cache_spec, FullAttentionSpec | ChunkedLocalAttentionSpec
|
||||||
), (
|
), (
|
||||||
"FullAttentionManager can only be used for full attention "
|
"FullAttentionManager can only be used for full attention "
|
||||||
"and chunked local attention groups"
|
"and chunked local attention groups"
|
||||||
)
|
)
|
||||||
|
block_size = kv_cache_spec.block_size
|
||||||
|
if dcp_world_size * pcp_world_size > 1:
|
||||||
|
# DCP/PCP shard each block's KV across ranks; hashes must be
|
||||||
|
# viewed at the sharded (scaled) block size.
|
||||||
|
block_size *= dcp_world_size * pcp_world_size
|
||||||
|
block_hashes = resolve_block_hashes(
|
||||||
|
block_hashes,
|
||||||
|
block_pool.hash_block_size,
|
||||||
|
block_size,
|
||||||
|
supports_fine_grained_hash_lookup=cls.supports_fine_grained_hash_lookup,
|
||||||
|
alignment_tokens=alignment_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Fine-grained mode (alignment_tokens == hash_block_size <
|
||||||
|
# block_size): resolve_block_hashes kept the raw hash-granularity
|
||||||
|
# list so interior boundaries can be probed.
|
||||||
|
fine_grained = (
|
||||||
|
alignment_tokens < block_size and block_size % alignment_tokens == 0
|
||||||
|
)
|
||||||
|
if fine_grained:
|
||||||
|
assert isinstance(block_hashes, list)
|
||||||
|
full_block_hashes: BlockHashList = BlockHashListWithBlockSize(
|
||||||
|
block_hashes, alignment_tokens, block_size
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
full_block_hashes = block_hashes
|
||||||
|
|
||||||
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
||||||
[] for _ in range(len(kv_cache_group_ids))
|
[] for _ in range(len(kv_cache_group_ids))
|
||||||
)
|
)
|
||||||
block_size = kv_cache_spec.block_size
|
# Phase 1: longest run of cached full blocks from the start. A missing
|
||||||
if dcp_world_size * pcp_world_size > 1:
|
# block implies every later block misses too (chained hashes).
|
||||||
block_size *= dcp_world_size * pcp_world_size
|
for block_hash in itertools.islice(full_block_hashes, max_length // block_size):
|
||||||
max_num_blocks = max_length // block_size
|
cached_block = block_pool.get_cached_block(block_hash, kv_cache_group_ids)
|
||||||
for block_hash in itertools.islice(block_hashes, max_num_blocks):
|
if not cached_block:
|
||||||
# block_hashes is a chain of block hashes. If a block hash is not
|
|
||||||
# in the cached_block_hash_to_id, the following block hashes are
|
|
||||||
# not computed yet for sure.
|
|
||||||
if cached_block := block_pool.get_cached_block(
|
|
||||||
block_hash, kv_cache_group_ids
|
|
||||||
):
|
|
||||||
for computed, cached in zip(computed_blocks, cached_block):
|
|
||||||
computed.append(cached)
|
|
||||||
else:
|
|
||||||
break
|
break
|
||||||
if drop_eagle_block and computed_blocks[0]:
|
for computed, cached in zip(computed_blocks, cached_block):
|
||||||
# Need to drop the last matched block if eagle is enabled.
|
computed.append(cached)
|
||||||
for computed in computed_blocks:
|
hit_length = len(computed_blocks[0]) * block_size
|
||||||
computed.pop()
|
|
||||||
while (
|
# Phase 2 (fine-grained only): extend into the first non-full block by
|
||||||
block_size != alignment_tokens # Faster for common case.
|
# probing its interior hash boundaries high-to-low (longest hit first).
|
||||||
and len(computed_blocks[0]) * block_size % alignment_tokens != 0
|
if fine_grained:
|
||||||
):
|
assert isinstance(block_hashes, list)
|
||||||
for computed in computed_blocks:
|
scale_factor = block_size // alignment_tokens
|
||||||
computed.pop()
|
first_partial_idx = len(computed_blocks[0]) * scale_factor
|
||||||
return computed_blocks
|
max_partial_idx = min(
|
||||||
|
first_partial_idx + scale_factor - 1,
|
||||||
|
max_length // alignment_tokens,
|
||||||
|
len(block_hashes),
|
||||||
|
)
|
||||||
|
for fine_idx in range(max_partial_idx - 1, first_partial_idx - 1, -1):
|
||||||
|
cached_tail = block_pool.get_cached_block(
|
||||||
|
block_hashes[fine_idx], kv_cache_group_ids
|
||||||
|
)
|
||||||
|
if not cached_tail:
|
||||||
|
continue
|
||||||
|
for computed, cached in zip(computed_blocks, cached_tail):
|
||||||
|
computed.append(cached)
|
||||||
|
hit_length = (fine_idx + 1) * alignment_tokens
|
||||||
|
break
|
||||||
|
|
||||||
|
# Eagle needs the tokens right before the generation point recomputed:
|
||||||
|
# drop one hash unit when fine-grained (the tail block's KV is
|
||||||
|
# append-only, so it still covers the reduced length), else one cache
|
||||||
|
# block.
|
||||||
|
if drop_eagle_block and hit_length > 0:
|
||||||
|
hit_length -= min(alignment_tokens, block_size)
|
||||||
|
# Round down to the alignment; a no-op when fine-grained (hits land on
|
||||||
|
# hash boundaries by construction) and when alignment_tokens ==
|
||||||
|
# block_size. Then trim blocks past the new tail.
|
||||||
|
hit_length -= hit_length % alignment_tokens
|
||||||
|
num_blocks = cdiv(hit_length, block_size)
|
||||||
|
for computed in computed_blocks:
|
||||||
|
del computed[num_blocks:]
|
||||||
|
return computed_blocks, hit_length
|
||||||
|
|
||||||
|
def cache_blocks(
|
||||||
|
self,
|
||||||
|
request: Request,
|
||||||
|
num_tokens: int,
|
||||||
|
retention_interval: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
super().cache_blocks(request, num_tokens, retention_interval=retention_interval)
|
||||||
|
hash_block_size = self.block_pool.hash_block_size
|
||||||
|
if self.block_size == hash_block_size:
|
||||||
|
return
|
||||||
|
self._cache_partial_tail_block(request, num_tokens)
|
||||||
|
|
||||||
|
def _cache_partial_tail_block(
|
||||||
|
self,
|
||||||
|
request: Request,
|
||||||
|
num_tokens: int,
|
||||||
|
) -> None:
|
||||||
|
"""Cache the prompt tail when it ends inside a cache block.
|
||||||
|
|
||||||
|
Only the final prompt hash boundary is registered as a partial
|
||||||
|
prefix-cache entry; intermediate hash boundaries inside the same cache
|
||||||
|
block are intentionally skipped.
|
||||||
|
"""
|
||||||
|
hash_block_size = self.block_pool.hash_block_size
|
||||||
|
boundary_tokens = request.num_prompt_tokens // hash_block_size * hash_block_size
|
||||||
|
if boundary_tokens == 0 or boundary_tokens > num_tokens:
|
||||||
|
return
|
||||||
|
if boundary_tokens % self.block_size == 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
blocks = self.req_to_blocks[request.request_id]
|
||||||
|
block_idx = boundary_tokens // self.block_size
|
||||||
|
if block_idx >= len(blocks):
|
||||||
|
return
|
||||||
|
self.block_pool.cache_partial_block(
|
||||||
|
request=request,
|
||||||
|
block=blocks[block_idx],
|
||||||
|
num_tokens=boundary_tokens,
|
||||||
|
kv_cache_group_id=self.kv_cache_group_id,
|
||||||
|
block_size=self.block_size,
|
||||||
|
)
|
||||||
|
|
||||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||||
blocks = self.req_to_blocks[running_request_id]
|
blocks = self.req_to_blocks[running_request_id]
|
||||||
@@ -699,12 +866,23 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
|||||||
alignment_tokens: int,
|
alignment_tokens: int,
|
||||||
dcp_world_size: int = 1,
|
dcp_world_size: int = 1,
|
||||||
pcp_world_size: int = 1,
|
pcp_world_size: int = 1,
|
||||||
) -> tuple[list[KVCacheBlock], ...]:
|
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||||
assert isinstance(kv_cache_spec, SlidingWindowSpec), (
|
assert isinstance(kv_cache_spec, SlidingWindowSpec), (
|
||||||
"SlidingWindowManager can only be used for sliding window groups"
|
"SlidingWindowManager can only be used for sliding window groups"
|
||||||
)
|
)
|
||||||
assert dcp_world_size == 1, "DCP not support sliding window attn now."
|
assert dcp_world_size == 1, "DCP not support sliding window attn now."
|
||||||
assert pcp_world_size == 1, "PCP not support sliding window attn now."
|
assert pcp_world_size == 1, "PCP not support sliding window attn now."
|
||||||
|
# Fine-grained partial hits are not supported for sliding window now
|
||||||
|
assert alignment_tokens % kv_cache_spec.block_size == 0, (
|
||||||
|
"SlidingWindowManager does not support fine-grained (partial) cache hits"
|
||||||
|
)
|
||||||
|
block_hashes = resolve_block_hashes(
|
||||||
|
block_hashes,
|
||||||
|
block_pool.hash_block_size,
|
||||||
|
kv_cache_spec.block_size,
|
||||||
|
supports_fine_grained_hash_lookup=cls.supports_fine_grained_hash_lookup,
|
||||||
|
alignment_tokens=alignment_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
# The number of contiguous blocks needed for a prefix cache hit.
|
# The number of contiguous blocks needed for a prefix cache hit.
|
||||||
sliding_window_contiguous_blocks = cls._contiguous_blocks_for_hit(
|
sliding_window_contiguous_blocks = cls._contiguous_blocks_for_hit(
|
||||||
@@ -717,7 +895,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
|||||||
# sliding_window_contiguous_blocks),
|
# sliding_window_contiguous_blocks),
|
||||||
# which is good for low cache hit rate scenarios.
|
# which is good for low cache hit rate scenarios.
|
||||||
max_num_blocks = max_length // kv_cache_spec.block_size
|
max_num_blocks = max_length // kv_cache_spec.block_size
|
||||||
computed_blocks = tuple(
|
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
||||||
[block_pool.null_block] * max_num_blocks
|
[block_pool.null_block] * max_num_blocks
|
||||||
for _ in range(len(kv_cache_group_ids))
|
for _ in range(len(kv_cache_group_ids))
|
||||||
)
|
)
|
||||||
@@ -772,7 +950,8 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
|||||||
):
|
):
|
||||||
for computed in computed_blocks:
|
for computed in computed_blocks:
|
||||||
computed.pop()
|
computed.pop()
|
||||||
return computed_blocks
|
hit_length = len(computed_blocks[0]) * block_size
|
||||||
|
return computed_blocks, hit_length
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def reachable_block_mask(
|
def reachable_block_mask(
|
||||||
@@ -893,7 +1072,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
|||||||
alignment_tokens: int,
|
alignment_tokens: int,
|
||||||
dcp_world_size: int = 1,
|
dcp_world_size: int = 1,
|
||||||
pcp_world_size: int = 1,
|
pcp_world_size: int = 1,
|
||||||
) -> tuple[list[KVCacheBlock], ...]:
|
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||||
"""
|
"""
|
||||||
For chunked local attention, we need to find the longest cache hit
|
For chunked local attention, we need to find the longest cache hit
|
||||||
prefix of the blocks that is not longer than `max_length`. The prefix
|
prefix of the blocks that is not longer than `max_length`. The prefix
|
||||||
@@ -942,6 +1121,13 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
|||||||
"KV cache groups with different block sizes are not compatible with "
|
"KV cache groups with different block sizes are not compatible with "
|
||||||
"chunked local attention now"
|
"chunked local attention now"
|
||||||
)
|
)
|
||||||
|
block_hashes = resolve_block_hashes(
|
||||||
|
block_hashes,
|
||||||
|
block_pool.hash_block_size,
|
||||||
|
kv_cache_spec.block_size,
|
||||||
|
supports_fine_grained_hash_lookup=cls.supports_fine_grained_hash_lookup,
|
||||||
|
alignment_tokens=alignment_tokens,
|
||||||
|
)
|
||||||
max_num_blocks = max_length // kv_cache_spec.block_size
|
max_num_blocks = max_length // kv_cache_spec.block_size
|
||||||
if max_length > 0:
|
if max_length > 0:
|
||||||
local_attention_start_idx = (
|
local_attention_start_idx = (
|
||||||
@@ -971,7 +1157,8 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
|||||||
computed.append(cached)
|
computed.append(cached)
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
return computed_blocks
|
hit_length = len(computed_blocks[0]) * kv_cache_spec.block_size
|
||||||
|
return computed_blocks, hit_length
|
||||||
|
|
||||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||||
"""
|
"""
|
||||||
@@ -1027,6 +1214,8 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
|||||||
|
|
||||||
|
|
||||||
class MambaManager(SingleTypeKVCacheManager):
|
class MambaManager(SingleTypeKVCacheManager):
|
||||||
|
supports_fine_grained_hash_lookup: ClassVar[bool] = True
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs
|
self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -1035,9 +1224,9 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
# recurrent state. Undo the DCP/PCP block_size scaling that the base
|
# recurrent state. Undo the DCP/PCP block_size scaling that the base
|
||||||
# class applies for attention groups whose KV cache is partitioned.
|
# class applies for attention groups whose KV cache is partitioned.
|
||||||
self.block_size = kv_cache_spec.block_size
|
self.block_size = kv_cache_spec.block_size
|
||||||
self.cached_blocks_this_step: set[BlockHashWithGroupId] = set()
|
|
||||||
self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode
|
self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode
|
||||||
self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks
|
self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks
|
||||||
|
self.cached_blocks_this_step: set[BlockHashWithGroupId] = set()
|
||||||
if self.mamba_cache_mode == "align":
|
if self.mamba_cache_mode == "align":
|
||||||
# Mapping from request ID to the index of the block
|
# Mapping from request ID to the index of the block
|
||||||
# allocated in the previous step
|
# allocated in the previous step
|
||||||
@@ -1057,17 +1246,46 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
alignment_tokens: int,
|
alignment_tokens: int,
|
||||||
dcp_world_size: int = 1,
|
dcp_world_size: int = 1,
|
||||||
pcp_world_size: int = 1,
|
pcp_world_size: int = 1,
|
||||||
) -> tuple[list[KVCacheBlock], ...]:
|
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||||
assert isinstance(kv_cache_spec, MambaSpec), (
|
assert isinstance(kv_cache_spec, MambaSpec), (
|
||||||
"MambaManager can only be used for mamba groups"
|
"MambaManager can only be used for mamba groups"
|
||||||
)
|
)
|
||||||
assert dcp_world_size == 1, "DCP not support mamba now."
|
assert dcp_world_size == 1, "DCP not support mamba now."
|
||||||
assert pcp_world_size == 1, "PCP not support mamba now."
|
assert pcp_world_size == 1, "PCP not support mamba now."
|
||||||
|
block_hashes = resolve_block_hashes(
|
||||||
|
block_hashes,
|
||||||
|
block_pool.hash_block_size,
|
||||||
|
kv_cache_spec.block_size,
|
||||||
|
supports_fine_grained_hash_lookup=cls.supports_fine_grained_hash_lookup,
|
||||||
|
alignment_tokens=alignment_tokens,
|
||||||
|
)
|
||||||
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
||||||
[] for _ in range(len(kv_cache_group_ids))
|
[] for _ in range(len(kv_cache_group_ids))
|
||||||
)
|
)
|
||||||
|
hit_length = 0
|
||||||
|
|
||||||
block_size = kv_cache_spec.block_size
|
block_size = kv_cache_spec.block_size
|
||||||
|
if alignment_tokens < block_size and block_size % alignment_tokens == 0:
|
||||||
|
assert isinstance(block_hashes, list)
|
||||||
|
hash_block_size = alignment_tokens
|
||||||
|
scale_factor = block_size // hash_block_size
|
||||||
|
max_num_partial_units = min(
|
||||||
|
max_length // hash_block_size, len(block_hashes)
|
||||||
|
)
|
||||||
|
for fine_idx in range(max_num_partial_units - 1, -1, -1):
|
||||||
|
num_tokens = (fine_idx + 1) * hash_block_size
|
||||||
|
block_hash = block_hashes[fine_idx]
|
||||||
|
if cached_block := block_pool.get_cached_block(
|
||||||
|
block_hash, kv_cache_group_ids
|
||||||
|
):
|
||||||
|
block_idx = fine_idx // scale_factor
|
||||||
|
for computed, cached in zip(computed_blocks, cached_block):
|
||||||
|
computed.extend([block_pool.null_block] * block_idx)
|
||||||
|
computed.append(cached)
|
||||||
|
hit_length = num_tokens
|
||||||
|
break
|
||||||
|
return computed_blocks, hit_length
|
||||||
|
|
||||||
max_num_blocks = max_length // block_size
|
max_num_blocks = max_length // block_size
|
||||||
# Search from right to left and early stop when a match is found.
|
# Search from right to left and early stop when a match is found.
|
||||||
for i in range(max_num_blocks - 1, -1, -1):
|
for i in range(max_num_blocks - 1, -1, -1):
|
||||||
@@ -1089,9 +1307,10 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
# so we insert dummy blocks at the beginning:
|
# so we insert dummy blocks at the beginning:
|
||||||
computed.extend([block_pool.null_block] * i)
|
computed.extend([block_pool.null_block] * i)
|
||||||
computed.append(cached)
|
computed.append(cached)
|
||||||
|
hit_length = (i + 1) * block_size
|
||||||
break # we just need the last match - early stopping
|
break # we just need the last match - early stopping
|
||||||
|
|
||||||
return computed_blocks
|
return computed_blocks, hit_length
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def reachable_block_mask(
|
def reachable_block_mask(
|
||||||
@@ -1189,6 +1408,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
num_tokens: int,
|
num_tokens: int,
|
||||||
new_computed_blocks: Sequence[KVCacheBlock],
|
new_computed_blocks: Sequence[KVCacheBlock],
|
||||||
total_computed_tokens: int,
|
total_computed_tokens: int,
|
||||||
|
num_local_computed_tokens: int,
|
||||||
num_tokens_main_model: int,
|
num_tokens_main_model: int,
|
||||||
apply_admission_cap: bool = False,
|
apply_admission_cap: bool = False,
|
||||||
) -> int:
|
) -> int:
|
||||||
@@ -1214,6 +1434,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
num_tokens,
|
num_tokens,
|
||||||
new_computed_blocks,
|
new_computed_blocks,
|
||||||
total_computed_tokens,
|
total_computed_tokens,
|
||||||
|
num_local_computed_tokens,
|
||||||
num_tokens_main_model,
|
num_tokens_main_model,
|
||||||
apply_admission_cap=apply_admission_cap,
|
apply_admission_cap=apply_admission_cap,
|
||||||
)
|
)
|
||||||
@@ -1235,15 +1456,26 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
- len(new_computed_blocks)
|
- len(new_computed_blocks)
|
||||||
- len(self.req_to_blocks[request_id])
|
- len(self.req_to_blocks[request_id])
|
||||||
)
|
)
|
||||||
|
has_partial_hit = (
|
||||||
|
self._has_partial_local_hit(
|
||||||
|
new_computed_blocks, num_local_computed_tokens
|
||||||
|
)
|
||||||
|
or request_id in self._partial_hit_reqs
|
||||||
|
)
|
||||||
|
if has_partial_hit:
|
||||||
|
num_new_blocks = max(num_new_blocks, 0) + 1
|
||||||
if num_new_blocks > 0:
|
if num_new_blocks > 0:
|
||||||
if request_id in self._allocated_block_reqs:
|
if request_id in self._allocated_block_reqs:
|
||||||
# Old request. Needs at most 1 more blocks as we can reuse the
|
# Old request. Needs at most 1 more blocks as we can reuse the
|
||||||
# speculative blocks in previous step.
|
# speculative blocks in previous step.
|
||||||
num_new_blocks = 1
|
num_new_blocks = 1 + int(has_partial_hit)
|
||||||
else:
|
else:
|
||||||
# First prefill. Allocate 1 block for running state and the
|
# First prefill. Allocate 1 block for running state, the
|
||||||
# speculative blocks.
|
# speculative blocks, and one extra block if a partial cache
|
||||||
num_new_blocks = 1 + self.num_speculative_blocks
|
# hit must be copy-on-written before the new tokens run.
|
||||||
|
num_new_blocks = (
|
||||||
|
1 + self.num_speculative_blocks + int(has_partial_hit)
|
||||||
|
)
|
||||||
|
|
||||||
num_evictable_computed_blocks = self._get_num_evictable_blocks(
|
num_evictable_computed_blocks = self._get_num_evictable_blocks(
|
||||||
new_computed_blocks
|
new_computed_blocks
|
||||||
@@ -1275,9 +1507,11 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
num_required_blocks = (
|
num_required_blocks = (
|
||||||
cdiv(num_tokens, self.block_size) + self.num_speculative_blocks
|
cdiv(num_tokens, self.block_size) + self.num_speculative_blocks
|
||||||
)
|
)
|
||||||
|
partial_hit = self._partial_hit_reqs.get(request_id)
|
||||||
|
has_partial_hit = partial_hit is not None
|
||||||
# `num_required_blocks` might be less than `len(req_blocks)` if blocks are
|
# `num_required_blocks` might be less than `len(req_blocks)` if blocks are
|
||||||
# over-allocated at last round.
|
# over-allocated at last round.
|
||||||
if num_required_blocks <= len(req_blocks):
|
if num_required_blocks <= len(req_blocks) and not has_partial_hit:
|
||||||
return []
|
return []
|
||||||
else:
|
else:
|
||||||
prev_block_len = len(req_blocks)
|
prev_block_len = len(req_blocks)
|
||||||
@@ -1317,14 +1551,42 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
num_new_blocks = num_required_blocks - len(req_blocks)
|
num_new_blocks = num_required_blocks - len(req_blocks)
|
||||||
|
if has_partial_hit:
|
||||||
|
num_new_blocks = max(num_new_blocks, 0) + 1
|
||||||
if blocks_allocated:
|
if blocks_allocated:
|
||||||
assert num_new_blocks <= 1
|
assert num_new_blocks <= 1 + int(has_partial_hit)
|
||||||
else:
|
else:
|
||||||
assert num_new_blocks <= self.num_speculative_blocks + 1
|
assert num_new_blocks <= self.num_speculative_blocks + 1 + int(
|
||||||
|
has_partial_hit
|
||||||
|
)
|
||||||
new_blocks = self.block_pool.get_new_blocks(num_new_blocks)
|
new_blocks = self.block_pool.get_new_blocks(num_new_blocks)
|
||||||
|
returned_blocks = req_blocks[prev_block_len:]
|
||||||
|
if partial_hit is not None:
|
||||||
|
block_idx, source_block = partial_hit
|
||||||
|
cow_block = new_blocks[0]
|
||||||
|
new_blocks = new_blocks[1:]
|
||||||
|
if blocks_allocated:
|
||||||
|
# The worker block table of a running request is
|
||||||
|
# append-only, so the request must stay on
|
||||||
|
# source_block. Move the cache entry to cow_block
|
||||||
|
# instead; the queued copy fills it before forward
|
||||||
|
# overwrites source_block.
|
||||||
|
assert req_blocks[block_idx] is source_block
|
||||||
|
self.block_pool.move_block_hashes(source_block, cow_block)
|
||||||
|
self._pending_cow_copies.append((source_block, cow_block))
|
||||||
|
source_block.ref_cnt += 1
|
||||||
|
if cow_block.block_hash is not None:
|
||||||
|
# The moved entry is only filled by this step's
|
||||||
|
# copy, so defer same-step hits on it.
|
||||||
|
self.cached_blocks_this_step.add(cow_block.block_hash)
|
||||||
|
else:
|
||||||
|
self._apply_cow(request_id, block_idx, source_block, cow_block)
|
||||||
|
returned_blocks = [cow_block] + returned_blocks
|
||||||
req_blocks.extend(new_blocks)
|
req_blocks.extend(new_blocks)
|
||||||
self._allocated_block_reqs.add(request_id)
|
self._allocated_block_reqs.add(request_id)
|
||||||
return req_blocks[prev_block_len:]
|
self._partial_hit_reqs.pop(request_id, None)
|
||||||
|
returned_blocks.extend(new_blocks)
|
||||||
|
return returned_blocks
|
||||||
|
|
||||||
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
||||||
if self.mamba_cache_mode == "align":
|
if self.mamba_cache_mode == "align":
|
||||||
@@ -1349,6 +1611,10 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0)
|
num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0)
|
||||||
super().cache_blocks(request, num_tokens, retention_interval=retention_interval)
|
super().cache_blocks(request, num_tokens, retention_interval=retention_interval)
|
||||||
num_cached_blocks_after = self.num_cached_block.get(request.request_id, 0)
|
num_cached_blocks_after = self.num_cached_block.get(request.request_id, 0)
|
||||||
|
if self.mamba_cache_mode == "align":
|
||||||
|
partial_hash = self._cache_partial_tail_block(request, num_tokens)
|
||||||
|
if partial_hash is not None:
|
||||||
|
self.cached_blocks_this_step.add(partial_hash)
|
||||||
if num_cached_blocks_after > num_cached_blocks_before:
|
if num_cached_blocks_after > num_cached_blocks_before:
|
||||||
for block in self.req_to_blocks[request.request_id][
|
for block in self.req_to_blocks[request.request_id][
|
||||||
num_cached_blocks_before:num_cached_blocks_after
|
num_cached_blocks_before:num_cached_blocks_after
|
||||||
@@ -1364,6 +1630,44 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
def new_step_starts(self) -> None:
|
def new_step_starts(self) -> None:
|
||||||
self.cached_blocks_this_step.clear()
|
self.cached_blocks_this_step.clear()
|
||||||
|
|
||||||
|
def _cache_partial_tail_block(
|
||||||
|
self,
|
||||||
|
request: Request,
|
||||||
|
num_tokens: int,
|
||||||
|
) -> BlockHashWithGroupId | None:
|
||||||
|
hash_block_size = self.block_pool.hash_block_size
|
||||||
|
if self.block_size == hash_block_size:
|
||||||
|
return None
|
||||||
|
if num_tokens % self.block_size == 0:
|
||||||
|
return None
|
||||||
|
if num_tokens % hash_block_size != 0:
|
||||||
|
return None
|
||||||
|
latest_prompt_hash_boundary = (
|
||||||
|
request.num_prompt_tokens // hash_block_size
|
||||||
|
) * hash_block_size
|
||||||
|
if num_tokens != latest_prompt_hash_boundary:
|
||||||
|
return None
|
||||||
|
|
||||||
|
block_idx = num_tokens // self.block_size
|
||||||
|
blocks = self.req_to_blocks[request.request_id]
|
||||||
|
if block_idx >= len(blocks):
|
||||||
|
return None
|
||||||
|
source_block = blocks[block_idx]
|
||||||
|
if source_block.is_null:
|
||||||
|
return None
|
||||||
|
|
||||||
|
partial_hash = self.block_pool.cache_partial_block(
|
||||||
|
request=request,
|
||||||
|
block=source_block,
|
||||||
|
num_tokens=num_tokens,
|
||||||
|
kv_cache_group_id=self.kv_cache_group_id,
|
||||||
|
block_size=self.block_size,
|
||||||
|
)
|
||||||
|
if partial_hash is not None:
|
||||||
|
self._partial_hit_reqs[request.request_id] = (block_idx, source_block)
|
||||||
|
self.num_cached_block[request.request_id] = block_idx
|
||||||
|
return partial_hash
|
||||||
|
|
||||||
|
|
||||||
class CrossAttentionManager(SingleTypeKVCacheManager):
|
class CrossAttentionManager(SingleTypeKVCacheManager):
|
||||||
"""Manager for cross-attention KV cache in encoder-decoder models."""
|
"""Manager for cross-attention KV cache in encoder-decoder models."""
|
||||||
@@ -1415,7 +1719,7 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
|
|||||||
alignment_tokens: int,
|
alignment_tokens: int,
|
||||||
dcp_world_size: int = 1,
|
dcp_world_size: int = 1,
|
||||||
pcp_world_size: int = 1,
|
pcp_world_size: int = 1,
|
||||||
) -> tuple[list[KVCacheBlock], ...]:
|
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||||
assert isinstance(kv_cache_spec, CrossAttentionSpec), (
|
assert isinstance(kv_cache_spec, CrossAttentionSpec), (
|
||||||
"CrossAttentionManager can only be used for cross-attention groups"
|
"CrossAttentionManager can only be used for cross-attention groups"
|
||||||
)
|
)
|
||||||
@@ -1435,16 +1739,18 @@ class SinkFullAttentionManager(FullAttentionManager):
|
|||||||
block_pool: BlockPool,
|
block_pool: BlockPool,
|
||||||
enable_caching: bool,
|
enable_caching: bool,
|
||||||
kv_cache_group_id: int,
|
kv_cache_group_id: int,
|
||||||
|
scheduler_block_size: int,
|
||||||
dcp_world_size: int = 1,
|
dcp_world_size: int = 1,
|
||||||
pcp_world_size: int = 1,
|
pcp_world_size: int = 1,
|
||||||
):
|
):
|
||||||
super().__init__(
|
super().__init__(
|
||||||
kv_cache_spec,
|
kv_cache_spec=kv_cache_spec,
|
||||||
block_pool,
|
block_pool=block_pool,
|
||||||
enable_caching,
|
enable_caching=enable_caching,
|
||||||
kv_cache_group_id,
|
kv_cache_group_id=kv_cache_group_id,
|
||||||
dcp_world_size,
|
scheduler_block_size=scheduler_block_size,
|
||||||
pcp_world_size,
|
dcp_world_size=dcp_world_size,
|
||||||
|
pcp_world_size=pcp_world_size,
|
||||||
)
|
)
|
||||||
sink_len = kv_cache_spec.sink_len
|
sink_len = kv_cache_spec.sink_len
|
||||||
assert sink_len is not None and sink_len > 0 and sink_len % self.block_size == 0
|
assert sink_len is not None and sink_len > 0 and sink_len % self.block_size == 0
|
||||||
|
|||||||
@@ -112,7 +112,7 @@ from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
|
|||||||
from vllm.v1.worker.gpu.states import RequestState
|
from vllm.v1.worker.gpu.states import RequestState
|
||||||
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
|
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
|
||||||
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
|
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
|
||||||
from vllm.v1.worker.utils import KVBlockZeroer
|
from vllm.v1.worker.utils import KVBlockZeroer, copy_kv_cache_blocks_inplace
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -838,6 +838,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
assert self.kv_block_zeroer is not None
|
assert self.kv_block_zeroer is not None
|
||||||
self.kv_block_zeroer.zero_block_ids(scheduler_output.new_block_ids_to_zero)
|
self.kv_block_zeroer.zero_block_ids(scheduler_output.new_block_ids_to_zero)
|
||||||
|
|
||||||
|
# Apply copy-on-write block copies for partial prefix-cache hits, after
|
||||||
|
# zeroing new blocks and before the forward pass reads them.
|
||||||
|
if scheduler_output.kv_cache_block_copies:
|
||||||
|
copy_kv_cache_blocks_inplace(
|
||||||
|
self.kv_caches,
|
||||||
|
self.kv_cache_config.num_blocks,
|
||||||
|
scheduler_output.kv_cache_block_copies,
|
||||||
|
)
|
||||||
|
|
||||||
def prepare_inputs(
|
def prepare_inputs(
|
||||||
self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor
|
self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor
|
||||||
) -> InputBatch:
|
) -> InputBatch:
|
||||||
|
|||||||
@@ -230,6 +230,7 @@ from .utils import (
|
|||||||
KVBlockZeroer,
|
KVBlockZeroer,
|
||||||
add_kv_sharing_layers_to_kv_cache_groups,
|
add_kv_sharing_layers_to_kv_cache_groups,
|
||||||
bind_kv_cache,
|
bind_kv_cache,
|
||||||
|
copy_kv_cache_blocks_inplace,
|
||||||
prepare_kernel_block_sizes,
|
prepare_kernel_block_sizes,
|
||||||
sanity_check_mm_encoder_outputs,
|
sanity_check_mm_encoder_outputs,
|
||||||
)
|
)
|
||||||
@@ -1188,6 +1189,12 @@ class GPUModelRunner(
|
|||||||
# stale NaN/data from corrupting attention or SSM computation.
|
# stale NaN/data from corrupting attention or SSM computation.
|
||||||
if scheduler_output.new_block_ids_to_zero:
|
if scheduler_output.new_block_ids_to_zero:
|
||||||
self._zero_block_ids(scheduler_output.new_block_ids_to_zero)
|
self._zero_block_ids(scheduler_output.new_block_ids_to_zero)
|
||||||
|
if scheduler_output.kv_cache_block_copies:
|
||||||
|
copy_kv_cache_blocks_inplace(
|
||||||
|
self.kv_caches,
|
||||||
|
self.kv_cache_config.num_blocks,
|
||||||
|
scheduler_output.kv_cache_block_copies,
|
||||||
|
)
|
||||||
|
|
||||||
# Free the cached encoder outputs.
|
# Free the cached encoder outputs.
|
||||||
for mm_hash in scheduler_output.free_encoder_mm_hashes:
|
for mm_hash in scheduler_output.free_encoder_mm_hashes:
|
||||||
|
|||||||
+43
-1
@@ -2,11 +2,12 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import math
|
import math
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable, Sequence
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from itertools import product as iprod
|
from itertools import product as iprod
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.config import CacheConfig, VllmConfig
|
from vllm.config import CacheConfig, VllmConfig
|
||||||
@@ -18,11 +19,13 @@ from vllm.platforms import current_platform
|
|||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
from vllm.utils.math_utils import largest_power_of_2_divisor
|
from vllm.utils.math_utils import largest_power_of_2_divisor
|
||||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib
|
from vllm.utils.mem_utils import MemorySnapshot, format_gib
|
||||||
|
from vllm.utils.torch_utils import async_tensor_h2d
|
||||||
from vllm.v1.attention.backend import (
|
from vllm.v1.attention.backend import (
|
||||||
AttentionBackend,
|
AttentionBackend,
|
||||||
AttentionMetadataBuilder,
|
AttentionMetadataBuilder,
|
||||||
MultipleOf,
|
MultipleOf,
|
||||||
)
|
)
|
||||||
|
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||||
from vllm.v1.kv_cache_interface import (
|
from vllm.v1.kv_cache_interface import (
|
||||||
AttentionSpec,
|
AttentionSpec,
|
||||||
EncoderOnlyAttentionSpec,
|
EncoderOnlyAttentionSpec,
|
||||||
@@ -535,6 +538,45 @@ def bind_kv_cache(
|
|||||||
forward_context[layer_name].kv_cache = kv_cache
|
forward_context[layer_name].kv_cache = kv_cache
|
||||||
|
|
||||||
|
|
||||||
|
def copy_kv_cache_blocks_inplace(
|
||||||
|
kv_caches: Iterable[torch.Tensor | list[torch.Tensor]],
|
||||||
|
num_blocks: int,
|
||||||
|
kv_cache_block_copies: Sequence[KVCacheBlockCopy],
|
||||||
|
) -> None:
|
||||||
|
if not kv_cache_block_copies:
|
||||||
|
return
|
||||||
|
|
||||||
|
storage_tensors: list[torch.Tensor] = []
|
||||||
|
seen_storage: set[int] = set()
|
||||||
|
for entry in kv_caches:
|
||||||
|
# Mamba layers hold a list of state tensors; attention layers a single
|
||||||
|
# tensor. Both alias the shared block-major backing storage.
|
||||||
|
tensors = entry if isinstance(entry, (list, tuple)) else (entry,)
|
||||||
|
for tensor in tensors:
|
||||||
|
ptr = tensor.untyped_storage().data_ptr()
|
||||||
|
if ptr in seen_storage:
|
||||||
|
continue
|
||||||
|
seen_storage.add(ptr)
|
||||||
|
storage_tensors.append(tensor)
|
||||||
|
|
||||||
|
if not storage_tensors:
|
||||||
|
return
|
||||||
|
device = storage_tensors[0].device
|
||||||
|
indices_np = np.array(kv_cache_block_copies, dtype=np.int64)
|
||||||
|
indices = async_tensor_h2d(indices_np, device=device)
|
||||||
|
src_indices, dst_indices = indices.unbind(dim=1)
|
||||||
|
|
||||||
|
for tensor in storage_tensors:
|
||||||
|
assert tensor.device == device
|
||||||
|
blocks = torch.empty(0, dtype=torch.uint8, device=device)
|
||||||
|
blocks.set_(tensor.untyped_storage())
|
||||||
|
# Block-major backing storage: block i owns the contiguous byte range
|
||||||
|
# [i * page_size, (i + 1) * page_size).
|
||||||
|
assert blocks.numel() % num_blocks == 0
|
||||||
|
blocks = blocks.view(num_blocks, -1)
|
||||||
|
blocks[dst_indices] = blocks[src_indices]
|
||||||
|
|
||||||
|
|
||||||
def is_residual_scattered_for_sp(
|
def is_residual_scattered_for_sp(
|
||||||
vllm_config: VllmConfig, num_input_tokens: int
|
vllm_config: VllmConfig, num_input_tokens: int
|
||||||
) -> bool:
|
) -> bool:
|
||||||
|
|||||||
Reference in New Issue
Block a user