[Bugfix][KV Connector] MooncakeStore: exclude non-prefix-cacheable (QSA ring) groups; fix align-mode check (#55027)
Signed-off-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Zhewen Li
OpenAI Codex
Claude Fable 5.1
parent
fe49665eb3
commit
58d45fd767
@@ -35,7 +35,7 @@ from vllm.multimodal.inputs import (
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.hashing import sha256, sha256_cbor, xxhash, xxhash_cbor
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
FreeKVCacheBlockQueue,
|
||||
@@ -61,6 +61,7 @@ from vllm.v1.hisparse.layout import (
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
HiSparseHotSpec,
|
||||
@@ -324,6 +325,45 @@ def test_kv_cache_config_selects_only_transferable_groups():
|
||||
)
|
||||
|
||||
|
||||
def test_kv_cache_config_selects_prefix_cacheable_groups():
|
||||
"""Prefix stores exclude scratch state without changing transfer groups."""
|
||||
full_group = KVCacheGroupSpec(["full"], new_kv_cache_spec())
|
||||
qsa_group = KVCacheGroupSpec(
|
||||
["qsa"],
|
||||
CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
head_size_v=0,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
)
|
||||
disabled_group = KVCacheGroupSpec(
|
||||
["disabled"], new_kv_cache_spec(), enable_kv_transfer=False
|
||||
)
|
||||
config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[full_group, qsa_group, disabled_group],
|
||||
)
|
||||
assert config.transfer_group_ids == (0, 1)
|
||||
assert config.select_transfer_block_ids(([1], [2], [3])) == ([1], [2])
|
||||
assert config.prefix_cacheable_group_ids == (0,)
|
||||
assert config.prefix_cacheable_groups == (full_group,)
|
||||
|
||||
|
||||
def test_kv_cache_blocks_selects_requested_groups():
|
||||
blocks = KVCacheBlocks(
|
||||
(
|
||||
[KVCacheBlock(1)],
|
||||
[KVCacheBlock(2)],
|
||||
[KVCacheBlock(3)],
|
||||
)
|
||||
)
|
||||
|
||||
assert blocks.get_block_ids(group_ids=(0, 2)) == ([1], [3])
|
||||
|
||||
|
||||
def new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import
|
||||
MooncakeStoreConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -91,6 +92,53 @@ def _make_kv_cache_config() -> KVCacheConfig:
|
||||
)
|
||||
|
||||
|
||||
def _make_qsa_hybrid_kv_cache_config(mamba_mode: str):
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=800, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
qsa_spec = CircularBufferSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
head_size_v=0,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=800,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode=mamba_mode,
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["full"], full_spec),
|
||||
KVCacheGroupSpec(["qsa"], qsa_spec),
|
||||
KVCacheGroupSpec(["mamba"], mamba_spec),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_validation_accepts_aligned_mamba_after_block_size_rewrite():
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.cache_config.block_size = 8
|
||||
|
||||
mooncake_store_connector.MooncakeStoreConnector._validate_kv_cache_config(
|
||||
vllm_config, _make_qsa_hybrid_kv_cache_config("align")
|
||||
)
|
||||
|
||||
|
||||
def test_validation_rejects_mamba_mode_directly():
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.cache_config.block_size = 800
|
||||
|
||||
with pytest.raises(ValueError, match="mamba_cache_mode=\x27none\x27"):
|
||||
mooncake_store_connector.MooncakeStoreConnector._validate_kv_cache_config(
|
||||
vllm_config, _make_qsa_hybrid_kv_cache_config("none")
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_requires_align_mode_for_mamba():
|
||||
vllm_config = _make_vllm_config()
|
||||
mamba_spec = MambaSpec(
|
||||
|
||||
@@ -759,8 +759,9 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
worker.pp_size = 1
|
||||
worker.num_kv_head = 8
|
||||
assert worker.coord.enable_partial_hash_hits
|
||||
# The scratch DB is keyed at its own block size and never probed.
|
||||
assert worker.token_dbs[2].hash_block_size == 4
|
||||
# Only the participating full-attention and Mamba groups are registered.
|
||||
assert len(worker.token_dbs) == 2
|
||||
assert all(db.hash_block_size == 8 for db in worker.token_dbs)
|
||||
|
||||
for g_idx, db in enumerate(worker.token_dbs):
|
||||
db.set_kv_caches_base_addr([g_idx * 10_000])
|
||||
@@ -772,11 +773,11 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
block_size=worker.block_size,
|
||||
coord=worker.coord,
|
||||
tp_rank=0,
|
||||
group_put_steps=[1, 1, 1],
|
||||
group_put_steps=[1, 1],
|
||||
kv_role="kv_both",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=MagicMock(),
|
||||
group_participates=[True, True, False],
|
||||
group_participates=[True, True],
|
||||
)
|
||||
|
||||
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//8-1]).
|
||||
@@ -784,7 +785,7 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1], [2], [3]),
|
||||
block_ids=([1], [2]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
@@ -797,8 +798,7 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
# A 13-token prompt sharing the prefix must hit the first hash unit.
|
||||
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 8
|
||||
# The scratch group's namespace never enters the store.
|
||||
scratch_prefix = worker.token_dbs[2].key_for(hs[0]).rsplit("@", 1)[0]
|
||||
assert not any(key.startswith(scratch_prefix) for key in store._data)
|
||||
assert not any("@group:2" in key for key in store._data)
|
||||
|
||||
|
||||
def test_ring_scratch_group_is_never_stored_and_does_not_block_hits():
|
||||
@@ -838,7 +838,8 @@ def test_ring_scratch_group_is_never_stored_and_does_not_block_hits():
|
||||
worker.tp_size = 1
|
||||
worker.pp_size = 1
|
||||
worker.num_kv_head = 8
|
||||
assert worker.token_dbs[1].hash_block_size == 8
|
||||
assert len(worker.token_dbs) == 1
|
||||
assert worker.token_dbs[0].hash_block_size == 16
|
||||
|
||||
raw_full = torch.zeros(4 * full.page_size_bytes, dtype=torch.int8)
|
||||
raw_ring = torch.zeros(4 * ring.page_size_bytes, dtype=torch.int8)
|
||||
@@ -879,14 +880,14 @@ def test_ring_scratch_group_is_never_stored_and_does_not_block_hits():
|
||||
kv_role=worker.kv_role,
|
||||
ready_event=threading.Event(),
|
||||
enable_kv_event=False,
|
||||
group_participates=[True, False],
|
||||
group_participates=[True],
|
||||
)
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
|
||||
save_req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
# The ring group holds one block for the request's lifetime.
|
||||
block_ids=([1, 2, 3, 4], [1]),
|
||||
# Scheduler metadata is projected to the participating store groups.
|
||||
block_ids=([1, 2, 3, 4],),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
store_job_id=1,
|
||||
|
||||
@@ -2,12 +2,15 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
LoadSpec,
|
||||
MooncakeLookupResult,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
ReqMeta,
|
||||
RequestTracker,
|
||||
@@ -16,7 +19,16 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor
|
||||
MooncakeStoreScheduler,
|
||||
)
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
||||
from vllm.v1.core.sched.output import KVConnectorBlockState
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
|
||||
def _make_bare_scheduler(
|
||||
@@ -35,9 +47,9 @@ def _make_bare_scheduler(
|
||||
scheduler._block_size = 16
|
||||
scheduler._hash_block_size = hash_block_size
|
||||
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
|
||||
scheduler.kv_cache_config = SimpleNamespace(
|
||||
select_transfer_block_ids=lambda block_ids: tuple(block_ids)
|
||||
)
|
||||
scheduler.kv_cache_config = SimpleNamespace()
|
||||
scheduler._store_group_ids = (0,)
|
||||
scheduler._store_group_id_by_kv_cache_group_id = {0: 0, 1: 1}
|
||||
scheduler.load_specs = {}
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
@@ -228,14 +240,82 @@ def test_pending_load_for_non_chosen_connector_is_dropped():
|
||||
assert "req-0" not in scheduler._request_trackers
|
||||
|
||||
|
||||
def _make_qsa_hybrid_cache_config():
|
||||
full = FullAttentionSpec(block_size=800, num_kv_heads=8, head_size=64, dtype=None)
|
||||
circular = CircularBufferSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
head_size_v=0,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
mamba = MambaSpec(
|
||||
block_size=800,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["full"], full),
|
||||
KVCacheGroupSpec(["qsa"], circular),
|
||||
KVCacheGroupSpec(["mamba"], mamba),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_projects_nonprefix_groups_and_mamba_ids():
|
||||
vllm_config = SimpleNamespace(
|
||||
kv_transfer_config=SimpleNamespace(
|
||||
kv_role="kv_both", kv_connector_extra_config={}
|
||||
),
|
||||
kv_events_config=None,
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=800, enable_prefix_caching=True, prefix_match_unit=None
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1, world_size=1),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"scheduler.LookupKeyClient"
|
||||
):
|
||||
scheduler = MooncakeStoreScheduler(vllm_config, _make_qsa_hybrid_cache_config())
|
||||
|
||||
assert scheduler._store_group_ids == (0, 2)
|
||||
assert scheduler._boundary_state_group_ids == frozenset({1})
|
||||
|
||||
|
||||
def test_current_save_block_ids_use_store_group_projection():
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.kv_cache_config = _make_qsa_hybrid_cache_config()
|
||||
scheduler._store_group_ids = (0, 2)
|
||||
meta = MooncakeStoreConnectorMetadata(set(), set())
|
||||
req_meta = ReqMeta(
|
||||
req_id="req-0",
|
||||
token_len_chunk=800,
|
||||
block_ids=(),
|
||||
block_hashes=[b"h0"],
|
||||
can_save=True,
|
||||
)
|
||||
meta.add_request(req_meta)
|
||||
output = SimpleNamespace(
|
||||
kv_connector_block_state=_make_connector_block_state(([10], [80], [30]))
|
||||
)
|
||||
|
||||
scheduler._apply_current_save_block_ids(meta, output)
|
||||
|
||||
assert req_meta.block_ids == ([10], [30])
|
||||
|
||||
|
||||
def test_update_state_excludes_nontransfer_groups():
|
||||
"""Store metadata must match the worker's registered cache groups."""
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.kv_cache_config = SimpleNamespace(
|
||||
select_transfer_block_ids=lambda block_ids: (block_ids[0],)
|
||||
)
|
||||
scheduler._store_group_ids = (0,)
|
||||
request = SimpleNamespace(request_id="req-1")
|
||||
blocks = SimpleNamespace(get_block_ids=lambda: ([1, 2], [9]))
|
||||
blocks = KVCacheBlocks(([KVCacheBlock(1), KVCacheBlock(2)], [KVCacheBlock(9)]))
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=32)
|
||||
|
||||
@@ -1003,6 +1083,7 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
|
||||
def test_finished_partial_tail_is_pre_pinned_as_store_job():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler._store_group_ids = (0, 1)
|
||||
scheduler.client = SimpleNamespace(discard=lambda *_: None)
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
@@ -1143,6 +1224,20 @@ def _make_offload_only_output(entries, block_ids=([0],)):
|
||||
)
|
||||
|
||||
|
||||
def test_boundary_state_group_ids_are_remapped_to_store_projection():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=800, enable_partial_hash_hits=True)
|
||||
scheduler.kv_cache_config = _make_qsa_hybrid_cache_config()
|
||||
scheduler._store_group_ids = (0, 2)
|
||||
scheduler._store_group_id_by_kv_cache_group_id = {0: 0, 2: 1}
|
||||
scheduler._boundary_state_group_ids = frozenset({1})
|
||||
_register_offload_request(scheduler, prefill_end_tokens=800, num_prompt_tokens=800)
|
||||
meta = MooncakeStoreConnectorMetadata(set(), set())
|
||||
|
||||
scheduler._handle_boundary_state_offloads({"req-0": [(2, 7, 800)]}, meta)
|
||||
|
||||
assert meta.requests[0].boundary_state_offloads == [(1, 7, 800)]
|
||||
|
||||
|
||||
def test_resumed_prefill_claims_boundaries_past_prompt_length():
|
||||
# A resumed request re-prefills its previously generated tokens, so its
|
||||
# save window (`prefill_end_tokens`) extends past `num_prompt_tokens`.
|
||||
@@ -1195,6 +1290,7 @@ def test_boundary_state_job_pins_exact_blocks_once():
|
||||
|
||||
def test_store_job_pins_current_non_null_non_mamba_blocks():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler._store_group_ids = (0, 1)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(48)),
|
||||
block_hashes=[bytes([i]) for i in range(12)],
|
||||
|
||||
@@ -46,8 +46,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
)
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
@@ -342,6 +345,43 @@ def _make_kv_cache_config(
|
||||
)
|
||||
|
||||
|
||||
def _make_qsa_hybrid_kv_cache_config():
|
||||
return KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=800,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["qsa"],
|
||||
CircularBufferSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
head_size_v=0,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=800,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _write_mooncake_config(tmp_path, config: dict[str, object]) -> str:
|
||||
config_path = tmp_path / "mooncake_config.json"
|
||||
config_path.write_text(json.dumps(config), encoding="utf-8")
|
||||
@@ -2107,6 +2147,31 @@ def test_recv_thread_reports_unsplittable_key_larger_than_budget():
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
|
||||
|
||||
|
||||
def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
worker.MooncakeStoreConfig,
|
||||
"load_from_config",
|
||||
staticmethod(lambda: _make_config()),
|
||||
)
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.cache_config.block_size = 800
|
||||
vllm_config.cache_config.enable_prefix_caching = True
|
||||
vllm_config.cache_config.prefix_match_unit = None
|
||||
|
||||
store_worker = worker.MooncakeStoreWorker(
|
||||
vllm_config, _make_qsa_hybrid_kv_cache_config()
|
||||
)
|
||||
|
||||
assert [
|
||||
group.kv_cache_spec.block_size for group in store_worker._kv_cache_groups
|
||||
] == [800, 800]
|
||||
assert [db.block_size for db in store_worker.token_dbs] == [800, 800]
|
||||
|
||||
|
||||
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
@@ -4144,6 +4209,31 @@ def test_register_kv_caches_uses_transfer_group_memory_domain():
|
||||
)
|
||||
|
||||
|
||||
def test_register_kv_caches_excludes_nonprefix_cacheable_layers():
|
||||
num_blocks = 2
|
||||
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
store_layout = MagicMock()
|
||||
worker.token_dbs[0].store_layout = store_layout
|
||||
full_cache = torch.zeros(num_blocks, 4, dtype=torch.float16)
|
||||
qsa_cache = torch.zeros(num_blocks, 4, dtype=torch.float16)
|
||||
|
||||
_register_with_mocked_threads(
|
||||
worker,
|
||||
{"layer0": full_cache, "qsa": qsa_cache},
|
||||
)
|
||||
|
||||
store_layout.register_kv_caches.assert_called_once()
|
||||
registered_caches, registered_num_blocks = (
|
||||
store_layout.register_kv_caches.call_args.args
|
||||
)
|
||||
assert registered_num_blocks == num_blocks
|
||||
assert len(registered_caches) == 1
|
||||
assert registered_caches[0] is full_cache
|
||||
worker.store.register_buffer.assert_called_once_with(
|
||||
full_cache.data_ptr(), full_cache.untyped_storage().nbytes()
|
||||
)
|
||||
|
||||
|
||||
def test_register_kv_caches_separate_head_groups():
|
||||
# LHBNC gives each head group its own region; the K/V split doubles heads.
|
||||
layout = KVCacheLayout.LHBNC
|
||||
|
||||
@@ -132,20 +132,22 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
|
||||
|
||||
unsupported: list[str] = []
|
||||
cache_block_size = vllm_config.cache_config.block_size
|
||||
for g_idx, g in enumerate(kv_cache_config.transfer_groups):
|
||||
spec = g.kv_cache_spec
|
||||
store_group_ids = kv_cache_config.prefix_cacheable_group_ids
|
||||
if not store_group_ids:
|
||||
raise ValueError(
|
||||
"MooncakeStore requires at least one prefix-cacheable KV cache group"
|
||||
)
|
||||
for group_id in store_group_ids:
|
||||
spec = kv_cache_config.kv_cache_groups[group_id].kv_cache_spec
|
||||
if isinstance(spec, CrossAttentionSpec):
|
||||
unsupported.append(f"group {g_idx}: CrossAttentionSpec")
|
||||
# Enforce Mamba align mode
|
||||
if isinstance(spec, MambaSpec) and spec.block_size != cache_block_size:
|
||||
unsupported.append(f"group {group_id}: CrossAttentionSpec")
|
||||
if isinstance(spec, MambaSpec) and spec.mamba_cache_mode != "align":
|
||||
unsupported.append(
|
||||
f"group {g_idx}: MambaSpec with block_size="
|
||||
f"{spec.block_size} != cache_config.block_size="
|
||||
f"{cache_block_size} (mamba_cache_mode != 'align')"
|
||||
f"group {group_id}: mamba_cache_mode="
|
||||
f"{spec.mamba_cache_mode!r} != 'align'"
|
||||
)
|
||||
pcp = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
if len(kv_cache_config.transfer_groups) > 1 and pcp > 1:
|
||||
if len(store_group_ids) > 1 and pcp > 1:
|
||||
unsupported.append(f"PCP > 1 (pcp={pcp}) with hybrid attention")
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
|
||||
@@ -452,7 +452,7 @@ def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
|
||||
|
||||
|
||||
def partial_hash_hits_enabled(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
kv_cache_groups: Sequence[KVCacheGroupSpec],
|
||||
hash_block_size: int,
|
||||
dcp_world_size: int = 1,
|
||||
) -> bool:
|
||||
|
||||
@@ -71,19 +71,27 @@ class MooncakeStoreScheduler:
|
||||
)
|
||||
self.client = LookupKeyClient(vllm_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._store_group_ids = kv_cache_config.prefix_cacheable_group_ids
|
||||
# Map scheduler group IDs to store group indices. Groups outside the
|
||||
# store projection, such as the QSA ring, map to None and are skipped.
|
||||
self._store_group_id_by_kv_cache_group_id = {
|
||||
group_id: store_group_id
|
||||
for store_group_id, group_id in enumerate(self._store_group_ids)
|
||||
}
|
||||
store_groups = kv_cache_config.prefix_cacheable_groups
|
||||
|
||||
# Align with the engine's own scheduler_block_size and hash_block_size.
|
||||
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
self.enable_partial_hash_hits = partial_hash_hits_enabled(
|
||||
kv_cache_config.kv_cache_groups,
|
||||
store_groups,
|
||||
self._hash_block_size,
|
||||
vllm_config.parallel_config.decode_context_parallel_size,
|
||||
)
|
||||
mamba_groups = {
|
||||
group_id: group.kv_cache_spec
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
for group_id, group in enumerate(store_groups)
|
||||
if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
}
|
||||
assert all(
|
||||
@@ -173,8 +181,8 @@ class MooncakeStoreScheduler:
|
||||
"""Update state after block allocation."""
|
||||
local_block_ids: tuple[list[int], ...] = ()
|
||||
if num_external_tokens > 0:
|
||||
local_block_ids = self.kv_cache_config.select_transfer_block_ids(
|
||||
blocks.get_block_ids()
|
||||
local_block_ids = blocks.get_block_ids(
|
||||
group_ids=self._store_group_ids,
|
||||
)
|
||||
|
||||
self._unfinished_requests[request.request_id] = (request, local_block_ids)
|
||||
@@ -239,10 +247,7 @@ class MooncakeStoreScheduler:
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
unfolded_block_ids = tuple(
|
||||
blocks.copy()
|
||||
for blocks in self.kv_cache_config.select_transfer_block_ids(
|
||||
request.block_ids
|
||||
)
|
||||
request.block_ids[group_id].copy() for group_id in self._store_group_ids
|
||||
)
|
||||
|
||||
prefill_tokens = _new_req_prefill_tokens(request)
|
||||
@@ -278,8 +283,8 @@ class MooncakeStoreScheduler:
|
||||
for i, req_id in enumerate(cached_reqs.req_ids):
|
||||
new_block_ids = cached_reqs.new_block_ids[i]
|
||||
if new_block_ids:
|
||||
new_block_ids = self.kv_cache_config.select_transfer_block_ids(
|
||||
new_block_ids
|
||||
new_block_ids = tuple(
|
||||
new_block_ids[group_id] for group_id in self._store_group_ids
|
||||
)
|
||||
|
||||
req_meta = None
|
||||
@@ -438,7 +443,9 @@ class MooncakeStoreScheduler:
|
||||
assert block_ids is not None, (
|
||||
f"Missing current block table for store request {req_meta.req_id}"
|
||||
)
|
||||
req_meta.block_ids = block_ids
|
||||
req_meta.block_ids = tuple(
|
||||
block_ids[group_id] for group_id in self._store_group_ids
|
||||
)
|
||||
|
||||
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
|
||||
"""Take a GPU block reference for every store job this step emits.
|
||||
@@ -509,12 +516,15 @@ class MooncakeStoreScheduler:
|
||||
return False
|
||||
|
||||
pinned_block_ids: list[int] = []
|
||||
for group_id, block_id, _ in partial_tail_offloads:
|
||||
if group_id not in self._boundary_state_group_ids:
|
||||
remapped_offloads: list[tuple[int, int, int]] = []
|
||||
for group_id, block_id, boundary in partial_tail_offloads:
|
||||
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
|
||||
if store_group_id not in self._boundary_state_group_ids:
|
||||
return False
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
return False
|
||||
pinned_block_ids.append(block_id)
|
||||
remapped_offloads.append((store_group_id, block_id, boundary))
|
||||
pinned_block_ids = list(dict.fromkeys(pinned_block_ids))
|
||||
|
||||
pool = self._gpu_block_pool
|
||||
@@ -530,12 +540,14 @@ class MooncakeStoreScheduler:
|
||||
self._finished_partial_tail_metas[request.request_id] = ReqMeta(
|
||||
req_id=request.request_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tuple(group.copy() for group in block_ids),
|
||||
block_ids=tuple(
|
||||
block_ids[group_id].copy() for group_id in self._store_group_ids
|
||||
),
|
||||
block_hashes=list(request.block_hashes),
|
||||
can_save=True,
|
||||
num_prompt_tokens=tracker.prefill_end_tokens,
|
||||
store_job_id=store_job_id,
|
||||
boundary_state_offloads=partial_tail_offloads,
|
||||
boundary_state_offloads=remapped_offloads,
|
||||
)
|
||||
tracker.has_pending_offload = True
|
||||
# The store job owns exact block refs, so request cleanup need not wait.
|
||||
@@ -574,9 +586,10 @@ class MooncakeStoreScheduler:
|
||||
continue
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
continue
|
||||
if group_id not in self._boundary_state_group_ids:
|
||||
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
|
||||
if store_group_id not in self._boundary_state_group_ids:
|
||||
continue
|
||||
accepted.append((group_id, block_id, boundary_tokens))
|
||||
accepted.append((store_group_id, block_id, boundary_tokens))
|
||||
if not accepted:
|
||||
continue
|
||||
tracker.has_pending_offload = True
|
||||
|
||||
@@ -1672,10 +1672,11 @@ class MooncakeStoreWorker:
|
||||
dataclasses.replace(
|
||||
group,
|
||||
kv_cache_spec=resolve_dcp_kv_cache_spec(
|
||||
group.kv_cache_spec, self.dcp_size
|
||||
group.kv_cache_spec,
|
||||
self.dcp_size,
|
||||
),
|
||||
)
|
||||
for group in kv_cache_config.transfer_groups
|
||||
for group in kv_cache_config.prefix_cacheable_groups
|
||||
]
|
||||
spec_cfg = getattr(vllm_config, "speculative_config", None)
|
||||
use_eagle_block_drop = bool(
|
||||
@@ -1923,10 +1924,16 @@ class MooncakeStoreWorker:
|
||||
use_group_regions = self._kv_cache_config.hisparse_host_num_blocks is not None
|
||||
|
||||
if not use_group_regions:
|
||||
store_layer_names = {
|
||||
layer_name
|
||||
for group in self._kv_cache_groups
|
||||
for layer_name in group.layer_names
|
||||
}
|
||||
seen_storage_ptrs: set[int] = set()
|
||||
cache_tensors = [
|
||||
group_kernel_blocks(_repr_tensor(cache), self.num_blocks)
|
||||
for cache in kv_caches.values()
|
||||
for layer_name, cache in kv_caches.items()
|
||||
if layer_name in store_layer_names
|
||||
]
|
||||
for cache in cache_tensors:
|
||||
cache_storage = cache.untyped_storage()
|
||||
|
||||
@@ -67,30 +67,45 @@ class KVCacheBlocks:
|
||||
def get_block_ids(
|
||||
self,
|
||||
allow_none: Literal[False] = False,
|
||||
*,
|
||||
group_ids: Sequence[int] | None = None,
|
||||
) -> tuple[list[int], ...]: ...
|
||||
|
||||
@overload
|
||||
def get_block_ids(
|
||||
self,
|
||||
allow_none: Literal[True] = True,
|
||||
*,
|
||||
group_ids: Sequence[int] | None = None,
|
||||
) -> tuple[list[int], ...] | None: ...
|
||||
|
||||
def get_block_ids(
|
||||
self,
|
||||
allow_none: bool = False,
|
||||
*,
|
||||
group_ids: Sequence[int] | None = None,
|
||||
) -> tuple[list[int], ...] | None:
|
||||
"""
|
||||
Converts the KVCacheBlocks instance to block_ids.
|
||||
|
||||
Args:
|
||||
allow_none: Return None when every selected group is empty.
|
||||
group_ids: KV cache groups to include. Includes all groups by default.
|
||||
|
||||
Returns:
|
||||
tuple[list[int], ...]: A tuple of lists where:
|
||||
- the outer tuple corresponds to KV cache groups
|
||||
- each inner list contains the block_ids of the blocks in that
|
||||
group
|
||||
"""
|
||||
if allow_none and all(len(group) == 0 for group in self.blocks):
|
||||
groups = (
|
||||
self.blocks
|
||||
if group_ids is None
|
||||
else tuple(self.blocks[group_id] for group_id in group_ids)
|
||||
)
|
||||
if allow_none and all(len(group) == 0 for group in groups):
|
||||
return None
|
||||
return tuple([blk.block_id for blk in group] for group in self.blocks)
|
||||
return tuple([blk.block_id for blk in group] for group in groups)
|
||||
|
||||
def get_unhashed_block_ids(self) -> list[int]:
|
||||
"""Get block_ids of unhashed blocks from KVCacheBlocks instance."""
|
||||
|
||||
@@ -1450,6 +1450,23 @@ class KVCacheConfig:
|
||||
if group.enable_kv_transfer
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def prefix_cacheable_group_ids(self) -> tuple[int, ...]:
|
||||
"""IDs of transferable groups eligible for hash-addressed stores."""
|
||||
return tuple(
|
||||
group_id
|
||||
for group_id in self.transfer_group_ids
|
||||
if self.kv_cache_groups[group_id].kv_cache_spec.prefix_cacheable
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def prefix_cacheable_groups(self) -> tuple[KVCacheGroupSpec, ...]:
|
||||
"""Transferable groups eligible for hash-addressed stores."""
|
||||
return tuple(
|
||||
self.kv_cache_groups[group_id]
|
||||
for group_id in self.prefix_cacheable_group_ids
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def transfer_groups(self) -> tuple[KVCacheGroupSpec, ...]:
|
||||
"""Cache groups that participate in external KV transfer."""
|
||||
|
||||
Reference in New Issue
Block a user