diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 3b29799d3f..dbddc556c9 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -35,7 +35,7 @@ from vllm.multimodal.inputs import ( from vllm.sampling_params import SamplingParams from vllm.utils.hashing import sha256, sha256_cbor, xxhash, xxhash_cbor from vllm.utils.mem_constants import GiB_bytes -from vllm.v1.core.kv_cache_manager import KVCacheManager +from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager from vllm.v1.core.kv_cache_utils import ( BlockHash, FreeKVCacheBlockQueue, @@ -61,6 +61,7 @@ from vllm.v1.hisparse.layout import ( ) from vllm.v1.kv_cache_interface import ( ChunkedLocalAttentionSpec, + CircularBufferSpec, FullAttentionSpec, HiddenStateCacheSpec, HiSparseHotSpec, @@ -324,6 +325,45 @@ def test_kv_cache_config_selects_only_transferable_groups(): ) +def test_kv_cache_config_selects_prefix_cacheable_groups(): + """Prefix stores exclude scratch state without changing transfer groups.""" + full_group = KVCacheGroupSpec(["full"], new_kv_cache_spec()) + qsa_group = KVCacheGroupSpec( + ["qsa"], + CircularBufferSpec( + block_size=4, + num_kv_heads=1, + head_size=64, + head_size_v=0, + dtype=torch.float16, + ), + ) + disabled_group = KVCacheGroupSpec( + ["disabled"], new_kv_cache_spec(), enable_kv_transfer=False + ) + config = KVCacheConfig( + num_blocks=1, + kv_cache_tensors=[], + kv_cache_groups=[full_group, qsa_group, disabled_group], + ) + assert config.transfer_group_ids == (0, 1) + assert config.select_transfer_block_ids(([1], [2], [3])) == ([1], [2]) + assert config.prefix_cacheable_group_ids == (0,) + assert config.prefix_cacheable_groups == (full_group,) + + +def test_kv_cache_blocks_selects_requested_groups(): + blocks = KVCacheBlocks( + ( + [KVCacheBlock(1)], + [KVCacheBlock(2)], + [KVCacheBlock(3)], + ) + ) + + assert blocks.get_block_ids(group_ids=(0, 2)) == ([1], [3]) + + def new_sliding_window_spec( block_size=16, num_kv_heads=2, diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py index 96315b026d..6afb529d14 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py @@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import MooncakeStoreConnectorStats, ) from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, @@ -91,6 +92,53 @@ def _make_kv_cache_config() -> KVCacheConfig: ) +def _make_qsa_hybrid_kv_cache_config(mamba_mode: str): + full_spec = FullAttentionSpec( + block_size=800, num_kv_heads=8, head_size=64, dtype=None + ) + qsa_spec = CircularBufferSpec( + block_size=8, + num_kv_heads=1, + head_size=64, + head_size_v=0, + dtype=torch.float16, + ) + mamba_spec = MambaSpec( + block_size=800, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode=mamba_mode, + ) + return KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec(["full"], full_spec), + KVCacheGroupSpec(["qsa"], qsa_spec), + KVCacheGroupSpec(["mamba"], mamba_spec), + ], + ) + + +def test_validation_accepts_aligned_mamba_after_block_size_rewrite(): + vllm_config = _make_vllm_config() + vllm_config.cache_config.block_size = 8 + + mooncake_store_connector.MooncakeStoreConnector._validate_kv_cache_config( + vllm_config, _make_qsa_hybrid_kv_cache_config("align") + ) + + +def test_validation_rejects_mamba_mode_directly(): + vllm_config = _make_vllm_config() + vllm_config.cache_config.block_size = 800 + + with pytest.raises(ValueError, match="mamba_cache_mode=\x27none\x27"): + mooncake_store_connector.MooncakeStoreConnector._validate_kv_cache_config( + vllm_config, _make_qsa_hybrid_kv_cache_config("none") + ) + + def test_scheduler_requires_align_mode_for_mamba(): vllm_config = _make_vllm_config() mamba_spec = MambaSpec( diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py index e6b7302429..a93698bba9 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -759,8 +759,9 @@ def test_worker_setup_tolerates_finer_scratch_group(): worker.pp_size = 1 worker.num_kv_head = 8 assert worker.coord.enable_partial_hash_hits - # The scratch DB is keyed at its own block size and never probed. - assert worker.token_dbs[2].hash_block_size == 4 + # Only the participating full-attention and Mamba groups are registered. + assert len(worker.token_dbs) == 2 + assert all(db.hash_block_size == 8 for db in worker.token_dbs) for g_idx, db in enumerate(worker.token_dbs): db.set_kv_caches_base_addr([g_idx * 10_000]) @@ -772,11 +773,11 @@ def test_worker_setup_tolerates_finer_scratch_group(): block_size=worker.block_size, coord=worker.coord, tp_rank=0, - group_put_steps=[1, 1, 1], + group_put_steps=[1, 1], kv_role="kv_both", ready_event=threading.Event(), replicate_config=MagicMock(), - group_participates=[True, True, False], + group_participates=[True, True], ) # Persist the sub-block partial tail at boundary 12 (keyed by hs[12//8-1]). @@ -784,7 +785,7 @@ def test_worker_setup_tolerates_finer_scratch_group(): req = ReqMeta( req_id="r0", token_len_chunk=0, - block_ids=([1], [2], [3]), + block_ids=([1], [2]), block_hashes=hs, can_save=True, num_prompt_tokens=20, @@ -797,8 +798,7 @@ def test_worker_setup_tolerates_finer_scratch_group(): # A 13-token prompt sharing the prefix must hit the first hash unit. assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 8 # The scratch group's namespace never enters the store. - scratch_prefix = worker.token_dbs[2].key_for(hs[0]).rsplit("@", 1)[0] - assert not any(key.startswith(scratch_prefix) for key in store._data) + assert not any("@group:2" in key for key in store._data) def test_ring_scratch_group_is_never_stored_and_does_not_block_hits(): @@ -838,7 +838,8 @@ def test_ring_scratch_group_is_never_stored_and_does_not_block_hits(): worker.tp_size = 1 worker.pp_size = 1 worker.num_kv_head = 8 - assert worker.token_dbs[1].hash_block_size == 8 + assert len(worker.token_dbs) == 1 + assert worker.token_dbs[0].hash_block_size == 16 raw_full = torch.zeros(4 * full.page_size_bytes, dtype=torch.int8) raw_ring = torch.zeros(4 * ring.page_size_bytes, dtype=torch.int8) @@ -879,14 +880,14 @@ def test_ring_scratch_group_is_never_stored_and_does_not_block_hits(): kv_role=worker.kv_role, ready_event=threading.Event(), enable_kv_event=False, - group_participates=[True, False], + group_participates=[True], ) hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)] save_req = ReqMeta( req_id="r0", token_len_chunk=64, - # The ring group holds one block for the request's lifetime. - block_ids=([1, 2, 3, 4], [1]), + # Scheduler metadata is projected to the participating store groups. + block_ids=([1, 2, 3, 4],), block_hashes=hs, can_save=True, store_job_id=1, diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index 67fa0c88aa..7ced38d65a 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -2,12 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from types import SimpleNamespace +from unittest.mock import patch import pytest +import torch from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( LoadSpec, MooncakeLookupResult, + MooncakeStoreConnectorMetadata, MooncakeStoreWorkerMetadata, ReqMeta, RequestTracker, @@ -16,7 +19,16 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor MooncakeStoreScheduler, ) from vllm.v1.core.block_pool import BlockPool +from vllm.v1.core.kv_cache_manager import KVCacheBlocks +from vllm.v1.core.kv_cache_utils import KVCacheBlock from vllm.v1.core.sched.output import KVConnectorBlockState +from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, + MambaSpec, +) def _make_bare_scheduler( @@ -35,9 +47,9 @@ def _make_bare_scheduler( scheduler._block_size = 16 scheduler._hash_block_size = hash_block_size scheduler.enable_partial_hash_hits = enable_partial_hash_hits - scheduler.kv_cache_config = SimpleNamespace( - select_transfer_block_ids=lambda block_ids: tuple(block_ids) - ) + scheduler.kv_cache_config = SimpleNamespace() + scheduler._store_group_ids = (0,) + scheduler._store_group_id_by_kv_cache_group_id = {0: 0, 1: 1} scheduler.load_specs = {} scheduler._unfinished_request_ids = {"req-0"} scheduler._unfinished_requests = {} @@ -228,14 +240,82 @@ def test_pending_load_for_non_chosen_connector_is_dropped(): assert "req-0" not in scheduler._request_trackers +def _make_qsa_hybrid_cache_config(): + full = FullAttentionSpec(block_size=800, num_kv_heads=8, head_size=64, dtype=None) + circular = CircularBufferSpec( + block_size=8, + num_kv_heads=1, + head_size=64, + head_size_v=0, + dtype=torch.float16, + ) + mamba = MambaSpec( + block_size=800, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + return KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec(["full"], full), + KVCacheGroupSpec(["qsa"], circular), + KVCacheGroupSpec(["mamba"], mamba), + ], + ) + + +def test_scheduler_projects_nonprefix_groups_and_mamba_ids(): + vllm_config = SimpleNamespace( + kv_transfer_config=SimpleNamespace( + kv_role="kv_both", kv_connector_extra_config={} + ), + kv_events_config=None, + cache_config=SimpleNamespace( + block_size=800, enable_prefix_caching=True, prefix_match_unit=None + ), + parallel_config=SimpleNamespace(decode_context_parallel_size=1, world_size=1), + ) + + with patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." + "scheduler.LookupKeyClient" + ): + scheduler = MooncakeStoreScheduler(vllm_config, _make_qsa_hybrid_cache_config()) + + assert scheduler._store_group_ids == (0, 2) + assert scheduler._boundary_state_group_ids == frozenset({1}) + + +def test_current_save_block_ids_use_store_group_projection(): + scheduler = _make_bare_scheduler() + scheduler.kv_cache_config = _make_qsa_hybrid_cache_config() + scheduler._store_group_ids = (0, 2) + meta = MooncakeStoreConnectorMetadata(set(), set()) + req_meta = ReqMeta( + req_id="req-0", + token_len_chunk=800, + block_ids=(), + block_hashes=[b"h0"], + can_save=True, + ) + meta.add_request(req_meta) + output = SimpleNamespace( + kv_connector_block_state=_make_connector_block_state(([10], [80], [30])) + ) + + scheduler._apply_current_save_block_ids(meta, output) + + assert req_meta.block_ids == ([10], [30]) + + def test_update_state_excludes_nontransfer_groups(): """Store metadata must match the worker's registered cache groups.""" scheduler = _make_bare_scheduler() - scheduler.kv_cache_config = SimpleNamespace( - select_transfer_block_ids=lambda block_ids: (block_ids[0],) - ) + scheduler._store_group_ids = (0,) request = SimpleNamespace(request_id="req-1") - blocks = SimpleNamespace(get_block_ids=lambda: ([1, 2], [9])) + blocks = KVCacheBlocks(([KVCacheBlock(1), KVCacheBlock(2)], [KVCacheBlock(9)])) scheduler.update_state_after_alloc(request, blocks, num_external_tokens=32) @@ -1003,6 +1083,7 @@ def test_pending_partial_tail_emits_offload_only_reqmeta(): def test_finished_partial_tail_is_pre_pinned_as_store_job(): scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + scheduler._store_group_ids = (0, 1) scheduler.client = SimpleNamespace(discard=lambda *_: None) request = SimpleNamespace( request_id="req-0", @@ -1143,6 +1224,20 @@ def _make_offload_only_output(entries, block_ids=([0],)): ) +def test_boundary_state_group_ids_are_remapped_to_store_projection(): + scheduler = _make_bare_scheduler(hash_block_size=800, enable_partial_hash_hits=True) + scheduler.kv_cache_config = _make_qsa_hybrid_cache_config() + scheduler._store_group_ids = (0, 2) + scheduler._store_group_id_by_kv_cache_group_id = {0: 0, 2: 1} + scheduler._boundary_state_group_ids = frozenset({1}) + _register_offload_request(scheduler, prefill_end_tokens=800, num_prompt_tokens=800) + meta = MooncakeStoreConnectorMetadata(set(), set()) + + scheduler._handle_boundary_state_offloads({"req-0": [(2, 7, 800)]}, meta) + + assert meta.requests[0].boundary_state_offloads == [(1, 7, 800)] + + def test_resumed_prefill_claims_boundaries_past_prompt_length(): # A resumed request re-prefills its previously generated tokens, so its # save window (`prefill_end_tokens`) extends past `num_prompt_tokens`. @@ -1195,6 +1290,7 @@ def test_boundary_state_job_pins_exact_blocks_once(): def test_store_job_pins_current_non_null_non_mamba_blocks(): scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + scheduler._store_group_ids = (0, 1) request = SimpleNamespace( all_token_ids=list(range(48)), block_hashes=[bytes([i]) for i in range(12)], diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index f613ba720e..9baf16e261 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -46,8 +46,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import from vllm.v1.attention.backends.utils import NULL_BLOCK_ID from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, + KVCacheConfig, KVCacheGroupSpec, + MambaSpec, RSWASpec, ) from vllm.v1.kv_cache_layout import KVCacheLayout @@ -342,6 +345,43 @@ def _make_kv_cache_config( ) +def _make_qsa_hybrid_kv_cache_config(): + return KVCacheConfig( + num_blocks=10, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=800, + num_kv_heads=8, + head_size=64, + dtype=None, + ), + ), + KVCacheGroupSpec( + ["qsa"], + CircularBufferSpec( + block_size=8, + num_kv_heads=1, + head_size=64, + head_size_v=0, + dtype=torch.float16, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=800, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ), + ), + ], + ) + + def _write_mooncake_config(tmp_path, config: dict[str, object]) -> str: config_path = tmp_path / "mooncake_config.json" config_path.write_text(json.dumps(config), encoding="utf-8") @@ -2107,6 +2147,31 @@ def test_recv_thread_reports_unsplittable_key_larger_than_budget(): assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2} +def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch): + store = MagicMock() + store.setup.return_value = 0 + _install_fake_mooncake(monkeypatch, store) + _patch_worker_runtime(monkeypatch) + monkeypatch.setattr( + worker.MooncakeStoreConfig, + "load_from_config", + staticmethod(lambda: _make_config()), + ) + vllm_config = _make_vllm_config() + vllm_config.cache_config.block_size = 800 + vllm_config.cache_config.enable_prefix_caching = True + vllm_config.cache_config.prefix_match_unit = None + + store_worker = worker.MooncakeStoreWorker( + vllm_config, _make_qsa_hybrid_kv_cache_config() + ) + + assert [ + group.kv_cache_spec.block_size for group in store_worker._kv_cache_groups + ] == [800, 800] + assert [db.block_size for db in store_worker.token_dbs] == [800, 800] + + def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch): store = MagicMock() store.setup.return_value = 0 @@ -4144,6 +4209,31 @@ def test_register_kv_caches_uses_transfer_group_memory_domain(): ) +def test_register_kv_caches_excludes_nonprefix_cacheable_layers(): + num_blocks = 2 + worker = _make_bare_worker(num_gpu_blocks=num_blocks) + store_layout = MagicMock() + worker.token_dbs[0].store_layout = store_layout + full_cache = torch.zeros(num_blocks, 4, dtype=torch.float16) + qsa_cache = torch.zeros(num_blocks, 4, dtype=torch.float16) + + _register_with_mocked_threads( + worker, + {"layer0": full_cache, "qsa": qsa_cache}, + ) + + store_layout.register_kv_caches.assert_called_once() + registered_caches, registered_num_blocks = ( + store_layout.register_kv_caches.call_args.args + ) + assert registered_num_blocks == num_blocks + assert len(registered_caches) == 1 + assert registered_caches[0] is full_cache + worker.store.register_buffer.assert_called_once_with( + full_cache.data_ptr(), full_cache.untyped_storage().nbytes() + ) + + def test_register_kv_caches_separate_head_groups(): # LHBNC gives each head group its own region; the K/V split doubles heads. layout = KVCacheLayout.LHBNC diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index 2c8d118cdd..134d5055b6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -132,20 +132,22 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec unsupported: list[str] = [] - cache_block_size = vllm_config.cache_config.block_size - for g_idx, g in enumerate(kv_cache_config.transfer_groups): - spec = g.kv_cache_spec + store_group_ids = kv_cache_config.prefix_cacheable_group_ids + if not store_group_ids: + raise ValueError( + "MooncakeStore requires at least one prefix-cacheable KV cache group" + ) + for group_id in store_group_ids: + spec = kv_cache_config.kv_cache_groups[group_id].kv_cache_spec if isinstance(spec, CrossAttentionSpec): - unsupported.append(f"group {g_idx}: CrossAttentionSpec") - # Enforce Mamba align mode - if isinstance(spec, MambaSpec) and spec.block_size != cache_block_size: + unsupported.append(f"group {group_id}: CrossAttentionSpec") + if isinstance(spec, MambaSpec) and spec.mamba_cache_mode != "align": unsupported.append( - f"group {g_idx}: MambaSpec with block_size=" - f"{spec.block_size} != cache_config.block_size=" - f"{cache_block_size} (mamba_cache_mode != 'align')" + f"group {group_id}: mamba_cache_mode=" + f"{spec.mamba_cache_mode!r} != 'align'" ) pcp = vllm_config.parallel_config.prefill_context_parallel_size - if len(kv_cache_config.transfer_groups) > 1 and pcp > 1: + if len(store_group_ids) > 1 and pcp > 1: unsupported.append(f"PCP > 1 (pcp={pcp}) with hybrid attention") if unsupported: raise ValueError( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py index 375730807d..d05b553b64 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py @@ -452,7 +452,7 @@ def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec: def partial_hash_hits_enabled( - kv_cache_groups: list[KVCacheGroupSpec], + kv_cache_groups: Sequence[KVCacheGroupSpec], hash_block_size: int, dcp_world_size: int = 1, ) -> bool: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index ae61710e25..8f277d7eea 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -71,19 +71,27 @@ class MooncakeStoreScheduler: ) self.client = LookupKeyClient(vllm_config) self.kv_cache_config = kv_cache_config + self._store_group_ids = kv_cache_config.prefix_cacheable_group_ids + # Map scheduler group IDs to store group indices. Groups outside the + # store projection, such as the QSA ring, map to None and are skipped. + self._store_group_id_by_kv_cache_group_id = { + group_id: store_group_id + for store_group_id, group_id in enumerate(self._store_group_ids) + } + store_groups = kv_cache_config.prefix_cacheable_groups # Align with the engine's own scheduler_block_size and hash_block_size. self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes( kv_cache_config, vllm_config ) self.enable_partial_hash_hits = partial_hash_hits_enabled( - kv_cache_config.kv_cache_groups, + store_groups, self._hash_block_size, vllm_config.parallel_config.decode_context_parallel_size, ) mamba_groups = { group_id: group.kv_cache_spec - for group_id, group in enumerate(kv_cache_config.kv_cache_groups) + for group_id, group in enumerate(store_groups) if isinstance(group.kv_cache_spec, MambaSpec) } assert all( @@ -173,8 +181,8 @@ class MooncakeStoreScheduler: """Update state after block allocation.""" local_block_ids: tuple[list[int], ...] = () if num_external_tokens > 0: - local_block_ids = self.kv_cache_config.select_transfer_block_ids( - blocks.get_block_ids() + local_block_ids = blocks.get_block_ids( + group_ids=self._store_group_ids, ) self._unfinished_requests[request.request_id] = (request, local_block_ids) @@ -239,10 +247,7 @@ class MooncakeStoreScheduler: request_real = request_tuple[0] # type: ignore[index] unfolded_block_ids = tuple( - blocks.copy() - for blocks in self.kv_cache_config.select_transfer_block_ids( - request.block_ids - ) + request.block_ids[group_id].copy() for group_id in self._store_group_ids ) prefill_tokens = _new_req_prefill_tokens(request) @@ -278,8 +283,8 @@ class MooncakeStoreScheduler: for i, req_id in enumerate(cached_reqs.req_ids): new_block_ids = cached_reqs.new_block_ids[i] if new_block_ids: - new_block_ids = self.kv_cache_config.select_transfer_block_ids( - new_block_ids + new_block_ids = tuple( + new_block_ids[group_id] for group_id in self._store_group_ids ) req_meta = None @@ -438,7 +443,9 @@ class MooncakeStoreScheduler: assert block_ids is not None, ( f"Missing current block table for store request {req_meta.req_id}" ) - req_meta.block_ids = block_ids + req_meta.block_ids = tuple( + block_ids[group_id] for group_id in self._store_group_ids + ) def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None: """Take a GPU block reference for every store job this step emits. @@ -509,12 +516,15 @@ class MooncakeStoreScheduler: return False pinned_block_ids: list[int] = [] - for group_id, block_id, _ in partial_tail_offloads: - if group_id not in self._boundary_state_group_ids: + remapped_offloads: list[tuple[int, int, int]] = [] + for group_id, block_id, boundary in partial_tail_offloads: + store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id) + if store_group_id not in self._boundary_state_group_ids: return False if block_id == NULL_BLOCK_ID: return False pinned_block_ids.append(block_id) + remapped_offloads.append((store_group_id, block_id, boundary)) pinned_block_ids = list(dict.fromkeys(pinned_block_ids)) pool = self._gpu_block_pool @@ -530,12 +540,14 @@ class MooncakeStoreScheduler: self._finished_partial_tail_metas[request.request_id] = ReqMeta( req_id=request.request_id, token_len_chunk=0, - block_ids=tuple(group.copy() for group in block_ids), + block_ids=tuple( + block_ids[group_id].copy() for group_id in self._store_group_ids + ), block_hashes=list(request.block_hashes), can_save=True, num_prompt_tokens=tracker.prefill_end_tokens, store_job_id=store_job_id, - boundary_state_offloads=partial_tail_offloads, + boundary_state_offloads=remapped_offloads, ) tracker.has_pending_offload = True # The store job owns exact block refs, so request cleanup need not wait. @@ -574,9 +586,10 @@ class MooncakeStoreScheduler: continue if block_id == NULL_BLOCK_ID: continue - if group_id not in self._boundary_state_group_ids: + store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id) + if store_group_id not in self._boundary_state_group_ids: continue - accepted.append((group_id, block_id, boundary_tokens)) + accepted.append((store_group_id, block_id, boundary_tokens)) if not accepted: continue tracker.has_pending_offload = True diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index ebcf5708c8..6f154ea991 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -1672,10 +1672,11 @@ class MooncakeStoreWorker: dataclasses.replace( group, kv_cache_spec=resolve_dcp_kv_cache_spec( - group.kv_cache_spec, self.dcp_size + group.kv_cache_spec, + self.dcp_size, ), ) - for group in kv_cache_config.transfer_groups + for group in kv_cache_config.prefix_cacheable_groups ] spec_cfg = getattr(vllm_config, "speculative_config", None) use_eagle_block_drop = bool( @@ -1923,10 +1924,16 @@ class MooncakeStoreWorker: use_group_regions = self._kv_cache_config.hisparse_host_num_blocks is not None if not use_group_regions: + store_layer_names = { + layer_name + for group in self._kv_cache_groups + for layer_name in group.layer_names + } seen_storage_ptrs: set[int] = set() cache_tensors = [ group_kernel_blocks(_repr_tensor(cache), self.num_blocks) - for cache in kv_caches.values() + for layer_name, cache in kv_caches.items() + if layer_name in store_layer_names ] for cache in cache_tensors: cache_storage = cache.untyped_storage() diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 20e037f60e..226da3a1ed 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -67,30 +67,45 @@ class KVCacheBlocks: def get_block_ids( self, allow_none: Literal[False] = False, + *, + group_ids: Sequence[int] | None = None, ) -> tuple[list[int], ...]: ... @overload def get_block_ids( self, allow_none: Literal[True] = True, + *, + group_ids: Sequence[int] | None = None, ) -> tuple[list[int], ...] | None: ... def get_block_ids( self, allow_none: bool = False, + *, + group_ids: Sequence[int] | None = None, ) -> tuple[list[int], ...] | None: """ Converts the KVCacheBlocks instance to block_ids. + Args: + allow_none: Return None when every selected group is empty. + group_ids: KV cache groups to include. Includes all groups by default. + Returns: tuple[list[int], ...]: A tuple of lists where: - the outer tuple corresponds to KV cache groups - each inner list contains the block_ids of the blocks in that group """ - if allow_none and all(len(group) == 0 for group in self.blocks): + groups = ( + self.blocks + if group_ids is None + else tuple(self.blocks[group_id] for group_id in group_ids) + ) + if allow_none and all(len(group) == 0 for group in groups): return None - return tuple([blk.block_id for blk in group] for group in self.blocks) + return tuple([blk.block_id for blk in group] for group in groups) def get_unhashed_block_ids(self) -> list[int]: """Get block_ids of unhashed blocks from KVCacheBlocks instance.""" diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index e2869e047f..8cd867eb9c 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -1450,6 +1450,23 @@ class KVCacheConfig: if group.enable_kv_transfer ) + @cached_property + def prefix_cacheable_group_ids(self) -> tuple[int, ...]: + """IDs of transferable groups eligible for hash-addressed stores.""" + return tuple( + group_id + for group_id in self.transfer_group_ids + if self.kv_cache_groups[group_id].kv_cache_spec.prefix_cacheable + ) + + @cached_property + def prefix_cacheable_groups(self) -> tuple[KVCacheGroupSpec, ...]: + """Transferable groups eligible for hash-addressed stores.""" + return tuple( + self.kv_cache_groups[group_id] + for group_id in self.prefix_cacheable_group_ids + ) + @cached_property def transfer_groups(self) -> tuple[KVCacheGroupSpec, ...]: """Cache groups that participate in external KV transfer."""