diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py index 46a70654b1..2ea776adf6 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -173,7 +173,6 @@ def test_e2e_swa_plus_full_save_then_lookup_hits(): worker = _build_worker_with_dict_store(vllm_config, cfg, store) worker.tp_size = 1 worker.pp_size = 1 - worker.put_step = 1 worker.num_kv_head = 8 # Register kv_caches using mocked thread classes so register_kv_caches @@ -215,7 +214,7 @@ def test_e2e_swa_plus_full_save_then_lookup_hits(): block_size=worker.block_size, coord=worker.coord, tp_rank=worker.tp_rank, - put_step=worker.put_step, + group_put_steps=worker._group_tp_replication_factors, kv_role=worker.kv_role, ready_event=ready, enable_kv_event=False, diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index ec47bfa238..7cddd6d4a7 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -90,7 +90,7 @@ def _make_store_sending_thread( block_size=block_size, coord=coord, tp_rank=tp_rank, - put_step=put_step, + group_put_steps=[put_step] * len(token_databases), kv_role="kv_producer", ready_event=threading.Event(), replicate_config=replicate_config, @@ -527,6 +527,29 @@ def test_store_sending_thread_delta_strides_with_local_phase(): assert store.batch_put_from_multi_buffers.call_args.args[0] == keys +def test_tp_sharded_group_saves_every_block_on_every_rank(): + """Sharded ranks must write every block because peers hold different bytes.""" + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys) + thread = _make_store_sending_thread(store, tp_rank=0, put_step=2) + thread.group_put_steps = [1] + + thread.add_stored_request("req-a") + thread._handle_request( + ReqMeta( + req_id="req-a", + token_len_chunk=64, + block_ids=([0, 1, 2, 3],), + block_hashes=[b"a0", b"a1", b"a2", b"a3"], + can_save=True, + ) + ) + + keys = store.batch_is_exist.call_args.args[0] + assert len(keys) == 4 + + def test_store_sending_thread_retries_skipped_range_after_pressure(): store = MagicMock() store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) @@ -1274,7 +1297,8 @@ def test_worker_put_striding_covers_every_rank_get_namespace( ] assert len(keys) == len(block_hashes) # PUT side: mirrors KVCacheStoreSendingThread's striding slice. - put_keys.update(keys[w.tp_rank % w.put_step :: w.put_step]) + put_step = w._group_tp_replication_factors[0] + put_keys.update(keys[w.tp_rank % put_step :: put_step]) # GET side: KVCacheStoreRecvingThread fetches every key. get_keys_per_rank[tp_rank] = set(keys) @@ -1613,6 +1637,15 @@ def _register_with_mocked_threads( worker.register_kv_caches(kv_caches) +def _refresh_group_tp_replication_factors( + worker: mooncake_store_worker.MooncakeStoreWorker, +) -> None: + worker._group_tp_replication_factors = ( + worker._compute_group_tp_replication_factors() + ) + worker._init_lookup_key_prefixes() + + def _make_bare_worker( *, num_gpu_blocks: int = 10, @@ -1630,11 +1663,9 @@ def _make_bare_worker( worker.cache_config.num_gpu_blocks = num_gpu_blocks worker.store = MagicMock() worker.store.register_buffer.return_value = 0 - worker.use_mla = False worker.kv_role = kv_role worker.block_size = block_size worker.tp_rank = 0 - worker.put_step = 1 worker.enable_kv_events = False worker.kv_send_thread = None worker.kv_recv_threads = [] @@ -1664,7 +1695,6 @@ def _make_bare_worker( worker.pcp_size = 1 worker.dcp_size = 1 worker.hash_block_size = block_size - worker.metadata = KeyMetadata("test-model", 0, 0, 0, 0) # Pre-build a single-group token_dbs so lookup-only tests don't have to # call register_kv_caches. worker.token_dbs = [ @@ -1679,7 +1709,7 @@ def _make_bare_worker( scheduler_block_size=block_size, hash_block_size=block_size, ) - worker._init_lookup_key_prefixes() + _refresh_group_tp_replication_factors(worker) return worker @@ -1688,9 +1718,8 @@ def test_lookup_key_prefixes_cover_dcp_rank_namespaces(): worker.tp_size = 4 worker.num_kv_head = 1 worker.dcp_size = 4 - worker._init_lookup_key_prefixes() + _refresh_group_tp_replication_factors(worker) - assert worker._lookup_expected_per_key == 4 assert worker._lookup_key_prefixes[0] == ( "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0", "test-model@tp_rank:1@pcp0@dcp1@pp_rank:0@group:0", @@ -1705,21 +1734,192 @@ def test_lookup_key_prefixes_cover_pcp_rank_namespaces(): worker.num_kv_head = 1 worker.pcp_size = 2 worker.dcp_size = 1 - worker._init_lookup_key_prefixes() + _refresh_group_tp_replication_factors(worker) - assert worker._lookup_expected_per_key == 2 assert worker._lookup_key_prefixes[0] == ( "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0", "test-model@tp_rank:0@pcp1@dcp0@pp_rank:0@group:0", ) +def test_lookup_key_prefixes_expand_tp_sharded_groups_per_rank(): + """Replicated attention needs one namespace; sharded Mamba needs every rank.""" + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + MambaSpec, + ) + + worker = _make_bare_worker(block_size=16) + worker.tp_size = 2 + worker.num_kv_head = 1 + fa = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + mamba = MambaSpec( + block_size=16, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + worker._kv_cache_groups = [ + KVCacheGroupSpec(["l0"], fa), + KVCacheGroupSpec(["l1"], mamba), + ] + worker.token_dbs = [ + ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), block_size=16 + ), + ChunkedTokenDatabase( + KeyMetadata("test-model", 1, 0, 0, 0, group_id=1), block_size=16 + ), + ] + _refresh_group_tp_replication_factors(worker) + + assert worker._lookup_key_prefixes[0] == ( + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0", + ) + assert worker._lookup_key_prefixes[1] == ( + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1", + "test-model@tp_rank:1@pcp0@dcp0@pp_rank:0@group:1", + ) + + +def test_group_tp_replication_factors_mixed_mla_gqa_mamba(): + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + MambaSpec, + MLAAttentionSpec, + ) + + worker = _make_bare_worker(block_size=16) + worker.tp_size = 4 + worker.num_kv_head = 2 + mla = MLAAttentionSpec(block_size=16, num_kv_heads=1, head_size=64, dtype=None) + gqa = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + mamba = MambaSpec( + block_size=16, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + worker._kv_cache_groups = [ + KVCacheGroupSpec(["l0"], mla), + KVCacheGroupSpec(["l1"], gqa), + KVCacheGroupSpec(["l2"], mamba), + ] + worker.token_dbs = [ + ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=g_idx), block_size=16 + ) + for g_idx in range(3) + ] + + _refresh_group_tp_replication_factors(worker) + assert worker._group_tp_replication_factors == (4, 2, 1) + assert worker._lookup_key_prefixes[0] == ( + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0", + ) + assert worker._lookup_key_prefixes[1] == ( + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1", + "test-model@tp_rank:1@pcp0@dcp0@pp_rank:0@group:1", + ) + assert worker._lookup_key_prefixes[2] == ( + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:2", + "test-model@tp_rank:1@pcp0@dcp0@pp_rank:0@group:2", + "test-model@tp_rank:2@pcp0@dcp0@pp_rank:0@group:2", + "test-model@tp_rank:3@pcp0@dcp0@pp_rank:0@group:2", + ) + + +@pytest.mark.parametrize("spec_order", [("mla", "gqa"), ("gqa", "mla")]) +def test_uniform_group_uses_common_inner_replication_factor(spec_order): + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + MLAAttentionSpec, + UniformTypeKVCacheSpecs, + ) + + worker = _make_bare_worker(block_size=16) + worker.tp_size = 4 + worker.num_kv_head = 2 + specs_by_name = { + "mla": MLAAttentionSpec( + block_size=16, num_kv_heads=1, head_size=64, dtype=None + ), + "gqa": FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=64, dtype=None + ), + } + inner_specs = {name: specs_by_name[name] for name in spec_order} + uniform_spec = UniformTypeKVCacheSpecs( + block_size=16, + kv_cache_specs=inner_specs, + ) + worker._kv_cache_groups = [ + KVCacheGroupSpec(list(inner_specs), uniform_spec), + ] + + _refresh_group_tp_replication_factors(worker) + + assert worker._group_tp_replication_factors == (2,) + assert worker._lookup_key_prefixes[0] == ( + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0", + "test-model@tp_rank:1@pcp0@dcp0@pp_rank:0@group:0", + ) + + +def test_lookup_rejects_boundary_missing_one_mamba_shard(): + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + MambaSpec, + ) + + worker = _make_bare_worker(block_size=16) + worker.tp_size = 2 + worker.num_kv_head = 1 + fa = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + mamba = MambaSpec( + block_size=16, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + worker._kv_cache_groups = [ + KVCacheGroupSpec(["l0"], fa), + KVCacheGroupSpec(["l1"], mamba), + ] + worker.token_dbs = [ + ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), block_size=16 + ), + ChunkedTokenDatabase( + KeyMetadata("test-model", 1, 0, 0, 0, group_id=1), block_size=16 + ), + ] + worker.coord = mooncake_store_worker.MooncakeStoreCoordinator( + worker._kv_cache_groups, + scheduler_block_size=16, + hash_block_size=16, + ) + _refresh_group_tp_replication_factors(worker) + + worker.store.batch_is_exist.side_effect = lambda keys: [1] * len(keys) + assert worker.lookup(32, [b"h0", b"h1"]) == 32 + + worker.store.batch_is_exist.side_effect = lambda keys: [ + 0 if "tp_rank:1" in k and "group:1" in k else 1 for k in keys + ] + assert worker.lookup(32, [b"h0", b"h1"]) == 0 + + def test_lookup_requires_all_dcp_rank_namespaces(): worker = _make_bare_worker(block_size=16) worker.tp_size = 4 worker.num_kv_head = 1 worker.dcp_size = 4 - worker._init_lookup_key_prefixes() + _refresh_group_tp_replication_factors(worker) worker.store.batch_is_exist.return_value = [1, 1, 0, 1] assert worker.lookup(16, [b"a0"]) == 0 @@ -1850,7 +2050,7 @@ def test_lookup_checks_all_potential_swa_hit_boundaries(): hash_block_size=8, retention_interval=0, ) - worker._init_lookup_key_prefixes() + _refresh_group_tp_replication_factors(worker) # Candidate order: 3 full-attention chunks, then SWA chunks 3, 7, 11. # Only the first full chunk and the SWA chunk ending at token 32 exist, so # lookup should recover a 32-token external prefix hit. A sparse @@ -1911,7 +2111,7 @@ def test_lookup_applies_swa_mask_before_accessing_hashes(): hash_block_size=8, retention_interval=0, ) - worker._init_lookup_key_prefixes() + _refresh_group_tp_replication_factors(worker) block_hashes = _RecordingBlockHashes([f"h{i}".encode() for i in range(12)]) accessed_before_rpc: list[int] = [] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 03b2589c07..cc49a3569d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -66,7 +66,15 @@ from vllm.v1.core.kv_cache_utils import ( maybe_convert_block_hash, resolve_kv_cache_block_sizes, ) -from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec +from vllm.v1.kv_cache_interface import ( + KVCacheConfig, + KVCacheGroupSpec, + KVCacheSpec, + MambaSpec, + MLAAttentionSpec, + SlidingWindowMLASpec, + UniformTypeKVCacheSpecs, +) from .metrics import MooncakeStoreConnectorStats @@ -448,7 +456,7 @@ class KVCacheStoreSendingThread(KVTransferThread): token_databases: list[ChunkedTokenDatabase], block_size: int, tp_rank: int, - put_step: int, + group_put_steps: Sequence[int], kv_role: str, ready_event: threading.Event, enable_kv_event: bool = False, @@ -464,7 +472,8 @@ class KVCacheStoreSendingThread(KVTransferThread): name="KVCacheStoreSendingThread", record_operation=record_operation, ) - self.put_step = put_step + # Only ranks with identical group bytes may stripe PUTs (e.g., MLA). + self.group_put_steps = group_put_steps self.coord = coord self.kv_role = kv_role self.stored_requests: defaultdict[str, int] = defaultdict(int) @@ -568,13 +577,14 @@ class KVCacheStoreSendingThread(KVTransferThread): group_indices: list[int] = [] for g_idx, db in enumerate(self.token_databases): # Rotate the stride phase per group to balance load across ranks. - put_step_rank = (self.tp_rank + g_idx) % self.put_step + put_step = self.group_put_steps[g_idx] + put_step_rank = (self.tp_rank + g_idx) % put_step for start, end, block_hash in db.process_tokens( token_len, req_meta.block_hashes, mask_num=save_start, chunk_mask=store_masks[g_idx], - put_step=self.put_step, + put_step=put_step, put_step_rank=put_step_rank, ): starts.append(start) @@ -1002,44 +1012,7 @@ class MooncakeStoreWorker: ) self.num_layers = model_config.get_num_layers(parallel_config) - self.use_mla = False - if ( - hasattr(model_config, "use_mla") - and isinstance(model_config.use_mla, bool) - and model_config.use_mla - ): - self.use_mla = True - - if self.use_mla: - self.num_kv_head = 1 - else: - self.num_kv_head = model_config.get_total_num_kv_heads() - - if self.num_kv_head < self.tp_size and self.dcp_size <= 1: - # Dedup: TP ranks holding the same KV heads stripe PUTs across - # one shared key namespace. DCP splits the TP group, so with - # DCP>1 those ranks have different `@dcpN` namespaces and - # striping would leave keys unwritten (OBJECT_NOT_FOUND on - # GET). PCP is outer to TP (pcp_rank is constant within a TP - # group), so it needs no guard. - self.put_step = self.tp_size // self.num_kv_head - self.head_or_tp_rank = self.tp_rank // self.put_step - else: - self.head_or_tp_rank = self.tp_rank - self.put_step = 1 - - self.metadata = KeyMetadata( - model_name=model_config.model.rstrip("/").split("/")[-1], - tp_rank=self.head_or_tp_rank, - pcp_rank=self.pcp_rank, - dcp_rank=self.dcp_rank, - pp_rank=self.pp_rank, - cache_prefix=str( - vllm_config.kv_transfer_config.kv_connector_extra_config.get( - "cache_prefix", "" - ) - ), - ) + self.num_kv_head = model_config.get_total_num_kv_heads() # Initialize MooncakeDistributedStore with its own TransferEngine store_config = MooncakeStoreConfig.load_from_config() @@ -1156,10 +1129,32 @@ class MooncakeStoreWorker: retention_interval=envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL, ) # One ChunkedTokenDatabase per group; addresses populated in - # register_kv_caches once the kv-cache layout is known. + # register_kv_caches once the kv-cache layout is known. Each group's + # key namespace is its TP shard id: ranks holding identical bytes + # (MLA / shared GQA KV heads) share a namespace, TP-sharded Mamba + # state gets one namespace per rank. + metadata = KeyMetadata( + model_name=model_config.model.rstrip("/").split("/")[-1], + tp_rank=self.tp_rank, + pcp_rank=self.pcp_rank, + dcp_rank=self.dcp_rank, + pp_rank=self.pp_rank, + cache_prefix=str( + vllm_config.kv_transfer_config.kv_connector_extra_config.get( + "cache_prefix", "" + ) + ), + ) + self._group_tp_replication_factors: tuple[int, ...] = ( + self._compute_group_tp_replication_factors() + ) self.token_dbs: list[ChunkedTokenDatabase] = [ ChunkedTokenDatabase( - dataclasses.replace(self.metadata, group_id=g_idx), + dataclasses.replace( + metadata, + group_id=g_idx, + tp_rank=self.tp_rank // self._group_tp_replication_factors[g_idx], + ), g.kv_cache_spec.block_size, hash_block_size=self.hash_block_size, ) @@ -1167,27 +1162,50 @@ class MooncakeStoreWorker: ] self._init_lookup_key_prefixes() - def _init_lookup_key_prefixes(self) -> None: - """Prepare per-group key prefixes across parallel rank namespaces.""" - # (tp_rank, pcp_rank, dcp_rank, pp_rank) namespaces + def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int: if self.dcp_size > 1: - # DCP reuses the TP workers and splits each TP group into - # contiguous DCP groups, so dcp_rank == tp_rank % dcp_size. - # Store/load paths do not apply KV-head dedup under DCP - rank_namespaces = tuple( - (tp_rank, pcp_rank, tp_rank % self.dcp_size, pp_rank) + return 1 + inner_specs = ( + tuple(spec.kv_cache_specs.values()) + if isinstance(spec, UniformTypeKVCacheSpecs) + else (spec,) + ) + # Any rank-specific state makes the whole packed value rank-specific. + if any(isinstance(inner, MambaSpec) for inner in inner_specs): + return 1 + # A pure MLA packed value is replicated on every TP rank. + if all( + isinstance(inner, (MLAAttentionSpec, SlidingWindowMLASpec)) + for inner in inner_specs + ): + return self.tp_size + return max(1, self.tp_size // self.num_kv_head) + + def _compute_group_tp_replication_factors(self) -> tuple[int, ...]: + """Return the number of byte-identical TP replicas per cache group. + + DCP and Mamba use 1; MLA uses ``tp_size``; GQA uses + ``tp_size // num_kv_head``. + """ + return tuple( + self._spec_tp_replication_factor(group.kv_cache_spec) + for group in self._kv_cache_groups + ) + + def _init_lookup_key_prefixes(self) -> None: + def rank_namespaces(factor: int) -> tuple[tuple[int, int, int, int], ...]: + if self.dcp_size > 1: + # DCP is a TP subdivision: dcp_rank == tp_rank % dcp_size. + return tuple( + (tp_rank, pcp_rank, tp_rank % self.dcp_size, pp_rank) + for pcp_rank in range(self.pcp_size) + for tp_rank in range(self.tp_size) + for pp_rank in range(self.pp_size) + ) + return tuple( + (shard_rank, pcp_rank, 0, pp_rank) for pcp_rank in range(self.pcp_size) - for tp_rank in range(self.tp_size) - for pp_rank in range(self.pp_size) - ) - else: - # Without DCP, TP ranks that share a KV head write identical KV, so - # lookup only needs one TP namespace per unique KV head. - tp_count = min(self.tp_size, self.num_kv_head) - rank_namespaces = tuple( - (tp_rank, pcp_rank, 0, pp_rank) - for pcp_rank in range(self.pcp_size) - for tp_rank in range(tp_count) + for shard_rank in range(self.tp_size // factor) for pp_rank in range(self.pp_size) ) @@ -1200,11 +1218,12 @@ class MooncakeStoreWorker: dcp_rank=dcp_rank, pp_rank=pp_rank, ) - for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces + for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces( + self._group_tp_replication_factors[g_idx] + ) ) - for db in self.token_dbs + for g_idx, db in enumerate(self.token_dbs) ) - self._lookup_expected_per_key = len(rank_namespaces) def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None: """Register a cross-layers KV cache tensor. @@ -1297,7 +1316,7 @@ class MooncakeStoreWorker: self.token_dbs, self.block_size, self.tp_rank, - self.put_step, + self._group_tp_replication_factors, self.kv_role, ready_event_sending, self.enable_kv_events, @@ -1527,16 +1546,16 @@ class MooncakeStoreWorker: logger.error("Remote connection failed in lookup: %s", e) return 0 - # A (group, hash) is "present" only when every TP*PP rank has it. - ranks_per_candidate = self._lookup_expected_per_key - exists_set = { - (g_idx, hash_bytes) - for i, (g_idx, hash_bytes) in enumerate(candidate_meta) - if all( - res[i * ranks_per_candidate + j] == 1 - for j in range(ranks_per_candidate) - ) - } + # A (group, hash) is "present" only when every namespace that will be + # loaded has it (per-group count: sharded groups need every rank's + # shard, replicated groups one namespace per unique KV head). + exists_set = set() + pos = 0 + for g_idx, hash_bytes in candidate_meta: + count = len(self._lookup_key_prefixes[g_idx]) + if all(res[pos + j] == 1 for j in range(count)): + exists_set.add((g_idx, hash_bytes)) + pos += count cached_block_pool = ExternalCachedBlockPool( self.hash_block_size,