[Bugfix][KV Connector][Mooncake] Keep TP-sharded Mamba state out of the KV-head dedup (#49499)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -173,7 +173,6 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
|
||||
worker.tp_size = 1
|
||||
worker.pp_size = 1
|
||||
worker.put_step = 1
|
||||
worker.num_kv_head = 8
|
||||
|
||||
# Register kv_caches using mocked thread classes so register_kv_caches
|
||||
@@ -215,7 +214,7 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
block_size=worker.block_size,
|
||||
coord=worker.coord,
|
||||
tp_rank=worker.tp_rank,
|
||||
put_step=worker.put_step,
|
||||
group_put_steps=worker._group_tp_replication_factors,
|
||||
kv_role=worker.kv_role,
|
||||
ready_event=ready,
|
||||
enable_kv_event=False,
|
||||
|
||||
@@ -90,7 +90,7 @@ def _make_store_sending_thread(
|
||||
block_size=block_size,
|
||||
coord=coord,
|
||||
tp_rank=tp_rank,
|
||||
put_step=put_step,
|
||||
group_put_steps=[put_step] * len(token_databases),
|
||||
kv_role="kv_producer",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=replicate_config,
|
||||
@@ -527,6 +527,29 @@ def test_store_sending_thread_delta_strides_with_local_phase():
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def test_tp_sharded_group_saves_every_block_on_every_rank():
|
||||
"""Sharded ranks must write every block because peers hold different bytes."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
|
||||
thread.group_put_steps = [1]
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
assert len(keys) == 4
|
||||
|
||||
|
||||
def test_store_sending_thread_retries_skipped_range_after_pressure():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
@@ -1274,7 +1297,8 @@ def test_worker_put_striding_covers_every_rank_get_namespace(
|
||||
]
|
||||
assert len(keys) == len(block_hashes)
|
||||
# PUT side: mirrors KVCacheStoreSendingThread's striding slice.
|
||||
put_keys.update(keys[w.tp_rank % w.put_step :: w.put_step])
|
||||
put_step = w._group_tp_replication_factors[0]
|
||||
put_keys.update(keys[w.tp_rank % put_step :: put_step])
|
||||
# GET side: KVCacheStoreRecvingThread fetches every key.
|
||||
get_keys_per_rank[tp_rank] = set(keys)
|
||||
|
||||
@@ -1613,6 +1637,15 @@ def _register_with_mocked_threads(
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
|
||||
def _refresh_group_tp_replication_factors(
|
||||
worker: mooncake_store_worker.MooncakeStoreWorker,
|
||||
) -> None:
|
||||
worker._group_tp_replication_factors = (
|
||||
worker._compute_group_tp_replication_factors()
|
||||
)
|
||||
worker._init_lookup_key_prefixes()
|
||||
|
||||
|
||||
def _make_bare_worker(
|
||||
*,
|
||||
num_gpu_blocks: int = 10,
|
||||
@@ -1630,11 +1663,9 @@ def _make_bare_worker(
|
||||
worker.cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
worker.store = MagicMock()
|
||||
worker.store.register_buffer.return_value = 0
|
||||
worker.use_mla = False
|
||||
worker.kv_role = kv_role
|
||||
worker.block_size = block_size
|
||||
worker.tp_rank = 0
|
||||
worker.put_step = 1
|
||||
worker.enable_kv_events = False
|
||||
worker.kv_send_thread = None
|
||||
worker.kv_recv_threads = []
|
||||
@@ -1664,7 +1695,6 @@ def _make_bare_worker(
|
||||
worker.pcp_size = 1
|
||||
worker.dcp_size = 1
|
||||
worker.hash_block_size = block_size
|
||||
worker.metadata = KeyMetadata("test-model", 0, 0, 0, 0)
|
||||
# Pre-build a single-group token_dbs so lookup-only tests don't have to
|
||||
# call register_kv_caches.
|
||||
worker.token_dbs = [
|
||||
@@ -1679,7 +1709,7 @@ def _make_bare_worker(
|
||||
scheduler_block_size=block_size,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
worker._init_lookup_key_prefixes()
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
return worker
|
||||
|
||||
|
||||
@@ -1688,9 +1718,8 @@ def test_lookup_key_prefixes_cover_dcp_rank_namespaces():
|
||||
worker.tp_size = 4
|
||||
worker.num_kv_head = 1
|
||||
worker.dcp_size = 4
|
||||
worker._init_lookup_key_prefixes()
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
|
||||
assert worker._lookup_expected_per_key == 4
|
||||
assert worker._lookup_key_prefixes[0] == (
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0",
|
||||
"test-model@tp_rank:1@pcp0@dcp1@pp_rank:0@group:0",
|
||||
@@ -1705,21 +1734,192 @@ def test_lookup_key_prefixes_cover_pcp_rank_namespaces():
|
||||
worker.num_kv_head = 1
|
||||
worker.pcp_size = 2
|
||||
worker.dcp_size = 1
|
||||
worker._init_lookup_key_prefixes()
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
|
||||
assert worker._lookup_expected_per_key == 2
|
||||
assert worker._lookup_key_prefixes[0] == (
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0",
|
||||
"test-model@tp_rank:0@pcp1@dcp0@pp_rank:0@group:0",
|
||||
)
|
||||
|
||||
|
||||
def test_lookup_key_prefixes_expand_tp_sharded_groups_per_rank():
|
||||
"""Replicated attention needs one namespace; sharded Mamba needs every rank."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.tp_size = 2
|
||||
worker.num_kv_head = 1
|
||||
fa = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
worker._kv_cache_groups = [
|
||||
KVCacheGroupSpec(["l0"], fa),
|
||||
KVCacheGroupSpec(["l1"], mamba),
|
||||
]
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), block_size=16
|
||||
),
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 1, 0, 0, 0, group_id=1), block_size=16
|
||||
),
|
||||
]
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
|
||||
assert worker._lookup_key_prefixes[0] == (
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0",
|
||||
)
|
||||
assert worker._lookup_key_prefixes[1] == (
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1",
|
||||
"test-model@tp_rank:1@pcp0@dcp0@pp_rank:0@group:1",
|
||||
)
|
||||
|
||||
|
||||
def test_group_tp_replication_factors_mixed_mla_gqa_mamba():
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.tp_size = 4
|
||||
worker.num_kv_head = 2
|
||||
mla = MLAAttentionSpec(block_size=16, num_kv_heads=1, head_size=64, dtype=None)
|
||||
gqa = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
worker._kv_cache_groups = [
|
||||
KVCacheGroupSpec(["l0"], mla),
|
||||
KVCacheGroupSpec(["l1"], gqa),
|
||||
KVCacheGroupSpec(["l2"], mamba),
|
||||
]
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=g_idx), block_size=16
|
||||
)
|
||||
for g_idx in range(3)
|
||||
]
|
||||
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
assert worker._group_tp_replication_factors == (4, 2, 1)
|
||||
assert worker._lookup_key_prefixes[0] == (
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0",
|
||||
)
|
||||
assert worker._lookup_key_prefixes[1] == (
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1",
|
||||
"test-model@tp_rank:1@pcp0@dcp0@pp_rank:0@group:1",
|
||||
)
|
||||
assert worker._lookup_key_prefixes[2] == (
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:2",
|
||||
"test-model@tp_rank:1@pcp0@dcp0@pp_rank:0@group:2",
|
||||
"test-model@tp_rank:2@pcp0@dcp0@pp_rank:0@group:2",
|
||||
"test-model@tp_rank:3@pcp0@dcp0@pp_rank:0@group:2",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spec_order", [("mla", "gqa"), ("gqa", "mla")])
|
||||
def test_uniform_group_uses_common_inner_replication_factor(spec_order):
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.tp_size = 4
|
||||
worker.num_kv_head = 2
|
||||
specs_by_name = {
|
||||
"mla": MLAAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=64, dtype=None
|
||||
),
|
||||
"gqa": FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=64, dtype=None
|
||||
),
|
||||
}
|
||||
inner_specs = {name: specs_by_name[name] for name in spec_order}
|
||||
uniform_spec = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs=inner_specs,
|
||||
)
|
||||
worker._kv_cache_groups = [
|
||||
KVCacheGroupSpec(list(inner_specs), uniform_spec),
|
||||
]
|
||||
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
|
||||
assert worker._group_tp_replication_factors == (2,)
|
||||
assert worker._lookup_key_prefixes[0] == (
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0",
|
||||
"test-model@tp_rank:1@pcp0@dcp0@pp_rank:0@group:0",
|
||||
)
|
||||
|
||||
|
||||
def test_lookup_rejects_boundary_missing_one_mamba_shard():
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.tp_size = 2
|
||||
worker.num_kv_head = 1
|
||||
fa = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
worker._kv_cache_groups = [
|
||||
KVCacheGroupSpec(["l0"], fa),
|
||||
KVCacheGroupSpec(["l1"], mamba),
|
||||
]
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), block_size=16
|
||||
),
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 1, 0, 0, 0, group_id=1), block_size=16
|
||||
),
|
||||
]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
)
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
|
||||
worker.store.batch_is_exist.side_effect = lambda keys: [1] * len(keys)
|
||||
assert worker.lookup(32, [b"h0", b"h1"]) == 32
|
||||
|
||||
worker.store.batch_is_exist.side_effect = lambda keys: [
|
||||
0 if "tp_rank:1" in k and "group:1" in k else 1 for k in keys
|
||||
]
|
||||
assert worker.lookup(32, [b"h0", b"h1"]) == 0
|
||||
|
||||
|
||||
def test_lookup_requires_all_dcp_rank_namespaces():
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.tp_size = 4
|
||||
worker.num_kv_head = 1
|
||||
worker.dcp_size = 4
|
||||
worker._init_lookup_key_prefixes()
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
worker.store.batch_is_exist.return_value = [1, 1, 0, 1]
|
||||
|
||||
assert worker.lookup(16, [b"a0"]) == 0
|
||||
@@ -1850,7 +2050,7 @@ def test_lookup_checks_all_potential_swa_hit_boundaries():
|
||||
hash_block_size=8,
|
||||
retention_interval=0,
|
||||
)
|
||||
worker._init_lookup_key_prefixes()
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
# Candidate order: 3 full-attention chunks, then SWA chunks 3, 7, 11.
|
||||
# Only the first full chunk and the SWA chunk ending at token 32 exist, so
|
||||
# lookup should recover a 32-token external prefix hit. A sparse
|
||||
@@ -1911,7 +2111,7 @@ def test_lookup_applies_swa_mask_before_accessing_hashes():
|
||||
hash_block_size=8,
|
||||
retention_interval=0,
|
||||
)
|
||||
worker._init_lookup_key_prefixes()
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
|
||||
block_hashes = _RecordingBlockHashes([f"h{i}".encode() for i in range(12)])
|
||||
accessed_before_rpc: list[int] = []
|
||||
|
||||
@@ -66,7 +66,15 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
maybe_convert_block_hash,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
from .metrics import MooncakeStoreConnectorStats
|
||||
|
||||
@@ -448,7 +456,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
token_databases: list[ChunkedTokenDatabase],
|
||||
block_size: int,
|
||||
tp_rank: int,
|
||||
put_step: int,
|
||||
group_put_steps: Sequence[int],
|
||||
kv_role: str,
|
||||
ready_event: threading.Event,
|
||||
enable_kv_event: bool = False,
|
||||
@@ -464,7 +472,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
name="KVCacheStoreSendingThread",
|
||||
record_operation=record_operation,
|
||||
)
|
||||
self.put_step = put_step
|
||||
# Only ranks with identical group bytes may stripe PUTs (e.g., MLA).
|
||||
self.group_put_steps = group_put_steps
|
||||
self.coord = coord
|
||||
self.kv_role = kv_role
|
||||
self.stored_requests: defaultdict[str, int] = defaultdict(int)
|
||||
@@ -568,13 +577,14 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
group_indices: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
# Rotate the stride phase per group to balance load across ranks.
|
||||
put_step_rank = (self.tp_rank + g_idx) % self.put_step
|
||||
put_step = self.group_put_steps[g_idx]
|
||||
put_step_rank = (self.tp_rank + g_idx) % put_step
|
||||
for start, end, block_hash in db.process_tokens(
|
||||
token_len,
|
||||
req_meta.block_hashes,
|
||||
mask_num=save_start,
|
||||
chunk_mask=store_masks[g_idx],
|
||||
put_step=self.put_step,
|
||||
put_step=put_step,
|
||||
put_step_rank=put_step_rank,
|
||||
):
|
||||
starts.append(start)
|
||||
@@ -1002,44 +1012,7 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
self.num_layers = model_config.get_num_layers(parallel_config)
|
||||
|
||||
self.use_mla = False
|
||||
if (
|
||||
hasattr(model_config, "use_mla")
|
||||
and isinstance(model_config.use_mla, bool)
|
||||
and model_config.use_mla
|
||||
):
|
||||
self.use_mla = True
|
||||
|
||||
if self.use_mla:
|
||||
self.num_kv_head = 1
|
||||
else:
|
||||
self.num_kv_head = model_config.get_total_num_kv_heads()
|
||||
|
||||
if self.num_kv_head < self.tp_size and self.dcp_size <= 1:
|
||||
# Dedup: TP ranks holding the same KV heads stripe PUTs across
|
||||
# one shared key namespace. DCP splits the TP group, so with
|
||||
# DCP>1 those ranks have different `@dcpN` namespaces and
|
||||
# striping would leave keys unwritten (OBJECT_NOT_FOUND on
|
||||
# GET). PCP is outer to TP (pcp_rank is constant within a TP
|
||||
# group), so it needs no guard.
|
||||
self.put_step = self.tp_size // self.num_kv_head
|
||||
self.head_or_tp_rank = self.tp_rank // self.put_step
|
||||
else:
|
||||
self.head_or_tp_rank = self.tp_rank
|
||||
self.put_step = 1
|
||||
|
||||
self.metadata = KeyMetadata(
|
||||
model_name=model_config.model.rstrip("/").split("/")[-1],
|
||||
tp_rank=self.head_or_tp_rank,
|
||||
pcp_rank=self.pcp_rank,
|
||||
dcp_rank=self.dcp_rank,
|
||||
pp_rank=self.pp_rank,
|
||||
cache_prefix=str(
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"cache_prefix", ""
|
||||
)
|
||||
),
|
||||
)
|
||||
self.num_kv_head = model_config.get_total_num_kv_heads()
|
||||
|
||||
# Initialize MooncakeDistributedStore with its own TransferEngine
|
||||
store_config = MooncakeStoreConfig.load_from_config()
|
||||
@@ -1156,10 +1129,32 @@ class MooncakeStoreWorker:
|
||||
retention_interval=envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL,
|
||||
)
|
||||
# One ChunkedTokenDatabase per group; addresses populated in
|
||||
# register_kv_caches once the kv-cache layout is known.
|
||||
# register_kv_caches once the kv-cache layout is known. Each group's
|
||||
# key namespace is its TP shard id: ranks holding identical bytes
|
||||
# (MLA / shared GQA KV heads) share a namespace, TP-sharded Mamba
|
||||
# state gets one namespace per rank.
|
||||
metadata = KeyMetadata(
|
||||
model_name=model_config.model.rstrip("/").split("/")[-1],
|
||||
tp_rank=self.tp_rank,
|
||||
pcp_rank=self.pcp_rank,
|
||||
dcp_rank=self.dcp_rank,
|
||||
pp_rank=self.pp_rank,
|
||||
cache_prefix=str(
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"cache_prefix", ""
|
||||
)
|
||||
),
|
||||
)
|
||||
self._group_tp_replication_factors: tuple[int, ...] = (
|
||||
self._compute_group_tp_replication_factors()
|
||||
)
|
||||
self.token_dbs: list[ChunkedTokenDatabase] = [
|
||||
ChunkedTokenDatabase(
|
||||
dataclasses.replace(self.metadata, group_id=g_idx),
|
||||
dataclasses.replace(
|
||||
metadata,
|
||||
group_id=g_idx,
|
||||
tp_rank=self.tp_rank // self._group_tp_replication_factors[g_idx],
|
||||
),
|
||||
g.kv_cache_spec.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
)
|
||||
@@ -1167,27 +1162,50 @@ class MooncakeStoreWorker:
|
||||
]
|
||||
self._init_lookup_key_prefixes()
|
||||
|
||||
def _init_lookup_key_prefixes(self) -> None:
|
||||
"""Prepare per-group key prefixes across parallel rank namespaces."""
|
||||
# (tp_rank, pcp_rank, dcp_rank, pp_rank) namespaces
|
||||
def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int:
|
||||
if self.dcp_size > 1:
|
||||
# DCP reuses the TP workers and splits each TP group into
|
||||
# contiguous DCP groups, so dcp_rank == tp_rank % dcp_size.
|
||||
# Store/load paths do not apply KV-head dedup under DCP
|
||||
rank_namespaces = tuple(
|
||||
(tp_rank, pcp_rank, tp_rank % self.dcp_size, pp_rank)
|
||||
return 1
|
||||
inner_specs = (
|
||||
tuple(spec.kv_cache_specs.values())
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs)
|
||||
else (spec,)
|
||||
)
|
||||
# Any rank-specific state makes the whole packed value rank-specific.
|
||||
if any(isinstance(inner, MambaSpec) for inner in inner_specs):
|
||||
return 1
|
||||
# A pure MLA packed value is replicated on every TP rank.
|
||||
if all(
|
||||
isinstance(inner, (MLAAttentionSpec, SlidingWindowMLASpec))
|
||||
for inner in inner_specs
|
||||
):
|
||||
return self.tp_size
|
||||
return max(1, self.tp_size // self.num_kv_head)
|
||||
|
||||
def _compute_group_tp_replication_factors(self) -> tuple[int, ...]:
|
||||
"""Return the number of byte-identical TP replicas per cache group.
|
||||
|
||||
DCP and Mamba use 1; MLA uses ``tp_size``; GQA uses
|
||||
``tp_size // num_kv_head``.
|
||||
"""
|
||||
return tuple(
|
||||
self._spec_tp_replication_factor(group.kv_cache_spec)
|
||||
for group in self._kv_cache_groups
|
||||
)
|
||||
|
||||
def _init_lookup_key_prefixes(self) -> None:
|
||||
def rank_namespaces(factor: int) -> tuple[tuple[int, int, int, int], ...]:
|
||||
if self.dcp_size > 1:
|
||||
# DCP is a TP subdivision: dcp_rank == tp_rank % dcp_size.
|
||||
return tuple(
|
||||
(tp_rank, pcp_rank, tp_rank % self.dcp_size, pp_rank)
|
||||
for pcp_rank in range(self.pcp_size)
|
||||
for tp_rank in range(self.tp_size)
|
||||
for pp_rank in range(self.pp_size)
|
||||
)
|
||||
return tuple(
|
||||
(shard_rank, pcp_rank, 0, pp_rank)
|
||||
for pcp_rank in range(self.pcp_size)
|
||||
for tp_rank in range(self.tp_size)
|
||||
for pp_rank in range(self.pp_size)
|
||||
)
|
||||
else:
|
||||
# Without DCP, TP ranks that share a KV head write identical KV, so
|
||||
# lookup only needs one TP namespace per unique KV head.
|
||||
tp_count = min(self.tp_size, self.num_kv_head)
|
||||
rank_namespaces = tuple(
|
||||
(tp_rank, pcp_rank, 0, pp_rank)
|
||||
for pcp_rank in range(self.pcp_size)
|
||||
for tp_rank in range(tp_count)
|
||||
for shard_rank in range(self.tp_size // factor)
|
||||
for pp_rank in range(self.pp_size)
|
||||
)
|
||||
|
||||
@@ -1200,11 +1218,12 @@ class MooncakeStoreWorker:
|
||||
dcp_rank=dcp_rank,
|
||||
pp_rank=pp_rank,
|
||||
)
|
||||
for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces
|
||||
for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces(
|
||||
self._group_tp_replication_factors[g_idx]
|
||||
)
|
||||
)
|
||||
for db in self.token_dbs
|
||||
for g_idx, db in enumerate(self.token_dbs)
|
||||
)
|
||||
self._lookup_expected_per_key = len(rank_namespaces)
|
||||
|
||||
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
|
||||
"""Register a cross-layers KV cache tensor.
|
||||
@@ -1297,7 +1316,7 @@ class MooncakeStoreWorker:
|
||||
self.token_dbs,
|
||||
self.block_size,
|
||||
self.tp_rank,
|
||||
self.put_step,
|
||||
self._group_tp_replication_factors,
|
||||
self.kv_role,
|
||||
ready_event_sending,
|
||||
self.enable_kv_events,
|
||||
@@ -1527,16 +1546,16 @@ class MooncakeStoreWorker:
|
||||
logger.error("Remote connection failed in lookup: %s", e)
|
||||
return 0
|
||||
|
||||
# A (group, hash) is "present" only when every TP*PP rank has it.
|
||||
ranks_per_candidate = self._lookup_expected_per_key
|
||||
exists_set = {
|
||||
(g_idx, hash_bytes)
|
||||
for i, (g_idx, hash_bytes) in enumerate(candidate_meta)
|
||||
if all(
|
||||
res[i * ranks_per_candidate + j] == 1
|
||||
for j in range(ranks_per_candidate)
|
||||
)
|
||||
}
|
||||
# A (group, hash) is "present" only when every namespace that will be
|
||||
# loaded has it (per-group count: sharded groups need every rank's
|
||||
# shard, replicated groups one namespace per unique KV head).
|
||||
exists_set = set()
|
||||
pos = 0
|
||||
for g_idx, hash_bytes in candidate_meta:
|
||||
count = len(self._lookup_key_prefixes[g_idx])
|
||||
if all(res[pos + j] == 1 for j in range(count)):
|
||||
exists_set.add((g_idx, hash_bytes))
|
||||
pos += count
|
||||
|
||||
cached_block_pool = ExternalCachedBlockPool(
|
||||
self.hash_block_size,
|
||||
|
||||
Reference in New Issue
Block a user