[KV Connector] Support symmetric DCP disagg for hybrid mamba models (#55531)
Signed-off-by: Wei Zhao <weizha@oci-aga-slurm-1-vscode-02.cm.cluster> Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Wei Zhao <weizha@oci-aga-slurm-1-vscode-02.cm.cluster> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Wei Zhao
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
983b7e28c9
commit
7cdd9304ae
@@ -530,9 +530,6 @@ class NixlBaseConnectorWorker:
|
||||
# DCP support is scoped to MLA, with dcp_size in (1, tp_size): either fully
|
||||
# replicated or fully sharded. A DCP rank is always derivable this way.
|
||||
self.dcp_rank = self.tp_rank % self.dcp_size
|
||||
if self._has_mamba and self.dcp_size > 1:
|
||||
# Prefix-cache-aware DCP slicing isn't implemented for the Mamba group.
|
||||
raise ValueError("DCP is not supported for hybrid MLA+Mamba models.")
|
||||
|
||||
self.num_blocks = kv_cache_config.num_blocks
|
||||
self.enable_permute_local_kv = False
|
||||
@@ -2177,6 +2174,11 @@ class NixlBaseConnectorWorker:
|
||||
f"DCP sizes must divide one another: local={self.dcp_size}, "
|
||||
f"remote={remote_dcp_size} (engine {remote_engine_id})."
|
||||
)
|
||||
if self._has_mamba and self.dcp_size != remote_dcp_size:
|
||||
raise RuntimeError(
|
||||
"Hybrid MLA+Mamba NIXL transfers require matching DCP sizes, "
|
||||
f"got local={self.dcp_size}, remote={remote_dcp_size}."
|
||||
)
|
||||
|
||||
tp_ratio = self.transfer_topo.tp_ratio(remote_tp_size)
|
||||
block_size_ratio = self.transfer_topo.block_size_ratio(
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
ReadSpec,
|
||||
_is_attention_spec,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -213,7 +214,9 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
local_ids = group_ids(meta.local_block_ids, rank)
|
||||
remote_ids = group_ids(remote_logical_block_ids, rank)
|
||||
for g in range(num_groups):
|
||||
if not local_ids[g]:
|
||||
if not local_ids[g] or not _is_attention_spec(
|
||||
self._group_spec_types[g]
|
||||
):
|
||||
continue
|
||||
local_ids[g], remote_ids[g] = self._apply_dcp_prefix_caching(
|
||||
local_ids[g],
|
||||
|
||||
Reference in New Issue
Block a user