From 15be78732bac03af62dc05d588e245271b72f10d Mon Sep 17 00:00:00 2001 From: Asaf Gardin <39553475+Josephasafg@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:50:41 +0300 Subject: [PATCH] [NIXL][Mamba] Add Mamba1 support to NIXL P/D disaggregation (#45019) Signed-off-by: Josephasafg Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../config_sweep_accuracy_test.sh | 2 + .../nixl_integration/test_accuracy.py | 1 + .../unit/test_nixl_connector_hma.py | 45 ++++++++++++ .../kv_connector/v1/nixl/base_worker.py | 22 +++--- .../v1/ssm_conv_transfer_utils.py | 69 +++++++++++-------- 5 files changed, 104 insertions(+), 35 deletions(-) diff --git a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh index 57602289ce..9ce225c3a4 100755 --- a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +++ b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh @@ -28,6 +28,8 @@ hybrid_ssm_configs=( # GDN (Qwen3.5) "VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B" "VLLM_SSM_CONV_STATE_LAYOUT=DS PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B" + # Mamba1 (Jamba) + "VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ai21labs/AI21-Jamba2-3B VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" ) sw_attn_configs=( # NOTE: gemma3 does not work with FlashInfer diff --git a/tests/v1/kv_connector/nixl_integration/test_accuracy.py b/tests/v1/kv_connector/nixl_integration/test_accuracy.py index eead3de153..bb68b7a572 100644 --- a/tests/v1/kv_connector/nixl_integration/test_accuracy.py +++ b/tests/v1/kv_connector/nixl_integration/test_accuracy.py @@ -25,6 +25,7 @@ EXPECTED_VALUES = { "ibm-granite/granite-4.0-h-tiny": 0.77, "Qwen/Qwen3.5-0.8B": 0.33, "google/gemma-4-E2B-it": 0.485, + "ai21labs/AI21-Jamba2-3B": 0.74, } SIMPLE_PROMPT = ( diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index d508f3cae2..c6bcdc2896 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -630,6 +630,19 @@ def _make_mock_worker_for_desc_ids( worker._has_mamba = has_mamba worker._group_spec_types = group_spec_types worker.block_len_per_layer = block_len_per_layer or [100] + worker._conv_decomp = None + if has_mamba: + from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501 + MambaConvSplitInfo, + ) + + # Mamba2/GDN layout: 3 conv sub-projections -> 4 NIXL regions per layer. + worker._conv_decomp = MambaConvSplitInfo( + conv_rows=3, + local_proj_dims=(1, 1, 1), + conv_dtype_size=2, + ssm_sizes=(0, 0), + ) worker._compute_desc_ids = NixlConnectorWorker._compute_desc_ids.__get__( worker, NixlConnectorWorker ) @@ -976,6 +989,37 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati (256, 256, 768), id="qwen35_27b_tp8", ), + # ai21labs/AI21-Jamba2-Mini (Mamba1) + # mamba d_inner = mamba_expand(2) * hidden_size(4096) = 8192 + # mamba_d_state=16, mamba_d_conv=4 → conv_rows=3. + # Conv state holds only x: a single contiguous sub-projection. + pytest.param( + "mamba1", + 1, + 8192, + 3, + (8192, 16), + (8192,), + id="jamba_mini_tp1", + ), + pytest.param( + "mamba1", + 4, + 2048, + 3, + (2048, 16), + (2048,), + id="jamba_mini_tp4", + ), + pytest.param( + "mamba1", + 8, + 1024, + 3, + (1024, 16), + (1024,), + id="jamba_mini_tp8", + ), ], ) def test_derive_mamba_conv_split( @@ -999,6 +1043,7 @@ def test_derive_mamba_conv_split( from vllm.v1.kv_cache_interface import MambaSpec _TYPE_MAP = { + "mamba1": MambaAttentionBackendEnum.MAMBA1, "mamba2": MambaAttentionBackendEnum.MAMBA2, "gdn_attention": MambaAttentionBackendEnum.GDN_ATTN, } diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 060fa5e322..66bee55f28 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -96,7 +96,13 @@ class NixlBaseConnectorWorker: ) -> np.ndarray: """Compute NIXL descriptor IDs for given block IDs.""" num_fa_regions = self.num_regions - num_ssm_regions = len(self.block_len_per_layer) * 4 if self._has_mamba else 0 + num_ssm_regions = 0 + if self._has_mamba: + assert self._conv_decomp is not None + # NIXL regions per SSM layer = conv sub-projections + 1 SSM temporal + # (Mamba2/GDN: 3+1=4; Mamba1: 1+1=2). + ssm_regions_per_layer = len(self._conv_decomp.local_conv_offsets) + 1 + num_ssm_regions = len(self.block_len_per_layer) * ssm_regions_per_layer num_blocks = dst_num_blocks if block_size_ratio is not None: @@ -279,8 +285,8 @@ class NixlBaseConnectorWorker: # ---- Model state (derived from model config) ---- mamba_ssm_size = (0, 0) # Conv state sub-projection decomposition (None when no Mamba). - # The 3-read transfer requires DS (dim, state_len) conv layout so - # that x/B/C sub-projections are contiguous in memory. + # The transfer requires DS (dim, state_len) conv layout so that + # conv sub-projections are contiguous in memory. self._conv_decomp: MambaConvSplitInfo | None = None self._has_mamba = any( isinstance(g.kv_cache_spec, MambaSpec) @@ -1186,8 +1192,8 @@ class NixlBaseConnectorWorker: base_addresses: list[int], block_size_ratio: int, ) -> list[tuple[int, int, int]]: - """Build 4 desc regions (x, B, C, ssm) per layer for local mamba - blocks, enabling the 3-read transfer with DS conv layout.""" + """Build desc regions (conv sub-projections + ssm) per layer for + local mamba blocks with DS conv layout.""" assert block_size_ratio == 1, ( "Mamba 3-read transfer with block_size_ratio != 1 is not tested. " f"Got block_size_ratio={block_size_ratio}." @@ -1227,9 +1233,9 @@ class NixlBaseConnectorWorker: tp_ratio: int, transfer_info: EngineTransferInfo, ) -> list[tuple[int, int, int]]: - """Build 4 remote desc regions (proj0, proj1, proj2, ssm) per layer - for the 3-read transfer. For hetero-TP, each D rank reads only its - sub-projection slice from the P rank.""" + """Build remote desc regions (conv sub-projections + ssm) per layer. + For hetero-TP, each D rank reads only its sub-projection slice from + the P rank.""" assert self._conv_decomp is not None effective_ratio = max(tp_ratio, 1) # Mamba conv state is always TP-sharded, even when attention KV diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py index 2a5510656b..00dc05bfc4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py @@ -1,12 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Mamba conv-state sub-projection decomposition for the 3-read transfer. +"""Mamba conv-state sub-projection decomposition for NIXL transfer. With DS conv state layout (dim, state_len), sub-projections are -contiguous in memory. Each D rank reads its slices via 3 separate +contiguous in memory. Each D rank reads its slices via separate RDMA transfers — no P-side permutation needed. Supported model types: + - Mamba1: conv = [x], temporal = (intermediate_size, state_size) - Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim) - GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)), temporal = (num_v_heads, v_dim, k_dim) @@ -24,18 +25,21 @@ from vllm.v1.kv_cache_interface import MambaSpec @dataclass(frozen=True) class MambaConvSplitInfo: - """Per-rank byte sizes of the 3 conv sub-projections. + """Per-rank byte sizes of the conv sub-projections. Used by both P and D sides for NIXL descriptor registration. All fields are LOCAL to this engine's TP (already divided by TP size). DS memory layout within one page (contiguous): + Mamba1: |---- x ----| (single sub-projection, no decomposition) Mamba2: |-- x --|- B -|- C -| (B == C) GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ) """ conv_rows: int # conv_kernel - 1 (typically 3) - local_proj_dims: tuple[int, int, int] # per-rank column counts per sub-proj + # Per-rank column counts per sub-projection: + # 1 entry for Mamba1, 3 for Mamba2/GDN. + local_proj_dims: tuple[int, ...] conv_dtype_size: int # bytes per element (e.g. 2 for float16) ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes) @@ -45,10 +49,10 @@ class MambaConvSplitInfo: return sum(self.local_proj_dims) @property - def proj_bytes(self) -> tuple[int, int, int]: - """Byte sizes of the 3 sub-projections for one rank.""" + def proj_bytes(self) -> tuple[int, ...]: + """Byte sizes of the sub-projections for one rank.""" row_bytes = self.conv_rows * self.conv_dtype_size - return tuple(d * row_bytes for d in self.local_proj_dims) # type: ignore[return-value] + return tuple(d * row_bytes for d in self.local_proj_dims) @property def local_conv_offsets(self) -> list[tuple[int, int]]: @@ -57,8 +61,12 @@ class MambaConvSplitInfo: Used by both P and D for local descriptor registration. """ - conv0, conv1, conv2 = self.proj_bytes - return [(0, conv0), (conv0, conv1), (conv0 + conv1, conv2)] + offsets: list[tuple[int, int]] = [] + offset = 0 + for size in self.proj_bytes: + offsets.append((offset, size)) + offset += size + return offsets def remote_conv_offsets( self, local_rank_offset: int, tp_ratio: int @@ -76,28 +84,23 @@ class MambaConvSplitInfo: P page. Local dims are scaled down by |tp_ratio| to get P-sized offsets. """ - conv0, conv1, conv2 = self.proj_bytes + offsets: list[tuple[int, int]] = [] if tp_ratio >= 1: - remote_conv0 = conv0 * tp_ratio - remote_conv1 = conv1 * tp_ratio - return [ - (local_rank_offset * conv0, conv0), - (remote_conv0 + local_rank_offset * conv1, conv1), - (remote_conv0 + remote_conv1 + local_rank_offset * conv2, conv2), - ] + remote_base = 0 + for size in self.proj_bytes: + offsets.append((remote_base + local_rank_offset * size, size)) + remote_base += size * tp_ratio else: # NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages # are smaller than D's. Local dims are D-sized, but we need # P-sized offsets. Scale down by |tp_ratio|. abs_ratio = -tp_ratio - remote_conv0 = conv0 // abs_ratio - remote_conv1 = conv1 // abs_ratio - remote_conv2 = conv2 // abs_ratio - return [ - (0, remote_conv0), - (remote_conv0, remote_conv1), - (remote_conv0 + remote_conv1, remote_conv2), - ] + remote_base = 0 + for size in self.proj_bytes: + remote_size = size // abs_ratio + offsets.append((remote_base, remote_size)) + remote_base += remote_size + return offsets def derive_mamba_conv_split( @@ -120,12 +123,13 @@ def derive_mamba_conv_split( conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes). """ _supported = ( + MambaAttentionBackendEnum.MAMBA1, MambaAttentionBackendEnum.MAMBA2, MambaAttentionBackendEnum.GDN_ATTN, ) if mamba_spec.mamba_type not in _supported: raise NotImplementedError( - f"3-read conv transfer only supports Mamba2 and GDN models, " + f"Conv transfer only supports Mamba1, Mamba2 and GDN models, " f"got mamba_type={mamba_spec.mamba_type!r}." ) @@ -149,7 +153,18 @@ def derive_mamba_conv_split( conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size - if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2: + local_proj_dims: tuple[int, ...] + if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA1: + # Mamba1 conv state holds only x (no B/C), so it's a single + # contiguous TP shard with no sub-projection decomposition. + temporal_shape = mamba_spec.shapes[1] + assert temporal_shape[0] == local_conv_dim, ( + f"Mamba1 temporal state dim ({temporal_shape[0]}) doesn't match " + f"conv dim ({local_conv_dim}); both should be " + f"intermediate_size/TP." + ) + local_proj_dims = (local_conv_dim,) + elif mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2: # NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored # in MambaSpec, so we reconstruct it from the SSM temporal state shape: # shapes[1] = (local_num_heads, head_dim), already divided by TP.