[NIXL][Mamba] Add Mamba1 support to NIXL P/D disaggregation (#45019)

Signed-off-by: Josephasafg <ajgard7@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Asaf Gardin
2026-06-25 05:50:41 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 92221485aa
commit 15be78732b
5 changed files with 104 additions and 35 deletions
@@ -28,6 +28,8 @@ hybrid_ssm_configs=(
# GDN (Qwen3.5)
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
"VLLM_SSM_CONV_STATE_LAYOUT=DS PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
# Mamba1 (Jamba)
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ai21labs/AI21-Jamba2-3B VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
)
sw_attn_configs=(
# NOTE: gemma3 does not work with FlashInfer
@@ -25,6 +25,7 @@ EXPECTED_VALUES = {
"ibm-granite/granite-4.0-h-tiny": 0.77,
"Qwen/Qwen3.5-0.8B": 0.33,
"google/gemma-4-E2B-it": 0.485,
"ai21labs/AI21-Jamba2-3B": 0.74,
}
SIMPLE_PROMPT = (
@@ -630,6 +630,19 @@ def _make_mock_worker_for_desc_ids(
worker._has_mamba = has_mamba
worker._group_spec_types = group_spec_types
worker.block_len_per_layer = block_len_per_layer or [100]
worker._conv_decomp = None
if has_mamba:
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501
MambaConvSplitInfo,
)
# Mamba2/GDN layout: 3 conv sub-projections -> 4 NIXL regions per layer.
worker._conv_decomp = MambaConvSplitInfo(
conv_rows=3,
local_proj_dims=(1, 1, 1),
conv_dtype_size=2,
ssm_sizes=(0, 0),
)
worker._compute_desc_ids = NixlConnectorWorker._compute_desc_ids.__get__(
worker, NixlConnectorWorker
)
@@ -976,6 +989,37 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati
(256, 256, 768),
id="qwen35_27b_tp8",
),
# ai21labs/AI21-Jamba2-Mini (Mamba1)
# mamba d_inner = mamba_expand(2) * hidden_size(4096) = 8192
# mamba_d_state=16, mamba_d_conv=4 → conv_rows=3.
# Conv state holds only x: a single contiguous sub-projection.
pytest.param(
"mamba1",
1,
8192,
3,
(8192, 16),
(8192,),
id="jamba_mini_tp1",
),
pytest.param(
"mamba1",
4,
2048,
3,
(2048, 16),
(2048,),
id="jamba_mini_tp4",
),
pytest.param(
"mamba1",
8,
1024,
3,
(1024, 16),
(1024,),
id="jamba_mini_tp8",
),
],
)
def test_derive_mamba_conv_split(
@@ -999,6 +1043,7 @@ def test_derive_mamba_conv_split(
from vllm.v1.kv_cache_interface import MambaSpec
_TYPE_MAP = {
"mamba1": MambaAttentionBackendEnum.MAMBA1,
"mamba2": MambaAttentionBackendEnum.MAMBA2,
"gdn_attention": MambaAttentionBackendEnum.GDN_ATTN,
}
@@ -96,7 +96,13 @@ class NixlBaseConnectorWorker:
) -> np.ndarray:
"""Compute NIXL descriptor IDs for given block IDs."""
num_fa_regions = self.num_regions
num_ssm_regions = len(self.block_len_per_layer) * 4 if self._has_mamba else 0
num_ssm_regions = 0
if self._has_mamba:
assert self._conv_decomp is not None
# NIXL regions per SSM layer = conv sub-projections + 1 SSM temporal
# (Mamba2/GDN: 3+1=4; Mamba1: 1+1=2).
ssm_regions_per_layer = len(self._conv_decomp.local_conv_offsets) + 1
num_ssm_regions = len(self.block_len_per_layer) * ssm_regions_per_layer
num_blocks = dst_num_blocks
if block_size_ratio is not None:
@@ -279,8 +285,8 @@ class NixlBaseConnectorWorker:
# ---- Model state (derived from model config) ----
mamba_ssm_size = (0, 0)
# Conv state sub-projection decomposition (None when no Mamba).
# The 3-read transfer requires DS (dim, state_len) conv layout so
# that x/B/C sub-projections are contiguous in memory.
# The transfer requires DS (dim, state_len) conv layout so that
# conv sub-projections are contiguous in memory.
self._conv_decomp: MambaConvSplitInfo | None = None
self._has_mamba = any(
isinstance(g.kv_cache_spec, MambaSpec)
@@ -1186,8 +1192,8 @@ class NixlBaseConnectorWorker:
base_addresses: list[int],
block_size_ratio: int,
) -> list[tuple[int, int, int]]:
"""Build 4 desc regions (x, B, C, ssm) per layer for local mamba
blocks, enabling the 3-read transfer with DS conv layout."""
"""Build desc regions (conv sub-projections + ssm) per layer for
local mamba blocks with DS conv layout."""
assert block_size_ratio == 1, (
"Mamba 3-read transfer with block_size_ratio != 1 is not tested. "
f"Got block_size_ratio={block_size_ratio}."
@@ -1227,9 +1233,9 @@ class NixlBaseConnectorWorker:
tp_ratio: int,
transfer_info: EngineTransferInfo,
) -> list[tuple[int, int, int]]:
"""Build 4 remote desc regions (proj0, proj1, proj2, ssm) per layer
for the 3-read transfer. For hetero-TP, each D rank reads only its
sub-projection slice from the P rank."""
"""Build remote desc regions (conv sub-projections + ssm) per layer.
For hetero-TP, each D rank reads only its sub-projection slice from
the P rank."""
assert self._conv_decomp is not None
effective_ratio = max(tp_ratio, 1)
# Mamba conv state is always TP-sharded, even when attention KV
@@ -1,12 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mamba conv-state sub-projection decomposition for the 3-read transfer.
"""Mamba conv-state sub-projection decomposition for NIXL transfer.
With DS conv state layout (dim, state_len), sub-projections are
contiguous in memory. Each D rank reads its slices via 3 separate
contiguous in memory. Each D rank reads its slices via separate
RDMA transfers — no P-side permutation needed.
Supported model types:
- Mamba1: conv = [x], temporal = (intermediate_size, state_size)
- Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim)
- GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)),
temporal = (num_v_heads, v_dim, k_dim)
@@ -24,18 +25,21 @@ from vllm.v1.kv_cache_interface import MambaSpec
@dataclass(frozen=True)
class MambaConvSplitInfo:
"""Per-rank byte sizes of the 3 conv sub-projections.
"""Per-rank byte sizes of the conv sub-projections.
Used by both P and D sides for NIXL descriptor registration.
All fields are LOCAL to this engine's TP (already divided by TP size).
DS memory layout within one page (contiguous):
Mamba1: |---- x ----| (single sub-projection, no decomposition)
Mamba2: |-- x --|- B -|- C -| (B == C)
GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ)
"""
conv_rows: int # conv_kernel - 1 (typically 3)
local_proj_dims: tuple[int, int, int] # per-rank column counts per sub-proj
# Per-rank column counts per sub-projection:
# 1 entry for Mamba1, 3 for Mamba2/GDN.
local_proj_dims: tuple[int, ...]
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
@@ -45,10 +49,10 @@ class MambaConvSplitInfo:
return sum(self.local_proj_dims)
@property
def proj_bytes(self) -> tuple[int, int, int]:
"""Byte sizes of the 3 sub-projections for one rank."""
def proj_bytes(self) -> tuple[int, ...]:
"""Byte sizes of the sub-projections for one rank."""
row_bytes = self.conv_rows * self.conv_dtype_size
return tuple(d * row_bytes for d in self.local_proj_dims) # type: ignore[return-value]
return tuple(d * row_bytes for d in self.local_proj_dims)
@property
def local_conv_offsets(self) -> list[tuple[int, int]]:
@@ -57,8 +61,12 @@ class MambaConvSplitInfo:
Used by both P and D for local descriptor registration.
"""
conv0, conv1, conv2 = self.proj_bytes
return [(0, conv0), (conv0, conv1), (conv0 + conv1, conv2)]
offsets: list[tuple[int, int]] = []
offset = 0
for size in self.proj_bytes:
offsets.append((offset, size))
offset += size
return offsets
def remote_conv_offsets(
self, local_rank_offset: int, tp_ratio: int
@@ -76,28 +84,23 @@ class MambaConvSplitInfo:
P page. Local dims are scaled down by |tp_ratio|
to get P-sized offsets.
"""
conv0, conv1, conv2 = self.proj_bytes
offsets: list[tuple[int, int]] = []
if tp_ratio >= 1:
remote_conv0 = conv0 * tp_ratio
remote_conv1 = conv1 * tp_ratio
return [
(local_rank_offset * conv0, conv0),
(remote_conv0 + local_rank_offset * conv1, conv1),
(remote_conv0 + remote_conv1 + local_rank_offset * conv2, conv2),
]
remote_base = 0
for size in self.proj_bytes:
offsets.append((remote_base + local_rank_offset * size, size))
remote_base += size * tp_ratio
else:
# NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages
# are smaller than D's. Local dims are D-sized, but we need
# P-sized offsets. Scale down by |tp_ratio|.
abs_ratio = -tp_ratio
remote_conv0 = conv0 // abs_ratio
remote_conv1 = conv1 // abs_ratio
remote_conv2 = conv2 // abs_ratio
return [
(0, remote_conv0),
(remote_conv0, remote_conv1),
(remote_conv0 + remote_conv1, remote_conv2),
]
remote_base = 0
for size in self.proj_bytes:
remote_size = size // abs_ratio
offsets.append((remote_base, remote_size))
remote_base += remote_size
return offsets
def derive_mamba_conv_split(
@@ -120,12 +123,13 @@ def derive_mamba_conv_split(
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
"""
_supported = (
MambaAttentionBackendEnum.MAMBA1,
MambaAttentionBackendEnum.MAMBA2,
MambaAttentionBackendEnum.GDN_ATTN,
)
if mamba_spec.mamba_type not in _supported:
raise NotImplementedError(
f"3-read conv transfer only supports Mamba2 and GDN models, "
f"Conv transfer only supports Mamba1, Mamba2 and GDN models, "
f"got mamba_type={mamba_spec.mamba_type!r}."
)
@@ -149,7 +153,18 @@ def derive_mamba_conv_split(
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2:
local_proj_dims: tuple[int, ...]
if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA1:
# Mamba1 conv state holds only x (no B/C), so it's a single
# contiguous TP shard with no sub-projection decomposition.
temporal_shape = mamba_spec.shapes[1]
assert temporal_shape[0] == local_conv_dim, (
f"Mamba1 temporal state dim ({temporal_shape[0]}) doesn't match "
f"conv dim ({local_conv_dim}); both should be "
f"intermediate_size/TP."
)
local_proj_dims = (local_conv_dim,)
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2:
# NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored
# in MambaSpec, so we reconstruct it from the SSM temporal state shape:
# shapes[1] = (local_num_heads, head_dim), already divided by TP.