[NIXL][Mamba] Add Mamba1 support to NIXL P/D disaggregation (#45019)
Signed-off-by: Josephasafg <ajgard7@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
92221485aa
commit
15be78732b
@@ -28,6 +28,8 @@ hybrid_ssm_configs=(
|
||||
# GDN (Qwen3.5)
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
|
||||
# Mamba1 (Jamba)
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ai21labs/AI21-Jamba2-3B VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
)
|
||||
sw_attn_configs=(
|
||||
# NOTE: gemma3 does not work with FlashInfer
|
||||
|
||||
@@ -25,6 +25,7 @@ EXPECTED_VALUES = {
|
||||
"ibm-granite/granite-4.0-h-tiny": 0.77,
|
||||
"Qwen/Qwen3.5-0.8B": 0.33,
|
||||
"google/gemma-4-E2B-it": 0.485,
|
||||
"ai21labs/AI21-Jamba2-3B": 0.74,
|
||||
}
|
||||
|
||||
SIMPLE_PROMPT = (
|
||||
|
||||
@@ -630,6 +630,19 @@ def _make_mock_worker_for_desc_ids(
|
||||
worker._has_mamba = has_mamba
|
||||
worker._group_spec_types = group_spec_types
|
||||
worker.block_len_per_layer = block_len_per_layer or [100]
|
||||
worker._conv_decomp = None
|
||||
if has_mamba:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501
|
||||
MambaConvSplitInfo,
|
||||
)
|
||||
|
||||
# Mamba2/GDN layout: 3 conv sub-projections -> 4 NIXL regions per layer.
|
||||
worker._conv_decomp = MambaConvSplitInfo(
|
||||
conv_rows=3,
|
||||
local_proj_dims=(1, 1, 1),
|
||||
conv_dtype_size=2,
|
||||
ssm_sizes=(0, 0),
|
||||
)
|
||||
worker._compute_desc_ids = NixlConnectorWorker._compute_desc_ids.__get__(
|
||||
worker, NixlConnectorWorker
|
||||
)
|
||||
@@ -976,6 +989,37 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati
|
||||
(256, 256, 768),
|
||||
id="qwen35_27b_tp8",
|
||||
),
|
||||
# ai21labs/AI21-Jamba2-Mini (Mamba1)
|
||||
# mamba d_inner = mamba_expand(2) * hidden_size(4096) = 8192
|
||||
# mamba_d_state=16, mamba_d_conv=4 → conv_rows=3.
|
||||
# Conv state holds only x: a single contiguous sub-projection.
|
||||
pytest.param(
|
||||
"mamba1",
|
||||
1,
|
||||
8192,
|
||||
3,
|
||||
(8192, 16),
|
||||
(8192,),
|
||||
id="jamba_mini_tp1",
|
||||
),
|
||||
pytest.param(
|
||||
"mamba1",
|
||||
4,
|
||||
2048,
|
||||
3,
|
||||
(2048, 16),
|
||||
(2048,),
|
||||
id="jamba_mini_tp4",
|
||||
),
|
||||
pytest.param(
|
||||
"mamba1",
|
||||
8,
|
||||
1024,
|
||||
3,
|
||||
(1024, 16),
|
||||
(1024,),
|
||||
id="jamba_mini_tp8",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_derive_mamba_conv_split(
|
||||
@@ -999,6 +1043,7 @@ def test_derive_mamba_conv_split(
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
_TYPE_MAP = {
|
||||
"mamba1": MambaAttentionBackendEnum.MAMBA1,
|
||||
"mamba2": MambaAttentionBackendEnum.MAMBA2,
|
||||
"gdn_attention": MambaAttentionBackendEnum.GDN_ATTN,
|
||||
}
|
||||
|
||||
@@ -96,7 +96,13 @@ class NixlBaseConnectorWorker:
|
||||
) -> np.ndarray:
|
||||
"""Compute NIXL descriptor IDs for given block IDs."""
|
||||
num_fa_regions = self.num_regions
|
||||
num_ssm_regions = len(self.block_len_per_layer) * 4 if self._has_mamba else 0
|
||||
num_ssm_regions = 0
|
||||
if self._has_mamba:
|
||||
assert self._conv_decomp is not None
|
||||
# NIXL regions per SSM layer = conv sub-projections + 1 SSM temporal
|
||||
# (Mamba2/GDN: 3+1=4; Mamba1: 1+1=2).
|
||||
ssm_regions_per_layer = len(self._conv_decomp.local_conv_offsets) + 1
|
||||
num_ssm_regions = len(self.block_len_per_layer) * ssm_regions_per_layer
|
||||
|
||||
num_blocks = dst_num_blocks
|
||||
if block_size_ratio is not None:
|
||||
@@ -279,8 +285,8 @@ class NixlBaseConnectorWorker:
|
||||
# ---- Model state (derived from model config) ----
|
||||
mamba_ssm_size = (0, 0)
|
||||
# Conv state sub-projection decomposition (None when no Mamba).
|
||||
# The 3-read transfer requires DS (dim, state_len) conv layout so
|
||||
# that x/B/C sub-projections are contiguous in memory.
|
||||
# The transfer requires DS (dim, state_len) conv layout so that
|
||||
# conv sub-projections are contiguous in memory.
|
||||
self._conv_decomp: MambaConvSplitInfo | None = None
|
||||
self._has_mamba = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
@@ -1186,8 +1192,8 @@ class NixlBaseConnectorWorker:
|
||||
base_addresses: list[int],
|
||||
block_size_ratio: int,
|
||||
) -> list[tuple[int, int, int]]:
|
||||
"""Build 4 desc regions (x, B, C, ssm) per layer for local mamba
|
||||
blocks, enabling the 3-read transfer with DS conv layout."""
|
||||
"""Build desc regions (conv sub-projections + ssm) per layer for
|
||||
local mamba blocks with DS conv layout."""
|
||||
assert block_size_ratio == 1, (
|
||||
"Mamba 3-read transfer with block_size_ratio != 1 is not tested. "
|
||||
f"Got block_size_ratio={block_size_ratio}."
|
||||
@@ -1227,9 +1233,9 @@ class NixlBaseConnectorWorker:
|
||||
tp_ratio: int,
|
||||
transfer_info: EngineTransferInfo,
|
||||
) -> list[tuple[int, int, int]]:
|
||||
"""Build 4 remote desc regions (proj0, proj1, proj2, ssm) per layer
|
||||
for the 3-read transfer. For hetero-TP, each D rank reads only its
|
||||
sub-projection slice from the P rank."""
|
||||
"""Build remote desc regions (conv sub-projections + ssm) per layer.
|
||||
For hetero-TP, each D rank reads only its sub-projection slice from
|
||||
the P rank."""
|
||||
assert self._conv_decomp is not None
|
||||
effective_ratio = max(tp_ratio, 1)
|
||||
# Mamba conv state is always TP-sharded, even when attention KV
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mamba conv-state sub-projection decomposition for the 3-read transfer.
|
||||
"""Mamba conv-state sub-projection decomposition for NIXL transfer.
|
||||
|
||||
With DS conv state layout (dim, state_len), sub-projections are
|
||||
contiguous in memory. Each D rank reads its slices via 3 separate
|
||||
contiguous in memory. Each D rank reads its slices via separate
|
||||
RDMA transfers — no P-side permutation needed.
|
||||
|
||||
Supported model types:
|
||||
- Mamba1: conv = [x], temporal = (intermediate_size, state_size)
|
||||
- Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim)
|
||||
- GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)),
|
||||
temporal = (num_v_heads, v_dim, k_dim)
|
||||
@@ -24,18 +25,21 @@ from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaConvSplitInfo:
|
||||
"""Per-rank byte sizes of the 3 conv sub-projections.
|
||||
"""Per-rank byte sizes of the conv sub-projections.
|
||||
|
||||
Used by both P and D sides for NIXL descriptor registration.
|
||||
All fields are LOCAL to this engine's TP (already divided by TP size).
|
||||
|
||||
DS memory layout within one page (contiguous):
|
||||
Mamba1: |---- x ----| (single sub-projection, no decomposition)
|
||||
Mamba2: |-- x --|- B -|- C -| (B == C)
|
||||
GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ)
|
||||
"""
|
||||
|
||||
conv_rows: int # conv_kernel - 1 (typically 3)
|
||||
local_proj_dims: tuple[int, int, int] # per-rank column counts per sub-proj
|
||||
# Per-rank column counts per sub-projection:
|
||||
# 1 entry for Mamba1, 3 for Mamba2/GDN.
|
||||
local_proj_dims: tuple[int, ...]
|
||||
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
|
||||
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
|
||||
|
||||
@@ -45,10 +49,10 @@ class MambaConvSplitInfo:
|
||||
return sum(self.local_proj_dims)
|
||||
|
||||
@property
|
||||
def proj_bytes(self) -> tuple[int, int, int]:
|
||||
"""Byte sizes of the 3 sub-projections for one rank."""
|
||||
def proj_bytes(self) -> tuple[int, ...]:
|
||||
"""Byte sizes of the sub-projections for one rank."""
|
||||
row_bytes = self.conv_rows * self.conv_dtype_size
|
||||
return tuple(d * row_bytes for d in self.local_proj_dims) # type: ignore[return-value]
|
||||
return tuple(d * row_bytes for d in self.local_proj_dims)
|
||||
|
||||
@property
|
||||
def local_conv_offsets(self) -> list[tuple[int, int]]:
|
||||
@@ -57,8 +61,12 @@ class MambaConvSplitInfo:
|
||||
|
||||
Used by both P and D for local descriptor registration.
|
||||
"""
|
||||
conv0, conv1, conv2 = self.proj_bytes
|
||||
return [(0, conv0), (conv0, conv1), (conv0 + conv1, conv2)]
|
||||
offsets: list[tuple[int, int]] = []
|
||||
offset = 0
|
||||
for size in self.proj_bytes:
|
||||
offsets.append((offset, size))
|
||||
offset += size
|
||||
return offsets
|
||||
|
||||
def remote_conv_offsets(
|
||||
self, local_rank_offset: int, tp_ratio: int
|
||||
@@ -76,28 +84,23 @@ class MambaConvSplitInfo:
|
||||
P page. Local dims are scaled down by |tp_ratio|
|
||||
to get P-sized offsets.
|
||||
"""
|
||||
conv0, conv1, conv2 = self.proj_bytes
|
||||
offsets: list[tuple[int, int]] = []
|
||||
if tp_ratio >= 1:
|
||||
remote_conv0 = conv0 * tp_ratio
|
||||
remote_conv1 = conv1 * tp_ratio
|
||||
return [
|
||||
(local_rank_offset * conv0, conv0),
|
||||
(remote_conv0 + local_rank_offset * conv1, conv1),
|
||||
(remote_conv0 + remote_conv1 + local_rank_offset * conv2, conv2),
|
||||
]
|
||||
remote_base = 0
|
||||
for size in self.proj_bytes:
|
||||
offsets.append((remote_base + local_rank_offset * size, size))
|
||||
remote_base += size * tp_ratio
|
||||
else:
|
||||
# NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages
|
||||
# are smaller than D's. Local dims are D-sized, but we need
|
||||
# P-sized offsets. Scale down by |tp_ratio|.
|
||||
abs_ratio = -tp_ratio
|
||||
remote_conv0 = conv0 // abs_ratio
|
||||
remote_conv1 = conv1 // abs_ratio
|
||||
remote_conv2 = conv2 // abs_ratio
|
||||
return [
|
||||
(0, remote_conv0),
|
||||
(remote_conv0, remote_conv1),
|
||||
(remote_conv0 + remote_conv1, remote_conv2),
|
||||
]
|
||||
remote_base = 0
|
||||
for size in self.proj_bytes:
|
||||
remote_size = size // abs_ratio
|
||||
offsets.append((remote_base, remote_size))
|
||||
remote_base += remote_size
|
||||
return offsets
|
||||
|
||||
|
||||
def derive_mamba_conv_split(
|
||||
@@ -120,12 +123,13 @@ def derive_mamba_conv_split(
|
||||
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
|
||||
"""
|
||||
_supported = (
|
||||
MambaAttentionBackendEnum.MAMBA1,
|
||||
MambaAttentionBackendEnum.MAMBA2,
|
||||
MambaAttentionBackendEnum.GDN_ATTN,
|
||||
)
|
||||
if mamba_spec.mamba_type not in _supported:
|
||||
raise NotImplementedError(
|
||||
f"3-read conv transfer only supports Mamba2 and GDN models, "
|
||||
f"Conv transfer only supports Mamba1, Mamba2 and GDN models, "
|
||||
f"got mamba_type={mamba_spec.mamba_type!r}."
|
||||
)
|
||||
|
||||
@@ -149,7 +153,18 @@ def derive_mamba_conv_split(
|
||||
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
|
||||
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
|
||||
|
||||
if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2:
|
||||
local_proj_dims: tuple[int, ...]
|
||||
if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA1:
|
||||
# Mamba1 conv state holds only x (no B/C), so it's a single
|
||||
# contiguous TP shard with no sub-projection decomposition.
|
||||
temporal_shape = mamba_spec.shapes[1]
|
||||
assert temporal_shape[0] == local_conv_dim, (
|
||||
f"Mamba1 temporal state dim ({temporal_shape[0]}) doesn't match "
|
||||
f"conv dim ({local_conv_dim}); both should be "
|
||||
f"intermediate_size/TP."
|
||||
)
|
||||
local_proj_dims = (local_conv_dim,)
|
||||
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2:
|
||||
# NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored
|
||||
# in MambaSpec, so we reconstruct it from the SSM temporal state shape:
|
||||
# shapes[1] = (local_num_heads, head_dim), already divided by TP.
|
||||
|
||||
Reference in New Issue
Block a user