[NIXL][Mamba] Add Mamba1 support to NIXL P/D disaggregation (#45019)

Signed-off-by: Josephasafg <ajgard7@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Asaf Gardin
2026-06-25 05:50:41 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 92221485aa
commit 15be78732b
5 changed files with 104 additions and 35 deletions
@@ -28,6 +28,8 @@ hybrid_ssm_configs=(
# GDN (Qwen3.5)
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
"VLLM_SSM_CONV_STATE_LAYOUT=DS PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
# Mamba1 (Jamba)
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ai21labs/AI21-Jamba2-3B VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
)
sw_attn_configs=(
# NOTE: gemma3 does not work with FlashInfer
@@ -25,6 +25,7 @@ EXPECTED_VALUES = {
"ibm-granite/granite-4.0-h-tiny": 0.77,
"Qwen/Qwen3.5-0.8B": 0.33,
"google/gemma-4-E2B-it": 0.485,
"ai21labs/AI21-Jamba2-3B": 0.74,
}
SIMPLE_PROMPT = (
@@ -630,6 +630,19 @@ def _make_mock_worker_for_desc_ids(
worker._has_mamba = has_mamba
worker._group_spec_types = group_spec_types
worker.block_len_per_layer = block_len_per_layer or [100]
worker._conv_decomp = None
if has_mamba:
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501
MambaConvSplitInfo,
)
# Mamba2/GDN layout: 3 conv sub-projections -> 4 NIXL regions per layer.
worker._conv_decomp = MambaConvSplitInfo(
conv_rows=3,
local_proj_dims=(1, 1, 1),
conv_dtype_size=2,
ssm_sizes=(0, 0),
)
worker._compute_desc_ids = NixlConnectorWorker._compute_desc_ids.__get__(
worker, NixlConnectorWorker
)
@@ -976,6 +989,37 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati
(256, 256, 768),
id="qwen35_27b_tp8",
),
# ai21labs/AI21-Jamba2-Mini (Mamba1)
# mamba d_inner = mamba_expand(2) * hidden_size(4096) = 8192
# mamba_d_state=16, mamba_d_conv=4 → conv_rows=3.
# Conv state holds only x: a single contiguous sub-projection.
pytest.param(
"mamba1",
1,
8192,
3,
(8192, 16),
(8192,),
id="jamba_mini_tp1",
),
pytest.param(
"mamba1",
4,
2048,
3,
(2048, 16),
(2048,),
id="jamba_mini_tp4",
),
pytest.param(
"mamba1",
8,
1024,
3,
(1024, 16),
(1024,),
id="jamba_mini_tp8",
),
],
)
def test_derive_mamba_conv_split(
@@ -999,6 +1043,7 @@ def test_derive_mamba_conv_split(
from vllm.v1.kv_cache_interface import MambaSpec
_TYPE_MAP = {
"mamba1": MambaAttentionBackendEnum.MAMBA1,
"mamba2": MambaAttentionBackendEnum.MAMBA2,
"gdn_attention": MambaAttentionBackendEnum.GDN_ATTN,
}