[NIXL][Mamba] Add Mamba1 support to NIXL P/D disaggregation (#45019)
Signed-off-by: Josephasafg <ajgard7@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
92221485aa
commit
15be78732b
@@ -28,6 +28,8 @@ hybrid_ssm_configs=(
|
||||
# GDN (Qwen3.5)
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
|
||||
# Mamba1 (Jamba)
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ai21labs/AI21-Jamba2-3B VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
)
|
||||
sw_attn_configs=(
|
||||
# NOTE: gemma3 does not work with FlashInfer
|
||||
|
||||
@@ -25,6 +25,7 @@ EXPECTED_VALUES = {
|
||||
"ibm-granite/granite-4.0-h-tiny": 0.77,
|
||||
"Qwen/Qwen3.5-0.8B": 0.33,
|
||||
"google/gemma-4-E2B-it": 0.485,
|
||||
"ai21labs/AI21-Jamba2-3B": 0.74,
|
||||
}
|
||||
|
||||
SIMPLE_PROMPT = (
|
||||
|
||||
@@ -630,6 +630,19 @@ def _make_mock_worker_for_desc_ids(
|
||||
worker._has_mamba = has_mamba
|
||||
worker._group_spec_types = group_spec_types
|
||||
worker.block_len_per_layer = block_len_per_layer or [100]
|
||||
worker._conv_decomp = None
|
||||
if has_mamba:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501
|
||||
MambaConvSplitInfo,
|
||||
)
|
||||
|
||||
# Mamba2/GDN layout: 3 conv sub-projections -> 4 NIXL regions per layer.
|
||||
worker._conv_decomp = MambaConvSplitInfo(
|
||||
conv_rows=3,
|
||||
local_proj_dims=(1, 1, 1),
|
||||
conv_dtype_size=2,
|
||||
ssm_sizes=(0, 0),
|
||||
)
|
||||
worker._compute_desc_ids = NixlConnectorWorker._compute_desc_ids.__get__(
|
||||
worker, NixlConnectorWorker
|
||||
)
|
||||
@@ -976,6 +989,37 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati
|
||||
(256, 256, 768),
|
||||
id="qwen35_27b_tp8",
|
||||
),
|
||||
# ai21labs/AI21-Jamba2-Mini (Mamba1)
|
||||
# mamba d_inner = mamba_expand(2) * hidden_size(4096) = 8192
|
||||
# mamba_d_state=16, mamba_d_conv=4 → conv_rows=3.
|
||||
# Conv state holds only x: a single contiguous sub-projection.
|
||||
pytest.param(
|
||||
"mamba1",
|
||||
1,
|
||||
8192,
|
||||
3,
|
||||
(8192, 16),
|
||||
(8192,),
|
||||
id="jamba_mini_tp1",
|
||||
),
|
||||
pytest.param(
|
||||
"mamba1",
|
||||
4,
|
||||
2048,
|
||||
3,
|
||||
(2048, 16),
|
||||
(2048,),
|
||||
id="jamba_mini_tp4",
|
||||
),
|
||||
pytest.param(
|
||||
"mamba1",
|
||||
8,
|
||||
1024,
|
||||
3,
|
||||
(1024, 16),
|
||||
(1024,),
|
||||
id="jamba_mini_tp8",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_derive_mamba_conv_split(
|
||||
@@ -999,6 +1043,7 @@ def test_derive_mamba_conv_split(
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
_TYPE_MAP = {
|
||||
"mamba1": MambaAttentionBackendEnum.MAMBA1,
|
||||
"mamba2": MambaAttentionBackendEnum.MAMBA2,
|
||||
"gdn_attention": MambaAttentionBackendEnum.GDN_ATTN,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user