PD disagg with NIXL Connector: GDN support (Qwen3.5) (#41869)
Signed-off-by: Zhanqiu Hu <zhu@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
c7560af424
commit
24337fb860
@@ -51,7 +51,7 @@ steps:
|
||||
|
||||
- label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs)
|
||||
key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 25
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -21,6 +21,9 @@ hybrid_ssm_configs=(
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code"
|
||||
# TODO: (NickLucche) Address async scheduling issue with TP>1 separately as this may impact other models.
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code,--no-async-scheduling"
|
||||
# GDN (Qwen3.5)
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B VLLM_SERVE_EXTRA_ARGS=--no-async-scheduling"
|
||||
)
|
||||
sw_attn_configs=(
|
||||
# NOTE: gemma3 does not work with FlashInfer
|
||||
|
||||
@@ -20,6 +20,7 @@ EXPECTED_VALUES = {
|
||||
"google/gemma-3-4b-it": 0.74,
|
||||
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": 0.84,
|
||||
"ibm-granite/granite-4.0-h-tiny": 0.80,
|
||||
"Qwen/Qwen3.5-0.8B": 0.33,
|
||||
}
|
||||
|
||||
SIMPLE_PROMPT = (
|
||||
|
||||
@@ -742,6 +742,132 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati
|
||||
assert compute_physical_blocks_per_logical(ssm_sizes, block_len) == expected_ratio
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"mamba_type,local_tp,conv_dim_local,conv_rows,temporal_shape,expected_proj_dims",
|
||||
[
|
||||
# nvidia/Nemotron-H-8B-Base-8K (Mamba2)
|
||||
# mamba_num_heads=128, head_dim=64, n_groups=8, ssm_state_size=128
|
||||
pytest.param(
|
||||
"mamba2",
|
||||
1,
|
||||
10240,
|
||||
3,
|
||||
(128, 64, 128),
|
||||
(8192, 1024, 1024),
|
||||
id="nemotron_h_8b_tp1",
|
||||
),
|
||||
pytest.param(
|
||||
"mamba2",
|
||||
4,
|
||||
2560,
|
||||
3,
|
||||
(32, 64, 128),
|
||||
(2048, 256, 256),
|
||||
id="nemotron_h_8b_tp4",
|
||||
),
|
||||
# nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B (Mamba2)
|
||||
# mamba_num_heads=64, head_dim=64, n_groups=8, ssm_state_size=128
|
||||
pytest.param(
|
||||
"mamba2",
|
||||
1,
|
||||
6144,
|
||||
3,
|
||||
(64, 64, 128),
|
||||
(4096, 1024, 1024),
|
||||
id="nemotron_nano_30b_tp1",
|
||||
),
|
||||
# Qwen/Qwen3.5-0.8B (GDN, symmetric: num_v=num_k=16)
|
||||
# key_dim=2048, value_dim=2048, conv_dim=6144
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
1,
|
||||
6144,
|
||||
3,
|
||||
(16, 128, 128),
|
||||
(2048, 2048, 2048),
|
||||
id="qwen35_08b_tp1",
|
||||
),
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
4,
|
||||
1536,
|
||||
3,
|
||||
(4, 128, 128),
|
||||
(512, 512, 512),
|
||||
id="qwen35_08b_tp4",
|
||||
),
|
||||
# Qwen/Qwen3.5-4B (GDN, asymmetric: num_v=32, num_k=16, K:V=1:2)
|
||||
# key_dim=2048, value_dim=4096, conv_dim=8192
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
1,
|
||||
8192,
|
||||
3,
|
||||
(32, 128, 128),
|
||||
(2048, 2048, 4096),
|
||||
id="qwen35_4b_tp1",
|
||||
),
|
||||
# Qwen/Qwen3.5-27B (GDN, asymmetric: num_v=48, num_k=16, K:V=1:3)
|
||||
# key_dim=2048, value_dim=6144, conv_dim=10240
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
1,
|
||||
10240,
|
||||
3,
|
||||
(48, 128, 128),
|
||||
(2048, 2048, 6144),
|
||||
id="qwen35_27b_tp1",
|
||||
),
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
8,
|
||||
1280,
|
||||
3,
|
||||
(6, 128, 128),
|
||||
(256, 256, 768),
|
||||
id="qwen35_27b_tp8",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_derive_mamba_conv_split(
|
||||
monkeypatch,
|
||||
mamba_type,
|
||||
local_tp,
|
||||
conv_dim_local,
|
||||
conv_rows,
|
||||
temporal_shape,
|
||||
expected_proj_dims,
|
||||
):
|
||||
"""Parametrized test for derive_mamba_conv_split with real model configs.
|
||||
|
||||
Values generated by verify_conv_split.py which loads HuggingFace configs
|
||||
and calls vLLM's derive_mamba_conv_split directly.
|
||||
"""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
|
||||
derive_mamba_conv_split,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
_TYPE_MAP = {
|
||||
"mamba2": MambaAttentionBackendEnum.MAMBA2,
|
||||
"gdn_attention": MambaAttentionBackendEnum.GDN_ATTN,
|
||||
}
|
||||
mamba_type_enum = _TYPE_MAP[mamba_type]
|
||||
|
||||
monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", "DS")
|
||||
spec = MambaSpec(
|
||||
block_size=64,
|
||||
shapes=((conv_dim_local, conv_rows), temporal_shape),
|
||||
dtypes=(torch.bfloat16, torch.bfloat16),
|
||||
mamba_type=mamba_type_enum,
|
||||
)
|
||||
out = derive_mamba_conv_split(spec, local_tp=local_tp)
|
||||
assert out.local_proj_dims == expected_proj_dims
|
||||
assert out.conv_rows == conv_rows
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"mamba_enabled,swa_enabled,"
|
||||
|
||||
@@ -1057,8 +1057,8 @@ class NixlConnectorWorker:
|
||||
tp_ratio: int,
|
||||
transfer_info: EngineTransferInfo,
|
||||
) -> list[tuple[int, int, int]]:
|
||||
"""Build 4 remote desc regions (x, B, C, ssm) per layer for
|
||||
the 3-read transfer. For hetero-TP, each D rank reads only its
|
||||
"""Build 4 remote desc regions (proj0, proj1, proj2, ssm) per layer
|
||||
for the 3-read transfer. For hetero-TP, each D rank reads only its
|
||||
sub-projection slice from the P rank."""
|
||||
assert self._conv_decomp is not None
|
||||
effective_ratio = max(tp_ratio, 1)
|
||||
@@ -1067,20 +1067,10 @@ class NixlConnectorWorker:
|
||||
local_offset = self.tp_rank % effective_ratio
|
||||
conv_size_remote = nixl_agent_meta.ssm_sizes[0]
|
||||
|
||||
conv_offsets = self._conv_decomp.remote_conv_offsets(local_offset, tp_ratio)
|
||||
if tp_ratio >= 1:
|
||||
# D_TP >= P_TP: P page is larger, D reads its slice.
|
||||
conv_offsets = self._conv_decomp.remote_conv_offsets(
|
||||
local_offset, effective_ratio
|
||||
)
|
||||
ssm_read_size = self._mamba_ssm_size[1]
|
||||
else:
|
||||
# NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages
|
||||
# are smaller than D's. self._conv_decomp has D-sized dimensions,
|
||||
# but we need P-sized offsets. Scale down by |tp_ratio|.
|
||||
abs_ratio = -tp_ratio
|
||||
xb_p = self._conv_decomp.x_bytes // abs_ratio
|
||||
bb_p = self._conv_decomp.b_bytes // abs_ratio
|
||||
conv_offsets = [(0, xb_p), (xb_p, bb_p), (xb_p + bb_p, bb_p)]
|
||||
ssm_read_size = nixl_agent_meta.ssm_sizes[1]
|
||||
|
||||
remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical
|
||||
|
||||
@@ -2,9 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mamba conv-state sub-projection decomposition for the 3-read transfer.
|
||||
|
||||
With DS conv state layout (dim, state_len), x/B/C sub-projections are
|
||||
contiguous in memory. Each D rank reads its x, B, C slices via 3
|
||||
separate RDMA transfers — no P-side permutation needed.
|
||||
With DS conv state layout (dim, state_len), sub-projections are
|
||||
contiguous in memory. Each D rank reads its slices via 3 separate
|
||||
RDMA transfers — no P-side permutation needed.
|
||||
|
||||
Supported model types:
|
||||
- Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim)
|
||||
- GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)),
|
||||
temporal = (num_v_heads, v_dim, k_dim)
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -19,97 +24,109 @@ from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaConvSplitInfo:
|
||||
"""Per-rank byte sizes of x, B, C sub-projections in the Mamba conv state.
|
||||
"""Per-rank byte sizes of the 3 conv sub-projections.
|
||||
|
||||
Used by both P and D sides for NIXL descriptor registration.
|
||||
All fields are LOCAL to this engine's TP (already divided by TP size).
|
||||
|
||||
DS memory layout within one page (contiguous in memory):
|
||||
|--- x (x_local * conv_rows) ---|- B (b_local * conv_rows) -|- C -|
|
||||
DS memory layout within one page (contiguous):
|
||||
Mamba2: |-- x --|- B -|- C -| (B == C)
|
||||
GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ)
|
||||
"""
|
||||
|
||||
conv_rows: int # conv_kernel - 1 (typically 3)
|
||||
x_local: int # intermediate_size / TP (columns for x)
|
||||
b_local: int # groups_ss / TP (columns for B; C is same size)
|
||||
local_proj_dims: tuple[int, int, int] # per-rank column counts per sub-proj
|
||||
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
|
||||
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
|
||||
|
||||
@property
|
||||
def conv_dim_local(self) -> int:
|
||||
"""Total conv columns per rank: x + B + C."""
|
||||
return self.x_local + 2 * self.b_local
|
||||
def local_conv_dim(self) -> int:
|
||||
"""Total conv columns per rank."""
|
||||
return sum(self.local_proj_dims)
|
||||
|
||||
@property
|
||||
def x_bytes(self) -> int:
|
||||
"""Byte size of the x sub-projection for one rank."""
|
||||
return self.x_local * self.conv_rows * self.conv_dtype_size
|
||||
|
||||
@property
|
||||
def b_bytes(self) -> int:
|
||||
"""Byte size of the B (or C) sub-projection for one rank."""
|
||||
return self.b_local * self.conv_rows * self.conv_dtype_size
|
||||
def proj_bytes(self) -> tuple[int, int, int]:
|
||||
"""Byte sizes of the 3 sub-projections for one rank."""
|
||||
row_bytes = self.conv_rows * self.conv_dtype_size
|
||||
return tuple(d * row_bytes for d in self.local_proj_dims) # type: ignore[return-value]
|
||||
|
||||
@property
|
||||
def local_conv_offsets(self) -> list[tuple[int, int]]:
|
||||
"""(byte_offset, byte_size) of x, B, C within this engine's page.
|
||||
"""(byte_offset, byte_size) of each sub-projection within this
|
||||
engine's page.
|
||||
|
||||
Used by both P and D for local descriptor registration.
|
||||
"""
|
||||
xb = self.x_bytes
|
||||
bb = self.b_bytes
|
||||
return [(0, xb), (xb, bb), (xb + bb, bb)]
|
||||
conv0, conv1, conv2 = self.proj_bytes
|
||||
return [(0, conv0), (conv0, conv1), (conv0 + conv1, conv2)]
|
||||
|
||||
def remote_conv_offsets(
|
||||
self, local_rank_offset: int, tp_ratio: int
|
||||
) -> list[tuple[int, int]]:
|
||||
"""(byte_offset, byte_size) of this D rank's x, B, C slice within
|
||||
one P page.
|
||||
"""(byte_offset, byte_size) of this D rank's sub-projection slices
|
||||
within one P page.
|
||||
|
||||
Used by D side only, during remote descriptor registration.
|
||||
|
||||
Args:
|
||||
local_rank_offset: which slice this D rank reads.
|
||||
tp_ratio > 0: tp_rank % tp_ratio (selects slice of P's page).
|
||||
tp_ratio < 0: always 0 (read P's full page).
|
||||
tp_ratio: effective ratio (>= 1 when D_TP > P_TP, 1 when
|
||||
P_TP > D_TP since each P rank is read in full).
|
||||
tp_ratio: signed TP ratio.
|
||||
>= 1: D_TP >= P_TP — P page is larger, D reads its slice.
|
||||
< 0: P_TP > D_TP — P pages are smaller, D reads entire
|
||||
P page. Local dims are scaled down by |tp_ratio|
|
||||
to get P-sized offsets.
|
||||
"""
|
||||
xb = self.x_bytes
|
||||
bb = self.b_bytes
|
||||
xr = xb * tp_ratio # full remote x section in bytes
|
||||
br = bb * tp_ratio # full remote B section in bytes
|
||||
return [
|
||||
(local_rank_offset * xb, xb),
|
||||
(xr + local_rank_offset * bb, bb),
|
||||
(xr + br + local_rank_offset * bb, bb),
|
||||
]
|
||||
conv0, conv1, conv2 = self.proj_bytes
|
||||
if tp_ratio >= 1:
|
||||
remote_conv0 = conv0 * tp_ratio
|
||||
remote_conv1 = conv1 * tp_ratio
|
||||
return [
|
||||
(local_rank_offset * conv0, conv0),
|
||||
(remote_conv0 + local_rank_offset * conv1, conv1),
|
||||
(remote_conv0 + remote_conv1 + local_rank_offset * conv2, conv2),
|
||||
]
|
||||
else:
|
||||
# NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages
|
||||
# are smaller than D's. Local dims are D-sized, but we need
|
||||
# P-sized offsets. Scale down by |tp_ratio|.
|
||||
abs_ratio = -tp_ratio
|
||||
remote_conv0 = conv0 // abs_ratio
|
||||
remote_conv1 = conv1 // abs_ratio
|
||||
remote_conv2 = conv2 // abs_ratio
|
||||
return [
|
||||
(0, remote_conv0),
|
||||
(remote_conv0, remote_conv1),
|
||||
(remote_conv0 + remote_conv1, remote_conv2),
|
||||
]
|
||||
|
||||
|
||||
def derive_mamba_conv_split(
|
||||
mamba_spec: MambaSpec,
|
||||
local_tp: int,
|
||||
) -> MambaConvSplitInfo:
|
||||
"""Derive per-rank x/B/C byte sizes from a MambaSpec.
|
||||
"""Derive per-rank sub-projection byte sizes from a MambaSpec.
|
||||
|
||||
Called once at init on both P and D. Decomposes the conv dimension
|
||||
(= intermediate_size + 2 * groups_ss) into its x, B, C parts.
|
||||
into its sub-projection parts based on the model type.
|
||||
|
||||
Args:
|
||||
mamba_spec: MambaSpec whose shapes are:
|
||||
shapes[0] = conv state: (conv_dim_local, conv_rows) in DS layout.
|
||||
shapes[1] = SSM temporal: (local_num_heads, head_dim).
|
||||
shapes[1] = temporal state (model-specific shape).
|
||||
local_tp: this engine's tensor-parallel size.
|
||||
|
||||
Returns:
|
||||
MambaConvSplitInfo with per-rank x_local, b_local, conv_rows,
|
||||
MambaConvSplitInfo with per-rank sub-projection dims, conv_rows,
|
||||
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
|
||||
"""
|
||||
if mamba_spec.mamba_type != MambaAttentionBackendEnum.MAMBA2:
|
||||
_supported = (
|
||||
MambaAttentionBackendEnum.MAMBA2,
|
||||
MambaAttentionBackendEnum.GDN_ATTN,
|
||||
)
|
||||
if mamba_spec.mamba_type not in _supported:
|
||||
raise NotImplementedError(
|
||||
f"3-read conv transfer only supports Mamba2 models, "
|
||||
f"got mamba_type={mamba_spec.mamba_type!r}. "
|
||||
f"Mamba1 SSM temporal shape is (intermediate_size // tp, state_size) "
|
||||
f"which cannot be used to reconstruct intermediate_size."
|
||||
f"3-read conv transfer only supports Mamba2 and GDN models, "
|
||||
f"got mamba_type={mamba_spec.mamba_type!r}."
|
||||
)
|
||||
|
||||
conv_shape = mamba_spec.shapes[0]
|
||||
@@ -121,29 +138,10 @@ def derive_mamba_conv_split(
|
||||
local_conv_dim = conv_shape[0] # DS: (conv_dim_local, conv_rows)
|
||||
conv_rows = conv_shape[1]
|
||||
|
||||
# NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored
|
||||
# in MambaSpec, so we reconstruct it from the SSM temporal state shape:
|
||||
# shapes[1] = (local_num_heads, head_dim), already divided by TP.
|
||||
head_dim = mamba_spec.shapes[1][1]
|
||||
local_num_heads = mamba_spec.shapes[1][0]
|
||||
intermediate_size = local_num_heads * local_tp * head_dim
|
||||
|
||||
# NOTE (ZhanqiuHu): global conv dim = intermediate_size + 2 * groups_ss,
|
||||
# where groups_ss is the B (= C) dimension. B and C are always the same
|
||||
# size, so we recover groups_ss from the remainder after subtracting x.
|
||||
remainder = local_conv_dim * local_tp - intermediate_size
|
||||
assert remainder > 0 and remainder % 2 == 0, (
|
||||
f"Conv dim ({local_conv_dim}*tp={local_tp}) doesn't decompose into "
|
||||
f"intermediate_size={intermediate_size} + 2*groups_ss. "
|
||||
f"remainder={remainder}"
|
||||
)
|
||||
groups_ss = remainder // 2
|
||||
|
||||
conv_dtype_size = torch.tensor(
|
||||
[],
|
||||
dtype=mamba_spec.dtypes[0], # type: ignore[misc]
|
||||
).element_size()
|
||||
|
||||
ssm_dtype_size = torch.tensor(
|
||||
[],
|
||||
dtype=mamba_spec.dtypes[1], # type: ignore[misc]
|
||||
@@ -151,11 +149,54 @@ def derive_mamba_conv_split(
|
||||
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
|
||||
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
|
||||
|
||||
# Divide by TP to get per-rank column counts.
|
||||
if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2:
|
||||
# NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored
|
||||
# in MambaSpec, so we reconstruct it from the SSM temporal state shape:
|
||||
# shapes[1] = (local_num_heads, head_dim), already divided by TP.
|
||||
head_dim = mamba_spec.shapes[1][1]
|
||||
local_num_heads = mamba_spec.shapes[1][0]
|
||||
intermediate_size = local_num_heads * local_tp * head_dim
|
||||
|
||||
# NOTE (ZhanqiuHu): global conv dim = intermediate_size + 2 * groups_ss,
|
||||
# where groups_ss is the B (= C) dimension. B and C are always the same
|
||||
# size, so we recover groups_ss from the remainder after subtracting x.
|
||||
remainder = local_conv_dim * local_tp - intermediate_size
|
||||
assert remainder > 0 and remainder % 2 == 0, (
|
||||
f"Conv dim ({local_conv_dim}*tp={local_tp}) doesn't decompose "
|
||||
f"into intermediate_size={intermediate_size} + 2*groups_ss. "
|
||||
f"remainder={remainder}"
|
||||
)
|
||||
groups_ss = remainder // 2
|
||||
|
||||
# Divide by TP to get per-rank column counts.
|
||||
x_local = intermediate_size // local_tp
|
||||
b_local = groups_ss // local_tp
|
||||
local_proj_dims = (x_local, b_local, b_local)
|
||||
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.GDN_ATTN:
|
||||
# GDN: conv = [Q, K, V] where dim(Q) == dim(K) == key_dim.
|
||||
# conv_dim = key_dim*2 + value_dim (all global, divided by TP).
|
||||
# Temporal state shape is (num_v_heads/TP, head_v_dim, head_k_dim).
|
||||
temporal_shape = mamba_spec.shapes[1]
|
||||
num_v_heads_local = temporal_shape[0]
|
||||
head_v_dim = temporal_shape[1]
|
||||
value_dim_local = num_v_heads_local * head_v_dim
|
||||
|
||||
remainder = local_conv_dim - value_dim_local
|
||||
assert remainder > 0 and remainder % 2 == 0, (
|
||||
f"GDN conv dim ({local_conv_dim}) doesn't decompose into "
|
||||
f"2*key_dim_local + value_dim_local={value_dim_local}. "
|
||||
f"remainder={remainder}"
|
||||
)
|
||||
key_dim_local = remainder // 2
|
||||
local_proj_dims = (key_dim_local, key_dim_local, value_dim_local)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Conv split not supported for mamba_type={mamba_spec.mamba_type!r}"
|
||||
)
|
||||
|
||||
return MambaConvSplitInfo(
|
||||
conv_rows=conv_rows,
|
||||
x_local=intermediate_size // local_tp,
|
||||
b_local=groups_ss // local_tp,
|
||||
local_proj_dims=local_proj_dims,
|
||||
conv_dtype_size=conv_dtype_size,
|
||||
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user