[Model] Add MiniMax M3 support (#45381)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
youkaichao
2026-06-16 01:01:25 +08:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py Thien Tran Bugen Zhao Jee Jee Li Roger Wang functionstackx Yongye Zhu Jee Jee Li
parent a3195fab7b
commit 0a1c5034f5
108 changed files with 14734 additions and 311 deletions
+33
View File
@@ -0,0 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MiniMax M3 model — hardware-isolated entry point.
The implementation lives under ``nvidia/`` and ``amd/``; this module picks the
right one for the current platform and re-exports the public classes used by
the model registry. (Mirrors ``vllm.models.deepseek_v4``.)
"""
from typing import TYPE_CHECKING
from vllm.platforms import current_platform
# The NVIDIA branch is the static default that type-checkers see; the ROCm
# branch overrides it at runtime (kept type-compatible via type: ignore).
if TYPE_CHECKING or not current_platform.is_rocm():
from .nvidia.model import (
MiniMaxM3SparseForCausalLM,
MiniMaxM3SparseForConditionalGeneration,
)
from .nvidia.mtp import MiniMaxM3MTP
else:
from .amd.model import ( # type: ignore[assignment]
MiniMaxM3SparseForCausalLM,
MiniMaxM3SparseForConditionalGeneration,
)
from .amd.mtp import MiniMaxM3MTP # type: ignore[assignment]
__all__ = [
"MiniMaxM3MTP",
"MiniMaxM3SparseForCausalLM",
"MiniMaxM3SparseForConditionalGeneration",
]
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
File diff suppressed because it is too large Load Diff
+330
View File
@@ -0,0 +1,330 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MiniMax M3 MTP (multi-token prediction) draft model -- ROCm/AMD variant.
Byte-identical to ``nvidia/mtp.py`` except this file lives under ``amd/`` so its
``from .model import ...`` resolves to the self-contained AMD model (native Gemma
RMSNorm, native MXFP8 MoE, Triton sparse attention). The MTP logic is
platform-agnostic. (Mirrors ``vllm.models.deepseek_v4.amd.mtp``.)
TODO(future, separate diff): since this is byte-identical to ``nvidia/mtp.py``,
both copies could be consolidated into a single ``common/mtp.py`` that dispatches
its model import (``..amd.model`` vs ``..nvidia.model``) via
``current_platform.is_rocm()`` -- the same dispatch ``minimax_m3/__init__.py``
uses. This was prototyped and VERIFIED working (``MiniMaxM3MTP`` resolves through
``common.mtp`` to the AMD decoder layer / RMSNorm on ROCm), but it deletes the
upstream ``nvidia/mtp.py`` and touches the NVIDIA load path, so it is deferred to
a dedicated refactor diff to keep this AMD-enablement change NVIDIA-untouched.
"""
from collections.abc import Iterable
import regex as re
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.linear import (
ReplicatedLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from .model import (
MiniMAXGemmaRMSNorm,
MiniMaxM3DecoderLayer,
)
class MiniMaxM3MultiTokenPredictorLayer(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.enorm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = ReplicatedLinear(
config.hidden_size * 2,
config.hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.eh_proj",
)
self.transformer_layer = MiniMaxM3DecoderLayer(
config=config,
prefix=prefix,
cache_config=cache_config,
quant_config=quant_config,
force_sparse_attn=True,
force_moe=True,
)
self.final_layernorm = MiniMAXGemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None
# Mask out inputs at position 0, as not needed by MTP.
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
# Combine the normalized token embeddings with the normalized
# previous hidden states.
inputs_embeds = self.enorm(inputs_embeds)
previous_hidden_states = self.hnorm(previous_hidden_states)
hidden_states, _ = self.eh_proj(
torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
)
# Apply transformer layer.
hidden_states, residual = self.transformer_layer(
positions=positions,
hidden_states=hidden_states,
residual=None,
)
hidden_states += residual
return hidden_states
class MiniMaxM3MultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
assert vllm_config.speculative_config is not None
# Use the draft (MTP) config, not the target model's. This is flat for a
# standalone checkpoint, and the promoted text_config for a bundled one.
config = vllm_config.speculative_config.draft_model_config.hf_config
self.num_mtp_layers = config.num_mtp_modules
self.layers = torch.nn.ModuleDict(
{
str(idx): MiniMaxM3MultiTokenPredictorLayer(
vllm_config, f"{prefix}.layers.{idx}"
)
for idx in range(self.num_mtp_layers)
}
)
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
current_step_idx = spec_step_idx % self.num_mtp_layers
return self.layers[str(current_step_idx)](
input_ids,
positions,
previous_hidden_states,
inputs_embeds,
current_step_idx,
)
class MiniMaxM3MTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
self.quant_config = vllm_config.quant_config
self.model = MiniMaxM3MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.lm_head = ParallelLMHead(
self.config.vocab_size,
self.config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(self.config.vocab_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
return self.model(
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor | None:
current_step_idx = spec_step_idx % self.model.num_mtp_layers
mtp_layer = self.model.layers[str(current_step_idx)]
return self.logits_processor(
self.lm_head, mtp_layer.final_layernorm(hidden_states)
)
def _get_mtp_layer_idx_from_weight_name(self, name: str) -> int | None:
"""Return the MTP layer index in *.mtp.layers.{idx}.*, else None."""
match = re.search(r"\.mtp\.layers\.(\d+)\.", name)
return int(match.group(1)) if match else None
def _map_checkpoint_name(self, name: str) -> str | None:
"""Map a full checkpoint key to this MTP module's parameter name.
The MTP module only owns the *.mtp.layers.* weights plus the token
embedding and LM head, which the checkpoint shares with the main model.
Everything else belongs to other modules and is ignored here by returning
None.
"""
# In the bundled checkpoint, the MTP weights are prefixed with
# "language_model". The standalone MTP checkpoint has no such prefix.
# Strip it if present.
name = name.removeprefix("language_model.")
if name == "model.embed_tokens.weight":
return "model.embed_tokens.weight"
if name == "lm_head.weight":
return "lm_head.weight"
if "model.mtp.layers" in name:
if "weight_scale_inv" in name:
# The checkpoint stores block scales as "weight_scale_inv".
# The ModelOpt MXFP8 layers expose them as "weight_scale".
name = name.replace("weight_scale_inv", "weight_scale")
# Strip "mtp" from prefix.
return name.replace(".mtp.", ".")
return None
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Map q/k/v projections to qkv_proj, and gate/up projections to gate_up_proj.
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".qkv_proj", ".index_q_proj", "index_q"),
(".qkv_proj", ".index_k_proj", "index_k"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
# Map expert weights w1/w2/w3 to gate/down/up.
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_local_experts,
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
loaded_mtp_layers: set[int] = set()
for name, loaded_weight in weights:
mtp_layer = self._get_mtp_layer_idx_from_weight_name(name)
mapped_name = self._map_checkpoint_name(name)
if mapped_name is None:
# This weight does not belong to the MTP module, so skip it.
continue
name = mapped_name
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
# Routed experts (w1/w2/w3) are handled below. Don't let the
# stacked mapping rewrite them.
if ("block_sparse_moe.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for (
param_name,
weight_name,
expert_id,
expert_shard_id,
) in expert_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=expert_shard_id,
expert_id=expert_id,
)
break
else:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None or remapped_name not in params_dict:
continue
name = remapped_name
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
if mtp_layer is not None:
loaded_mtp_layers.add(mtp_layer)
# Validate that weights were loaded for each MTP layer.
for layer_idx in range(self.model.num_mtp_layers):
if layer_idx not in loaded_mtp_layers:
raise ValueError(
f"Failed to load MTP layer {layer_idx} weights from checkpoint."
)
return loaded_params
@@ -0,0 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AMD/ROCm fused Triton ops for MiniMax-M3.
These replace per-element PyTorch fallbacks (FlashInfer / fused HIP kernels are
unavailable on ROCm) with single-pass Triton kernels to cut launch overhead and
intermediate-tensor traffic during decode.
"""
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import (
gemma_fused_add_rmsnorm,
gemma_rmsnorm,
)
from vllm.models.minimax_m3.amd.ops.swiglu_oai import (
swiglu_oai_quantize_mxfp8,
swiglu_oai_split,
)
__all__ = [
"gemma_rmsnorm",
"gemma_fused_add_rmsnorm",
"swiglu_oai_split",
"swiglu_oai_quantize_mxfp8",
]
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused Gemma-style RMSNorm for AMD ROCm via Triton.
Gemma RMSNorm = normalize(x) * (1 + weight), computed in fp32. FlashInfer's
``gemma_rmsnorm`` / ``gemma_fused_add_rmsnorm`` CUDA kernels are unavailable on
ROCm, so the AMD path previously used a ~8-op PyTorch sequence (float cast, add,
pow, mean, rsqrt, two muls, cast) — each a separate kernel launch materializing
fp32 intermediates. These kernels collapse that into a single pass per row.
Two entry points:
* ``gemma_rmsnorm(x, w, eps)`` -> normalized tensor
* ``gemma_fused_add_rmsnorm(x, res, w, eps)`` -> (normalized, x + res)
Both normalize over the last dim and broadcast ``weight`` (shape [N]) over it,
so they serve both the full-hidden norms (input/post-attn/final) and the
per-head q_norm/k_norm (N == head_dim). Inputs may be non-contiguous views
(e.g. ``qkv.split`` slices); strides are passed through and outputs are written
contiguous.
"""
import torch
from vllm.triton_utils import tl, triton
@triton.jit
def _gemma_rmsnorm_kernel(
x_ptr,
w_ptr,
out_ptr,
n_cols,
stride_row,
stride_col,
eps,
BLOCK_N: tl.constexpr,
):
row = tl.program_id(0)
cols = tl.arange(0, BLOCK_N)
mask = cols < n_cols
x = tl.load(x_ptr + row * stride_row + cols * stride_col, mask=mask, other=0.0).to(
tl.float32
)
var = tl.sum(x * x, axis=0) / n_cols
rstd = 1.0 / tl.sqrt(var + eps)
w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32)
out = x * rstd * (1.0 + w)
tl.store(
out_ptr + row * n_cols + cols,
out.to(out_ptr.dtype.element_ty),
mask=mask,
)
@triton.jit
def _gemma_fused_add_rmsnorm_kernel(
x_ptr,
res_ptr,
w_ptr,
out_ptr,
res_out_ptr,
n_cols,
stride_xrow,
stride_xcol,
stride_rrow,
stride_rcol,
eps,
BLOCK_N: tl.constexpr,
):
row = tl.program_id(0)
cols = tl.arange(0, BLOCK_N)
mask = cols < n_cols
x = tl.load(
x_ptr + row * stride_xrow + cols * stride_xcol, mask=mask, other=0.0
).to(tl.float32)
r = tl.load(
res_ptr + row * stride_rrow + cols * stride_rcol, mask=mask, other=0.0
).to(tl.float32)
s = x + r
# residual_out is the pre-norm sum (consumed by the next layer's add).
tl.store(
res_out_ptr + row * n_cols + cols,
s.to(res_out_ptr.dtype.element_ty),
mask=mask,
)
var = tl.sum(s * s, axis=0) / n_cols
rstd = 1.0 / tl.sqrt(var + eps)
w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32)
out = s * rstd * (1.0 + w)
tl.store(
out_ptr + row * n_cols + cols,
out.to(out_ptr.dtype.element_ty),
mask=mask,
)
def _num_warps(block_n: int) -> int:
if block_n >= 4096:
return 16
if block_n >= 1024:
return 8
return 4
def gemma_rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
orig_shape = x.shape
n = orig_shape[-1]
x2 = x.reshape(-1, n)
m = x2.shape[0]
out = torch.empty((m, n), dtype=x.dtype, device=x.device)
block_n = triton.next_power_of_2(n)
_gemma_rmsnorm_kernel[(m,)](
x2,
weight,
out,
n,
x2.stride(0),
x2.stride(1),
eps,
BLOCK_N=block_n,
num_warps=_num_warps(block_n),
)
return out.reshape(orig_shape)
def gemma_fused_add_rmsnorm(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
orig_shape = x.shape
n = orig_shape[-1]
x2 = x.reshape(-1, n)
r2 = residual.reshape(-1, n)
m = x2.shape[0]
out = torch.empty((m, n), dtype=x.dtype, device=x.device)
res_out = torch.empty((m, n), dtype=x.dtype, device=x.device)
block_n = triton.next_power_of_2(n)
_gemma_fused_add_rmsnorm_kernel[(m,)](
x2,
r2,
weight,
out,
res_out,
n,
x2.stride(0),
x2.stride(1),
r2.stride(0),
r2.stride(1),
eps,
BLOCK_N=block_n,
num_warps=_num_warps(block_n),
)
return out.reshape(orig_shape), res_out.reshape(orig_shape)
@@ -0,0 +1,221 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused SwiGLU-OAI activation (split layout) for AMD ROCm via Triton.
SwiGLU-OAI on a ``[*, 2I]`` split-layout input (gate = first half, up = second
half):
gate = clamp(gate, max=limit)
up = clamp(up, -limit, +limit)
out = gate * sigmoid(alpha * gate) * (up + beta)
On ROCm the dense MLP and the native MXFP8 MoE (between its two GEMMs) fell back
to a chain of elementwise PyTorch ops with fp32 intermediates: vLLM's shared
``SiluAndMulWithClamp`` blanket-routes ROCm to ``forward_native``, and the MoE
applies the activation inline in PyTorch. This Triton kernel collapses that into
a single pass producing the ``[*, I]`` output directly, and computes in fp32
(rel ~1e-6 vs reference).
Note: the vectorized ``torch.ops._C.silu_and_mul_with_clamp`` op IS built on
ROCm and is ~1.2-2.2x faster in isolation, but the win is launch overhead that
HIP graphs already eliminate — measured end-to-end throughput is identical
(within noise), so we keep the fp32-accurate Triton kernel.
"""
import torch
from vllm.triton_utils import tl, triton
@triton.jit
def _swiglu_oai_kernel(
g_ptr,
out_ptr,
n_inter,
stride_gm,
stride_gn,
stride_om,
stride_on,
alpha,
beta,
limit,
HAS_LIMIT: tl.constexpr,
BLOCK_I: tl.constexpr,
):
row = tl.program_id(0)
pid_i = tl.program_id(1)
cols = pid_i * BLOCK_I + tl.arange(0, BLOCK_I)
mask = cols < n_inter
gate = tl.load(g_ptr + row * stride_gm + cols * stride_gn, mask=mask, other=0.0).to(
tl.float32
)
up = tl.load(
g_ptr + row * stride_gm + (n_inter + cols) * stride_gn,
mask=mask,
other=0.0,
).to(tl.float32)
if HAS_LIMIT:
gate = tl.minimum(gate, limit)
up = tl.minimum(tl.maximum(up, -limit), limit)
out = gate * tl.sigmoid(alpha * gate) * (up + beta)
tl.store(
out_ptr + row * stride_om + cols * stride_on,
out.to(out_ptr.dtype.element_ty),
mask=mask,
)
@triton.jit
def _swiglu_oai_quant_kernel(
g_ptr,
aq_ptr,
as_ptr,
M,
n_inter,
stride_gm,
stride_gn,
stride_qm,
stride_qn,
stride_sm,
stride_sk,
alpha,
beta,
limit,
HAS_LIMIT: tl.constexpr,
BLOCK_M: tl.constexpr,
):
"""SwiGLU-OAI (split layout) fused with per-32-block MXFP8 (E4M3 + E8M0)
quant. Each program handles ``[BLOCK_M, 32]`` of the ``[M, I]`` output (one
MX block): it reads the matching gate/up columns from ``g1`` (``[M, 2I]``),
computes the SwiGLU in fp32, then derives the block E8M0 scale and emits the
FP8 values + scale in a single pass — no bf16 ``act`` round-trip to HBM.
"""
pid_m = tl.program_id(0)
pid_b = tl.program_id(1) # which 32-element block along I
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_c = pid_b * 32 + tl.arange(0, 32)
m_mask = offs_m < M
gate = tl.load(
g_ptr + offs_m[:, None] * stride_gm + offs_c[None, :] * stride_gn,
mask=m_mask[:, None],
other=0.0,
).to(tl.float32)
up = tl.load(
g_ptr + offs_m[:, None] * stride_gm + (n_inter + offs_c)[None, :] * stride_gn,
mask=m_mask[:, None],
other=0.0,
).to(tl.float32)
if HAS_LIMIT:
gate = tl.minimum(gate, limit)
up = tl.minimum(tl.maximum(up, -limit), limit)
act = gate * tl.sigmoid(alpha * gate) * (up + beta) # [BLOCK_M, 32] fp32
amax = tl.maximum(tl.max(tl.abs(act), axis=1), 1e-30) # [BLOCK_M]
sb = tl.minimum(tl.maximum(tl.floor(tl.log2(amax)) + 127.0, 0.0), 254.0)
descale = tl.exp2(sb - 127.0)
aq = (act / descale[:, None]).to(aq_ptr.dtype.element_ty)
tl.store(
aq_ptr + offs_m[:, None] * stride_qm + offs_c[None, :] * stride_qn,
aq,
mask=m_mask[:, None],
)
tl.store(
as_ptr + offs_m * stride_sm + pid_b * stride_sk, sb.to(tl.uint8), mask=m_mask
)
def swiglu_oai_quantize_mxfp8(
gate_up: torch.Tensor,
alpha: float,
beta: float,
limit: float | None,
block_m: int = 64,
) -> tuple[torch.Tensor, torch.Tensor]:
"""SwiGLU-OAI on split-layout ``[M, 2I]`` fused with MXFP8 activation-quant.
Returns ``(act_q [M, I] float8_e4m3fn, act_scale [M, I//32] uint8 E8M0)``,
identical to ``mxfp8_e4m3_quantize(swiglu_oai_split(gate_up))`` but in a
single Triton pass (no bf16 intermediate). Used between the two GEMMs of the
native MXFP8 MoE. Numerically equivalent to the unfused chain (bit-exact on
measured MoE shapes); marginally more accurate (fp32 act, no bf16 round-trip).
"""
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
two_i = gate_up.shape[-1]
n_inter = two_i // 2
assert n_inter % MXFP8_BLOCK_SIZE == 0, (
f"fused swiglu+quant needs I % {MXFP8_BLOCK_SIZE} == 0, got I={n_inter}"
)
g1 = gate_up.reshape(-1, two_i).contiguous()
M = g1.shape[0]
aq = torch.empty((M, n_inter), dtype=MXFP8_VALUE_DTYPE, device=g1.device)
asc = torch.empty(
(M, n_inter // MXFP8_BLOCK_SIZE), dtype=MXFP8_SCALE_DTYPE, device=g1.device
)
grid = (triton.cdiv(M, block_m), n_inter // MXFP8_BLOCK_SIZE)
_swiglu_oai_quant_kernel[grid](
g1,
aq,
asc,
M,
n_inter,
g1.stride(0),
g1.stride(1),
aq.stride(0),
aq.stride(1),
asc.stride(0),
asc.stride(1),
float(alpha),
float(beta),
0.0 if limit is None else float(limit),
HAS_LIMIT=limit is not None,
BLOCK_M=block_m,
num_warps=4,
)
return aq, asc
def swiglu_oai_split(
gate_up: torch.Tensor,
alpha: float,
beta: float,
limit: float | None,
out_dtype: torch.dtype | None = None,
) -> torch.Tensor:
"""SwiGLU-OAI on a split-layout ``[*, 2I]`` tensor -> ``[*, I]``."""
orig_shape = gate_up.shape
two_i = orig_shape[-1]
n_inter = two_i // 2
x2 = gate_up.reshape(-1, two_i)
m = x2.shape[0]
dt = out_dtype if out_dtype is not None else gate_up.dtype
out = torch.empty((m, n_inter), dtype=dt, device=gate_up.device)
# Tile tuned on gfx950. The SwiGLU intermediate is sharded across tensor
# parallel ranks (per-rank n_inter = I / tp: dense I=12288, MoE I=3072), and
# a 512-wide tile (4 warps, ~2 elems/lane) only helps once the per-rank slice
# is large enough to be bandwidth-bound — at TP=1 prefill that is ~1.25-1.35x
# faster than 256. For small sharded slices (high TP) the kernel is launch-
# bound (~12us) and a wide tile can slightly regress, so fall back to 256.
# Decode is launch-bound at every TP. num_warps=8 underfills this tile, so it
# is pinned to 4.
block_i = 512 if n_inter >= 2048 else 256
grid = (m, triton.cdiv(n_inter, block_i))
_swiglu_oai_kernel[grid](
x2,
out,
n_inter,
x2.stride(0),
x2.stride(1),
out.stride(0),
out.stride(1),
float(alpha),
float(beta),
0.0 if limit is None else float(limit),
HAS_LIMIT=limit is not None,
BLOCK_I=block_i,
num_warps=4,
)
return out.reshape(*orig_shape[:-1], n_inter)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+512
View File
@@ -0,0 +1,512 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MiniMax M3 lightning indexer: side cache, metadata, and impl.
The indexer scores KV blocks with the index heads and selects the top-k blocks
(plus fixed init/local blocks) that the main block-sparse attention
(``sparse_attention.py``) then attends to. It owns its own side cache
(``MiniMaxM3IndexerCache``, one index-key vector per token), metadata, and
metadata builder, mirroring how DeepSeek V4 keeps the indexer separate from the
main attention.
``MiniMaxM3Indexer`` is the ``nn.Module`` the attention layer holds (like
``DeepseekV4Indexer``); it picks a kernel impl in ``__init__`` (via
``select_indexer_impl_cls``) and delegates ``forward`` to it.
"""
from dataclasses import dataclass
from typing import ClassVar
import torch
from torch import nn
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
from vllm.config.attention import IndexerKVDType
from vllm.config.cache import CacheDType
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.models.minimax_m3.common.ops.index_topk import (
minimax_m3_index_decode,
minimax_m3_index_score,
minimax_m3_index_topk,
)
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionMetadata,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheSpec,
MLAAttentionSpec,
)
class MiniMaxM3IndexerBackend(AttentionBackend):
"""Indexer side-cache backend (key-only)."""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16, torch.float16]
# bf16 today; mirrors the main backend to keep spec validation permissive.
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"bfloat16",
"fp8",
"fp8_e4m3",
"fp8_e5m2",
]
@staticmethod
def get_name() -> str:
return "MINIMAX_M3_SPARSE_INDEXER"
@staticmethod
def get_impl_cls() -> type["MiniMaxM3IndexerImpl"]:
# Concrete impl chosen by select_indexer_impl_cls; base for introspection.
return MiniMaxM3IndexerImpl
@staticmethod
def get_builder_cls() -> type["MiniMaxM3IndexerMetadataBuilder"]:
return MiniMaxM3IndexerTritonMetadataBuilder
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
return [128]
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [128]
@classmethod
def is_sparse(cls) -> bool:
return True
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, block_size, head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
# M3 does not use cross-layer (per-layer-stacked) KV blocks.
raise NotImplementedError
return (0, 1, 2)
class MiniMaxM3IndexerCache(nn.Module, AttentionLayerBase):
"""Side KV cache for the indexer's per-token index keys (key-only).
Registers itself in the static forward context so the KV-cache manager
allocates it (like ``DeepseekV32IndexerCache``).
"""
def __init__(
self,
head_dim: int,
prefix: str,
cache_config: CacheConfig | None = None,
indexer_kv_dtype: IndexerKVDType = "bf16",
backend_cls: type[AttentionBackend] = MiniMaxM3IndexerBackend,
) -> None:
super().__init__()
if indexer_kv_dtype != "bf16":
raise NotImplementedError(
f"indexer_kv_dtype={indexer_kv_dtype!r} is not supported yet "
"for the MiniMax M3 indexer cache (only 'bf16')."
)
self.kv_cache = torch.tensor([])
self.head_dim = head_dim
self.indexer_kv_dtype = indexer_kv_dtype
# Storage dtype for the side cache (bf16 today; quantized layouts later).
self.dtype = torch.bfloat16
self.prefix = prefix
self.cache_config = cache_config
# Impl-chosen backend -> each impl gets its own builder (get_attn_backend).
self.backend_cls = backend_cls
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
# Key-only: MLAAttentionSpec budgets one vector/token (not 2x for K+V).
return MLAAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=1,
head_size=self.head_dim,
dtype=self.dtype,
)
def forward(self) -> None: ...
def get_attn_backend(self) -> type[AttentionBackend]:
return self.backend_cls
@dataclass
class MiniMaxM3IndexerPrefillMetadata:
"""Per-prefill index-scoring state."""
cu_seqlens_q: torch.Tensor # [num_prefills + 1] int32, rebased to 0
seq_lens: torch.Tensor # [num_prefills] int32, total KV lengths
context_lens: torch.Tensor # [num_prefills] int32 (cached/context tokens)
block_table: torch.Tensor
max_query_len: int
max_seq_len: int
@dataclass
class MiniMaxM3IndexerDecodeMetadata:
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
per-request query length (1, or 1 + num_speculative_tokens)."""
seq_lens: torch.Tensor # [num_decodes] int32
block_table: torch.Tensor
max_seq_len: int
decode_query_len: int
@dataclass
class MiniMaxM3IndexerMetadata(AttentionMetadata):
"""Indexer metadata, split into prefill and decode sub-metadata."""
seq_lens: torch.Tensor
max_seq_len: int
slot_mapping: torch.Tensor
num_actual_tokens: int # total query tokens (decode-first batch)
# Split counts; identical to the main metadata's (same reorder threshold).
num_decodes: int
num_decode_tokens: int
num_prefills: int
num_prefill_tokens: int
prefill: MiniMaxM3IndexerPrefillMetadata | None = None
decode: MiniMaxM3IndexerDecodeMetadata | None = None
class MiniMaxM3IndexerMetadataBuilder(
AttentionMetadataBuilder[MiniMaxM3IndexerMetadata]
):
"""Abstract base: shared setup only. The Triton and MSA builders are
parallel subclasses that each own their full ``build`` (no shared code)."""
# Full cudagraphs for uniform decode batches (incl. spec-decode verify).
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
# Raised to 1 + num_speculative_tokens by _init_reorder_batch_threshold when
# spec decode is on; matches the main builder so the splits agree.
reorder_batch_threshold: int = 1
def __init__(
self,
kv_cache_spec: AttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
hf_config = vllm_config.model_config.hf_config
text_config = getattr(hf_config, "text_config", hf_config)
sparse_cfg = text_config.sparse_attention_config
# Index-query head count from model config (cache spec has 1 vec/token).
total_index_heads = sparse_cfg["sparse_num_index_heads"]
tp_size = get_tensor_model_parallel_world_size()
if total_index_heads >= tp_size:
assert total_index_heads % tp_size == 0
else:
assert tp_size % total_index_heads == 0
self.num_index_heads = max(1, total_index_heads // tp_size)
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
# Stable context-length buffer for decode cudagraph replays.
self.context_len_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
dtype=torch.int32,
device=device,
)
class MiniMaxM3IndexerTritonMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
"""Triton indexer metadata: no SM100 fmha_sm100 plan."""
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> MiniMaxM3IndexerMetadata:
num_reqs = common_attn_metadata.num_reqs
num_tokens = common_attn_metadata.num_actual_tokens
query_start_loc = common_attn_metadata.query_start_loc
seq_lens = common_attn_metadata.seq_lens
block_table = common_attn_metadata.block_table_tensor
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
require_uniform=True,
)
)
assert num_decodes + num_prefills == num_reqs
assert num_decode_tokens + num_prefill_tokens == num_tokens
# Decode-first batch: context lengths into the stable cudagraph buffer.
context_lens = self.context_len_buffer[:num_reqs]
context_lens.copy_(
common_attn_metadata.compute_num_computed_tokens(), non_blocking=True
)
prefill_metadata: MiniMaxM3IndexerPrefillMetadata | None = None
if num_prefills > 0:
prefill_metadata = MiniMaxM3IndexerPrefillMetadata(
cu_seqlens_q=(query_start_loc[num_decodes:] - num_decode_tokens).to(
torch.int32
),
seq_lens=seq_lens[num_decodes:],
context_lens=context_lens[num_decodes:],
block_table=block_table[num_decodes:],
max_query_len=common_attn_metadata.max_query_len,
max_seq_len=common_attn_metadata.max_seq_len,
)
decode_metadata: MiniMaxM3IndexerDecodeMetadata | None = None
if num_decodes > 0:
qsl_cpu = common_attn_metadata.query_start_loc_cpu
query_lens_cpu = qsl_cpu[1 : num_decodes + 1] - qsl_cpu[:num_decodes]
decode_query_len = int(query_lens_cpu[0].item())
assert decode_query_len > 0
assert torch.all(
(query_lens_cpu == decode_query_len) | (query_lens_cpu == 0)
)
assert num_decode_tokens == num_decodes * decode_query_len
decode_metadata = MiniMaxM3IndexerDecodeMetadata(
seq_lens=seq_lens[:num_decodes],
block_table=block_table[:num_decodes],
max_seq_len=common_attn_metadata.max_seq_len,
decode_query_len=decode_query_len,
)
return MiniMaxM3IndexerMetadata(
seq_lens=seq_lens,
max_seq_len=common_attn_metadata.max_seq_len,
slot_mapping=common_attn_metadata.slot_mapping,
num_actual_tokens=num_tokens,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
num_prefills=num_prefills,
num_prefill_tokens=num_prefill_tokens,
prefill=prefill_metadata,
decode=decode_metadata,
)
class MiniMaxM3IndexerImpl(nn.Module):
"""Abstract base for the indexer kernel impls.
Each impl owns its side cache and reports its backend via
``indexer_backend_cls`` (so each gets its own builder). The Triton and MSA
subclasses each own a full ``forward`` returning ``(decode_topk,
prefill_topk)`` -- no shared forward code.
"""
# Set by each impl so the side cache reports the matching backend + builder.
indexer_backend_cls: ClassVar[type[AttentionBackend]] = MiniMaxM3IndexerBackend
def __init__(
self,
*,
num_kv_heads: int,
scale: float,
topk_blocks: int,
sparse_block_size: int,
num_index_heads: int,
index_head_dim: int,
prefix: str,
init_blocks: int = 0,
local_blocks: int = 0,
score_type: str = "max",
cache_config: CacheConfig | None = None,
indexer_kv_dtype: IndexerKVDType = "bf16",
) -> None:
super().__init__()
self.num_kv_heads = num_kv_heads
self.scale = scale
self.topk_blocks = topk_blocks
self.block_size = sparse_block_size
self.init_blocks = init_blocks
self.local_blocks = local_blocks
self.score_type = score_type
self.num_index_heads = num_index_heads
self.index_head_dim = index_head_dim
self.indexer_kv_dtype = indexer_kv_dtype
# Owns the side cache (registers itself in the static forward context).
self.index_cache = MiniMaxM3IndexerCache(
head_dim=index_head_dim,
prefix=f"{prefix}.index_cache",
cache_config=cache_config,
indexer_kv_dtype=indexer_kv_dtype,
backend_cls=type(self).indexer_backend_cls,
)
def forward(
self,
index_query: torch.Tensor,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
"""Return ``(decode_topk, prefill_topk)``; implemented per kernel impl."""
raise NotImplementedError
class MiniMaxM3IndexerTritonImpl(MiniMaxM3IndexerImpl):
"""Triton indexer score + top-k for both prefill and decode."""
def forward(
self,
index_query: torch.Tensor,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
return None, None # profiling run; caches unbound
index_md = attn_metadata[self.index_cache.prefix]
assert isinstance(index_md, MiniMaxM3IndexerMetadata)
num_tokens = index_md.num_actual_tokens
nd = index_md.num_decode_tokens
iq = index_query[:num_tokens].view(
-1, self.num_index_heads, self.index_head_dim
)
kv = self.index_cache.kv_cache
decode_topk: torch.Tensor | None = None
prefill_topk: torch.Tensor | None = None
if index_md.num_decodes > 0:
d = index_md.decode
assert d is not None
decode_topk = minimax_m3_index_decode(
iq[:nd],
kv,
d.block_table,
d.seq_lens,
d.max_seq_len,
self.topk_blocks,
self.init_blocks,
self.local_blocks,
self.num_kv_heads,
self.scale,
d.decode_query_len,
)
if index_md.num_prefills > 0:
p = index_md.prefill
assert p is not None
score = minimax_m3_index_score(
iq[nd:],
kv,
p.block_table,
p.cu_seqlens_q,
p.seq_lens,
p.context_lens,
p.max_query_len,
p.max_seq_len,
self.num_kv_heads,
self.scale,
)
prefill_topk = minimax_m3_index_topk(
score,
p.cu_seqlens_q,
p.context_lens,
p.max_query_len,
self.topk_blocks,
self.init_blocks,
self.local_blocks,
)
return decode_topk, prefill_topk
def select_indexer_impl_cls(
*,
indexer_kv_dtype: IndexerKVDType = "bf16",
) -> type[MiniMaxM3IndexerImpl]:
"""Pick the indexer impl off the index-cache dtype.
The SM100 MSA indexer score path is disabled for now; use the local Triton
indexer. If re-enabled, add a NVIDIA-specific ``MiniMaxM3IndexerImpl`` here.
"""
if indexer_kv_dtype in ("mxfp4", "nvfp4"):
raise NotImplementedError(
f"indexer_kv_dtype={indexer_kv_dtype!r} needs the (not-yet-added) "
"CuteDSL indexer impl."
)
if indexer_kv_dtype != "bf16":
raise NotImplementedError(
f"indexer_kv_dtype={indexer_kv_dtype!r} is not supported by the "
"Triton indexer impl."
)
return MiniMaxM3IndexerTritonImpl
class MiniMaxM3Indexer(nn.Module):
"""Indexer module held by the attention layer (like ``DeepseekV4Indexer``).
Picks the kernel impl in ``__init__`` (``select_indexer_impl_cls``) and
delegates ``forward``; exposes the impl's side cache via ``index_cache``.
"""
def __init__(
self,
*,
num_kv_heads: int,
scale: float,
topk_blocks: int,
sparse_block_size: int,
num_index_heads: int,
index_head_dim: int,
prefix: str,
init_blocks: int = 0,
local_blocks: int = 0,
score_type: str = "max",
cache_config: CacheConfig | None = None,
indexer_kv_dtype: IndexerKVDType = "bf16",
) -> None:
super().__init__()
impl_cls = select_indexer_impl_cls(
indexer_kv_dtype=indexer_kv_dtype,
)
self.impl = impl_cls(
num_kv_heads=num_kv_heads,
scale=scale,
topk_blocks=topk_blocks,
sparse_block_size=sparse_block_size,
num_index_heads=num_index_heads,
index_head_dim=index_head_dim,
prefix=prefix,
init_blocks=init_blocks,
local_blocks=local_blocks,
score_type=score_type,
cache_config=cache_config,
indexer_kv_dtype=indexer_kv_dtype,
)
@property
def index_cache(self) -> MiniMaxM3IndexerCache:
return self.impl.index_cache
@property
def num_index_heads(self) -> int:
return self.impl.num_index_heads
def forward(
self,
index_query: torch.Tensor,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
return self.impl(index_query)
@@ -0,0 +1,514 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Mapping, Sequence
from typing import cast
import torch
from transformers import BatchFeature
from transformers.video_utils import VideoMetadata
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
ImageSize,
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.video import (
VIDEO_LOADER_REGISTRY,
VideoBackend,
VideoSourceMetadata,
VideoTargetMetadata,
)
from vllm.transformers_utils.configs.minimax_m3 import MiniMaxM3Config
from vllm.transformers_utils.processors.minimax_m3 import (
MIN_SHORT_SIDE_PIXEL,
MiniMaxM3VLImageProcessor,
MiniMaxM3VLVideoProcessor,
MiniMaxVLProcessor,
smart_resize,
)
# Upper bound on the number of frames used to build the dummy video during
# memory profiling. Sized to the worst-case video the processor accepts:
# ``max_total_pixels // max_pixels_per_frame`` = 301,056,000 // 602,112 = 500
# frames, each at the video processor's per-frame ``max_pixels`` (768 * 28 * 28
# = 602,112). This reaches the true worst-case ~192,000 vision tokens, but only
# because the dummy video is sized via ``get_video_size_with_most_features()``
# (the video ``max_pixels`` bound), not the smaller image bound. Without a cap,
# ``_get_max_video_frames(seq_len)`` with M3's large ``max_model_len`` yields
# ~1400 frames, producing a multi-GB dummy tensor that overflows the
# multimodal encoder cache.
_MAX_FRAMES_PER_VIDEO = 500
class MiniMaxM3VLProcessingInfo(BaseProcessingInfo):
IMAGE_TOKEN = "]<]image[>["
VIDEO_TOKEN = "]<]video[>["
VISION_START_TOKEN = "]<]start of image[>["
VISION_END_TOKEN = "]<]end of image[>["
def get_hf_config(self) -> MiniMaxM3Config:
return self.ctx.get_hf_config(MiniMaxM3Config)
def get_hf_processor(self, **kwargs: object) -> MiniMaxVLProcessor:
# The released checkpoint only ships the processor as remote code
# (via ``auto_map``). Construct the vendored processor directly so the
# model loads without ``--trust-remote-code``.
return self.ctx.get_hf_processor(MiniMaxVLProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None}
def get_mm_max_tokens_per_item(
self,
seq_len: int,
mm_counts: Mapping[str, int],
) -> Mapping[str, int]:
return {
"image": self.get_max_image_tokens(),
"video": self.get_max_video_tokens(seq_len, mm_counts),
}
def get_image_processor(self, **kwargs: object) -> MiniMaxM3VLImageProcessor:
return self.get_hf_processor(**kwargs).image_processor
def get_video_processor(self, **kwargs: object) -> MiniMaxM3VLVideoProcessor:
return self.get_hf_processor(**kwargs).video_processor
def _get_vision_info(
self,
*,
image_width: int,
image_height: int,
num_frames: int,
image_processor,
) -> tuple[ImageSize, int]:
"""Compute resized image size and number of vision tokens.
Mirrors the processor's Qwen-style ``smart_resize`` (area bound by
``max_pixels``) so token counts match the actual processor output.
"""
patch_size: int = image_processor.patch_size
merge_size: int = image_processor.merge_size
temporal_patch_size: int = image_processor.temporal_patch_size
factor = patch_size * merge_size
max_pixels: int = image_processor.max_pixels
# Long-side resize spec (opt-in). ``image_processor`` is the *video*
# processor when counting video tokens, so read the bounds off it.
max_long_side_pixel = getattr(image_processor, "max_long_side_pixel", None)
min_short_side_pixel = getattr(
image_processor, "min_short_side_pixel", MIN_SHORT_SIDE_PIXEL
)
new_h, new_w = smart_resize(
image_height,
image_width,
factor=factor,
max_pixels=max_pixels,
max_long_side_pixel=max_long_side_pixel,
min_short_side_pixel=min_short_side_pixel,
# Token counting must not raise; the volumetric/area cap is enforced
# in the processor's _preprocess on the real inputs.
max_total_pixels=None,
)
grid_h = new_h // patch_size
grid_w = new_w // patch_size
# Pad frames to be divisible by temporal_patch_size
padded_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(padded_frames // temporal_patch_size, 1)
num_tokens = grid_t * grid_h * grid_w // (merge_size**2)
return ImageSize(width=new_w, height=new_h), num_tokens
def get_num_image_tokens(
self,
*,
image_width: int,
image_height: int,
image_processor,
mm_kwargs: Mapping[str, object],
) -> int:
_, n = self._get_vision_info(
image_width=image_width,
image_height=image_height,
num_frames=1,
image_processor=image_processor,
)
return n
def get_num_video_tokens(
self,
*,
image_width: int,
image_height: int,
num_frames: int,
image_processor,
mm_kwargs: Mapping[str, object],
) -> int:
_, n = self._get_vision_info(
image_width=image_width,
image_height=image_height,
num_frames=num_frames,
image_processor=image_processor,
)
return n
def get_image_size_with_most_features(self) -> ImageSize:
# Largest square (a multiple of patch_size*merge_size) whose area is
# within the image processor's bound — this yields the most vision
# tokens for one image. With the long-side spec the square side is
# capped by ``max_long_side_pixel`` (and the fixed ``max_total_pixels``);
# otherwise it is bound by the ``max_pixels`` area.
image_processor = self.get_image_processor()
factor = image_processor.patch_size * image_processor.merge_size
max_long_side_pixel = getattr(image_processor, "max_long_side_pixel", None)
if max_long_side_pixel is not None:
side_px = min(
max_long_side_pixel,
math.isqrt(image_processor.max_total_pixels),
)
else:
side_px = math.isqrt(image_processor.max_pixels)
side = max(factor, (side_px // factor) * factor)
return ImageSize(width=side, height=side)
def get_video_size_with_most_features(self) -> ImageSize:
# Per-frame size that yields the most vision tokens, bound by the
# *video* processor's ``max_pixels`` (which differs from the image
# bound). Token count depends only on area, so maximize the area
# achievable with both sides a multiple of patch_size*merge_size rather
# than picking the largest square — a square (e.g. 756x756 for M3's
# 602,112 bound) leaves area on the table, undercounting frames.
video_processor = self.get_video_processor()
factor = video_processor.patch_size * video_processor.merge_size
per_frame_pixels = video_processor.max_pixels
max_long_side_pixel = getattr(video_processor, "max_long_side_pixel", None)
if max_long_side_pixel is not None:
# Long-side spec: a frame's worst case is a square capped by
# ``max_long_side_pixel`` (per-frame area, not the volumetric cap).
per_frame_pixels = min(per_frame_pixels, max_long_side_pixel**2)
units = per_frame_pixels // (factor * factor) # h_u * w_u
h_u = math.isqrt(units)
while units % h_u:
h_u -= 1
return ImageSize(width=(units // h_u) * factor, height=h_u * factor)
def get_max_image_tokens(self) -> int:
image_processor = self.get_image_processor()
size = self.get_image_size_with_most_features()
return self.get_num_image_tokens(
image_width=size.width,
image_height=size.height,
image_processor=image_processor,
mm_kwargs={},
)
def _get_max_video_frames(self, max_tokens: int) -> int:
video_processor = self.get_video_processor()
size = self.get_video_size_with_most_features()
num_frames = 1
while True:
next_n = self.get_num_video_tokens(
image_width=size.width,
image_height=size.height,
num_frames=num_frames + 1,
image_processor=video_processor,
mm_kwargs={},
)
if next_n > max_tokens:
break
num_frames += 1
return num_frames
def get_num_frames_with_most_features(
self,
seq_len: int,
mm_counts: Mapping[str, int],
max_frames_per_video: int = _MAX_FRAMES_PER_VIDEO,
) -> int:
max_videos = mm_counts.get("video", 0)
max_total_frames = self._get_max_video_frames(seq_len)
max_frames_per_video = min(
max_total_frames // max(max_videos, 1), max_frames_per_video
)
return max(max_frames_per_video, 1)
def get_max_video_tokens(
self,
seq_len: int,
mm_counts: Mapping[str, int],
) -> int:
video_processor = self.get_video_processor()
size = self.get_video_size_with_most_features()
return self.get_num_video_tokens(
image_width=size.width,
image_height=size.height,
num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
image_processor=video_processor,
mm_kwargs={},
)
class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
image_token: str = self.info.IMAGE_TOKEN
video_token: str = self.info.VIDEO_TOKEN
return image_token * num_images + video_token * num_videos
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
) -> MultiModalDataDict:
size = self.info.get_image_size_with_most_features()
video_size = self.info.get_video_size_with_most_features()
num_frames = self.info.get_num_frames_with_most_features(seq_len, mm_counts)
return {
"image": self._get_dummy_images(
width=size.width,
height=size.height,
num_images=mm_counts.get("image", 0),
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
),
"video": self._get_dummy_videos(
width=video_size.width,
height=video_size.height,
num_frames=num_frames,
num_videos=mm_counts.get("video", 0),
overrides=cast(VideoDummyOptions | None, mm_options.get("video")),
),
}
class MiniMaxM3VLMultiModalProcessor(
BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]
):
def _get_data_parser(self) -> MultiModalDataParser:
# Request video metadata (fps + sampled frame indices) so the HF
# processor can emit per-frame ``]<]X.X seconds[>[`` timestamp markers,
# matching MiniMax's reference video token stream. ``_get_prompt_updates``
# reconstructs the same markers from the metadata to keep the prompt
# replacement aligned with the processor output.
return MultiModalDataParser(video_needs_metadata=True)
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
# With ``video_needs_metadata=True`` each video arrives as a
# ``(frames, metadata)`` tuple. Split the frames back out and forward the
# metadata as ``VideoMetadata`` so the processor emits timestamps.
videos = cast(list | None, mm_data.get("videos"))
video_metadata: list[VideoMetadata] | None = None
if videos:
frames_only = []
video_metadata = []
for item in videos:
if isinstance(item, tuple) and len(item) == 2:
frames, meta = item
else:
frames, meta = item, {}
frames_only.append(frames)
meta = {
k: v for k, v in (meta or {}).items() if k != "do_sample_frames"
}
# VideoMetadata requires total_num_frames; derive it for
# dummy/profiling videos whose metadata omits it. fps and
# frames_indices default to None there → no timestamps, which
# stays consistent with _get_prompt_updates.
meta.setdefault("total_num_frames", len(frames))
video_metadata.append(VideoMetadata(**meta))
mm_data["videos"] = frames_only
# Override the video processor's default do_resize=False (set for a
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
merged = dict(do_resize=True, **mm_kwargs, **tok_kwargs)
data = dict(text=prompt, **mm_data)
if video_metadata is not None:
data["video_metadata"] = video_metadata
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
data,
merged,
)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
image_grid_thw = hf_inputs.get("image_grid_thw")
video_grid_thw = hf_inputs.get("video_grid_thw")
# Total patches per item (grid_t * grid_h * grid_w)
image_grid_sizes = (
image_grid_thw.prod(-1)
if image_grid_thw is not None
else torch.empty(0, dtype=torch.long)
)
video_grid_sizes = (
video_grid_thw.prod(-1)
if video_grid_thw is not None
else torch.empty(0, dtype=torch.long)
)
return {
"pixel_values": MultiModalFieldConfig.flat_from_sizes(
"image", image_grid_sizes
),
"image_grid_thw": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
"pixel_values_videos": MultiModalFieldConfig.flat_from_sizes(
"video", video_grid_sizes
),
"video_grid_thw": MultiModalFieldConfig.batched("video", keep_on_cpu=True),
}
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
image_token_id: int = vocab[self.info.IMAGE_TOKEN]
video_token_id: int = vocab[self.info.VIDEO_TOKEN]
start_token_id: int = vocab[self.info.VISION_START_TOKEN]
end_token_id: int = vocab[self.info.VISION_END_TOKEN]
merge_length: int = hf_processor.image_processor.merge_size**2
def get_image_replacement(item_idx: int):
grid_thw: torch.Tensor = out_mm_kwargs["image"][item_idx][
"image_grid_thw"
].data
# grid_thw shape: (3,) = [1, grid_h, grid_w]
N = int(grid_thw.prod().item()) // merge_length
full = [start_token_id] + [image_token_id] * N + [end_token_id]
return PromptUpdateDetails.select_token_id(full, image_token_id)
# Per-video metadata (fps + sampled frame indices) is carried on the
# parsed video items; used to reproduce the HF processor's timestamps.
video_items = mm_items.get("video")
video_metadata = getattr(video_items, "metadata", None)
temporal_patch_size: int = hf_processor.video_processor.temporal_patch_size
def get_video_replacement(item_idx: int):
grid_thw: torch.Tensor = out_mm_kwargs["video"][item_idx][
"video_grid_thw"
].data
# grid_thw shape: (3,) = [grid_t, grid_h, grid_w]
# HF model uses VIDEO_TOKEN (not IMAGE_TOKEN) for video frame content:
# processing_minimax.py L245: replace(placeholder, self.VIDEO_TOKEN)
T = int(grid_thw[0].item())
M = int(grid_thw[1].item() * grid_thw[2].item()) // merge_length
# Reproduce the HF processor's per-frame timestamp markers
# (processing_minimax.py: ts = frames_indices[frame*tps] / fps,
# rendered as "]<]X.X seconds[>["). Falls back to no timestamps when
# metadata is unavailable (keeping the replacement aligned with the
# processor output in both cases).
meta = (
video_metadata[item_idx]
if video_metadata is not None and item_idx < len(video_metadata)
else None
)
fps = meta.get("fps") if meta else None
frames_indices = meta.get("frames_indices") if meta else None
full: list[int] = []
for frame_idx in range(T):
if fps is not None and frames_indices is not None:
idx = min(frame_idx * temporal_patch_size, len(frames_indices) - 1)
ts = frames_indices[idx] / fps
full += tokenizer.encode(
f"]<]{ts:.1f} seconds[>[", add_special_tokens=False
)
full += [start_token_id] + [video_token_id] * M + [end_token_id]
return PromptUpdateDetails.select_token_id(full, video_token_id)
return [
PromptReplacement(
modality="image",
target=[image_token_id],
replacement=get_image_replacement,
),
PromptReplacement(
modality="video",
target=[video_token_id],
replacement=get_video_replacement,
),
]
# TODO(Isotr0py): Tie with MinimaxVideoProcessor
# after https://github.com/vllm-project/vllm/pull/44126
@VIDEO_LOADER_REGISTRY.register("minimax_m3_vl")
class MiniMaxM3VideoBackend(VideoBackend):
@classmethod
def compute_frames_index_to_sample(
cls,
source: VideoSourceMetadata,
target: VideoTargetMetadata,
**kwargs,
) -> list[int]:
total_frames = source.total_frames_num
video_fps = source.original_fps
fps = target.fps
if total_frames <= 0 or video_fps <= 0 or fps <= 0:
return [0] if total_frames > 0 else []
read_time_interval = 1.0 / fps
eps = 1e-4
indices: list[int] = []
prev_kept_ts = -float("inf")
while True:
if not indices:
target_frame = 0
else:
target_ts = prev_kept_ts + read_time_interval - eps
target_frame = math.ceil(target_ts * video_fps)
target_frame = max(target_frame, indices[-1] + 1)
if target_frame >= total_frames:
break
indices.append(target_frame)
prev_kept_ts = target_frame / video_fps
last_frame_idx = total_frames - 1
last_ts = last_frame_idx / video_fps
if indices and indices[-1] != last_frame_idx and last_ts - prev_kept_ts > eps:
indices.append(last_frame_idx)
if not indices:
indices = [0]
return indices
@@ -0,0 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cross-platform (Triton) kernels for MiniMax M3 sparse attention."""
from .index_topk import (
minimax_m3_index_decode,
minimax_m3_index_score,
minimax_m3_index_topk,
)
from .sparse_attn import minimax_m3_sparse_attn, minimax_m3_sparse_attn_decode
__all__ = [
"minimax_m3_index_decode",
"minimax_m3_index_score",
"minimax_m3_index_topk",
"minimax_m3_sparse_attn",
"minimax_m3_sparse_attn_decode",
]
@@ -0,0 +1,898 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton kernels for MiniMax M3 lightning-indexer block scoring + top-k.
Index queries score each 128-token block of index keys (max over the block),
then the top-k blocks (plus forced init/local blocks) are selected per query
token. Adapted to vLLM's paged KV cache: the KV page size is forced to equal the
sparse block size (128), so one sparse block maps to exactly one page.
Index-K cache layout (vLLM): ``(num_blocks, 128, idx_head_dim)`` (single head).
Only the paths MiniMax M3 uses are implemented: score_type="max", index value
disabled (score-only indexer), single shared index head. The selected block ids
feed the block-sparse attention kernels in ``sparse_attn``.
"""
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import round_up
# One sparse block == one KV page.
SPARSE_BLOCK_SIZE = 128
# ---------------------------------------------------------------------------
# Bitonic top-k helpers (layout-agnostic).
# ---------------------------------------------------------------------------
@triton.jit
def _compare_and_swap(x, ids, flip, i: tl.constexpr, n_dims: tl.constexpr):
n_outer: tl.constexpr = x.numel >> n_dims
shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)]
y = tl.reshape(x, shape)
mask = tl.arange(0, 2)[None, :, None]
left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype)
right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype)
left = tl.reshape(left, x.shape)
right = tl.reshape(right, x.shape)
y_idx = tl.reshape(ids, shape)
left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape)
right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape)
left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype)
right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype)
idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True)
ileft = left.to(idtype, bitcast=True)
iright = right.to(idtype, bitcast=True)
ix = x.to(idtype, bitcast=True)
cond = (left > right) != flip
ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix))
new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids))
return ret.to(x.dtype, bitcast=True), new_ids
@triton.jit
def _bitonic_merge(
x, ids, stage: tl.constexpr, order: tl.constexpr, n_dims: tl.constexpr
):
n_outer: tl.constexpr = x.numel >> n_dims
tl.static_assert(stage <= n_dims)
if order == 2:
shape: tl.constexpr = [n_outer * 2 ** (n_dims - 1 - stage), 2, 2**stage]
flip = tl.reshape(
tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape
)
else:
flip = order
for i in tl.static_range(stage):
x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims)
return x, ids
# ---------------------------------------------------------------------------
# Index block-score kernel (paged). score[h, token, block] = max over the
# 128-token block of (idx_q . index_k), causal-masked. BLOCK_SIZE_K == 128 so
# each K-tile is exactly one page (BLOCKS_PER_K_BLOCK == 1).
# ---------------------------------------------------------------------------
# since prefill metadata is sliced from mixed batch metadata, seq_lens and prefix_lens
# might lose pointer alignment, which trigger Triton recompiles. we don't actually
# need pointer alignment for those tensors anyway because we do scalar load.
@triton.jit(do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])
def _index_block_score_kernel(
q_ptr, # idx_q: [total_q, num_idx_heads, head_dim]
ik_cache_ptr, # index-K cache: [num_blocks, 128, head_dim]
score_ptr, # [num_idx_heads, total_q, max_block]
block_table_ptr, # [num_reqs, max_blocks]
cu_seqlens, # [batch+1] query start offsets
seq_lens, # [batch] total K length
prefix_lens, # [batch] context length before this chunk's queries
num_idx_heads,
head_dim: tl.constexpr,
sm_scale,
stride_q_n,
stride_q_h,
stride_q_d,
stride_ik_blk,
stride_ik_pos,
stride_ik_d,
stride_s_h,
stride_s_n,
stride_s_k,
stride_bt_b,
BLOCK_SIZE_Q: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
):
sm_scale_log2e = sm_scale * 1.4426950409
pid_q = tl.program_id(0)
pid_bh = tl.program_id(1)
pid_b = pid_bh // num_idx_heads
pid_h = pid_bh % num_idx_heads
seq_start = tl.load(cu_seqlens + pid_b)
q_len = tl.load(cu_seqlens + pid_b + 1) - seq_start
seq_len = tl.load(seq_lens + pid_b)
prefix_len = tl.load(prefix_lens + pid_b)
if BLOCK_SIZE_Q * pid_q >= q_len:
return
q_ptrs = tl.make_block_ptr(
base=q_ptr + seq_start * stride_q_n + pid_h * stride_q_h,
shape=(q_len, head_dim),
strides=(stride_q_n, stride_q_d),
offsets=(pid_q * BLOCK_SIZE_Q, 0),
block_shape=(BLOCK_SIZE_Q, head_dim),
order=(1, 0),
)
q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")
q_start = prefix_len + pid_q * BLOCK_SIZE_Q
off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len
off_k = tl.arange(0, BLOCK_SIZE_K)
off_d = tl.arange(0, head_dim)
# Block table row for this request.
bt_row = block_table_ptr + pid_b * stride_bt_b
# Causal window: only blocks up to the last query token's position.
hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q)
for i in tl.range(0, hi, BLOCK_SIZE_K):
blk = i // BLOCK_SIZE_K
page = tl.load(bt_row + blk).to(tl.int64)
pos = i + off_k
# index-K for this page: [BLOCK_SIZE_D, BLOCK_SIZE_K] (transposed)
# we don't need masked load for K, because KV cache ensures
# allocation is multiple of BLOCK_SIZE_K.
# for tokens beyond seqlen, they will be masked in qk later.
k = tl.load(
ik_cache_ptr
+ page * stride_ik_blk
+ off_k[None, :] * stride_ik_pos
+ off_d[:, None] * stride_ik_d,
)
qk = tl.dot(q, k) * sm_scale_log2e
# apply causal mask as needed
if q_start < i + BLOCK_SIZE_K:
qk = tl.where(off_q[:, None] >= pos[None, :], qk, float("-inf"))
# one sparse block per K-tile -> max over the 128 positions
score = tl.max(qk, axis=1) # [BLOCK_SIZE_Q]
s_ptrs = (
score_ptr
+ pid_h * stride_s_h
+ (seq_start + pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q))
* stride_s_n
+ blk * stride_s_k
)
q_store_mask = (pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q)) < q_len
tl.store(s_ptrs, score, mask=q_store_mask)
# ---------------------------------------------------------------------------
# Top-k selection over per-token block scores (layout-agnostic). block_size_q
# is 1 for M3, so top-k is computed per query token.
# ---------------------------------------------------------------------------
# since prefill metadata is sliced from mixed batch metadata, prefix_lens
# might lose pointer alignment, which trigger Triton recompiles. we don't actually
# need pointer alignment for those tensors anyway because we do scalar load.
@triton.heuristics({"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"])})
@triton.autotune(
configs=[
triton.Config({"BLOCK_SIZE_K": 2048}, num_warps=8, num_stages=2),
triton.Config({"BLOCK_SIZE_K": 1024}, num_warps=8, num_stages=2),
triton.Config({"BLOCK_SIZE_K": 512}, num_warps=8, num_stages=2),
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),
triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),
],
key=["BLOCK_SIZE_T"],
)
@triton.jit(do_not_specialize_on_alignment=["prefix_lens"])
def _topk_index_kernel(
s_ptr, # [num_heads, total_q, max_block]
ti_ptr, # [num_heads, total_q, topk]
sample_interval: tl.constexpr, # block_size_q (1 for M3)
block_size: tl.constexpr, # sparse block size (128)
cu_seqlens,
cu_seqblocks_q,
prefix_lens,
topk,
init_blocks: tl.constexpr,
local_blocks: tl.constexpr,
stride_s_h,
stride_s_n,
stride_s_k,
stride_ti_h,
stride_ti_n,
stride_ti_t,
BLOCK_SIZE_K: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
MASK_INIT: tl.constexpr,
MASK_LOCAL: tl.constexpr,
):
tl.static_assert(BLOCK_SIZE_K > BLOCK_SIZE_T)
pid_q = tl.program_id(0)
pid_b = tl.program_id(1)
pid_h = tl.program_id(2)
seq_start = tl.load(cu_seqlens + pid_b)
block_start = tl.load(cu_seqblocks_q + pid_b)
block_num = tl.load(cu_seqblocks_q + pid_b + 1) - block_start
prefix_len = tl.load(prefix_lens + pid_b)
if pid_q >= block_num:
return
off_k = tl.arange(0, BLOCK_SIZE_K)
off_t = tl.arange(0, BLOCK_SIZE_T)
s_ptrs = (
s_ptr
+ (seq_start + pid_q * sample_interval) * stride_s_n
+ pid_h * stride_s_h
+ off_k * stride_s_k
)
topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
valid_blocks = (prefix_len + pid_q * sample_interval + block_size) // block_size
for i in tl.range(0, valid_blocks, BLOCK_SIZE_K):
causal_mask = i + off_k < valid_blocks
local_mask = i + off_k >= max(0, valid_blocks - local_blocks)
init_mask = i + off_k < init_blocks
score = tl.load(s_ptrs, mask=causal_mask, other=-1e30).to(tl.float32)
score = tl.where(score != score, -1e30, score)
s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
if MASK_INIT:
score = tl.where(causal_mask & init_mask, score - 1e29, score)
else:
score = tl.where(causal_mask & init_mask, 1e30, score)
if MASK_LOCAL:
score = tl.where(causal_mask & local_mask, score - 1e28, score)
else:
score = tl.where(causal_mask & local_mask, 1e29, score)
topk_score, last_topk_score = score, topk_score
topk_idx, last_topk_idx = (tl.where(causal_mask, i + off_k + 1, 0), topk_idx)
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
for j in tl.static_range(1, n_dims):
topk_score, topk_idx = _bitonic_merge(
topk_score, topk_idx.to(tl.int32), j, 2, n_dims
)
if i != 0:
topk_score, topk_idx = _bitonic_merge(
topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
)
topk_score_new = last_topk_score * left_half_mask + topk_score * (
1 - left_half_mask
)
topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
1 - left_half_mask
)
topk_score, topk_idx = _bitonic_merge(
topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
)
else:
topk_score, topk_idx = _bitonic_merge(
topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
)
topk_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
topk_idx = tl.sum(
topk_mask[:, None]
* tl.reshape(topk_idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
axis=0,
)
ti_ptrs = (
ti_ptr
+ (block_start + pid_q) * stride_ti_n
+ pid_h * stride_ti_h
+ off_t * stride_ti_t
)
store_mask = off_t < topk
valid_mask = off_t < valid_blocks
topk_idx = tl.where(store_mask & valid_mask, topk_idx, -1)
tl.store(ti_ptrs, topk_idx.to(ti_ptrs.dtype.element_ty), mask=store_mask)
# ---------------------------------------------------------------------------
# Decode index-score kernel (split-K over seq blocks). Decode batches are
# flattened request-major, with a runtime query length used to map each query
# token back to its request metadata. Chunk counts depend only on shape
# constants so the grid is fixed within a cuda graph. Base-2 (exp2/log2)
# softmax matches prefill.
# ---------------------------------------------------------------------------
@triton.jit(do_not_specialize=["num_kv_chunks", "decode_query_len"])
def _decode_index_score_kernel(
q_ptr, # idx_q: [total_q, num_idx_heads, head_dim]
ik_cache_ptr, # index-K cache: [num_blocks, 128, head_dim]
score_ptr, # [num_idx_heads, total_q, max_block]
block_table_ptr, # [num_reqs, max_blocks]
seq_lens, # [num_reqs]
num_idx_heads: tl.constexpr,
head_dim: tl.constexpr,
init_blocks,
local_blocks,
sm_scale,
decode_query_len,
stride_q_n,
stride_q_h,
stride_q_d,
stride_ik_blk,
stride_ik_pos,
stride_ik_d,
stride_s_h,
stride_s_n,
stride_s_k,
stride_bt_b,
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
num_kv_chunks,
USE_PDL: tl.constexpr,
):
sm_scale_log2e = sm_scale * 1.4426950409
pid_b = tl.program_id(0) # flattened query-token id
pid_c = tl.program_id(1)
req_id = pid_b // decode_query_len
q_offset = pid_b - req_id * decode_query_len
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
seq_len = tl.load(seq_lens + req_id)
query_pos = seq_len - decode_query_len + q_offset
# Full-CG padding uses zero-length request rows. Clamp to an empty
# attention range instead of letting padded rows produce negative lengths.
kv_len = tl.maximum(query_pos + 1, 0)
num_blocks = (kv_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K
# block-aligned fixed-count split: grid independent of seq_len (cuda graph).
chunk_size_blocks = (num_blocks + num_kv_chunks - 1) // num_kv_chunks
chunk_start_block = pid_c * chunk_size_blocks
chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
if chunk_start_block >= chunk_end_block:
return
off_k = tl.arange(0, BLOCK_SIZE_K) # positions within a 128-block
off_d = tl.arange(0, head_dim)
bt_row = block_table_ptr + req_id * stride_bt_b
# Force-select init (1e30) and local (1e29, higher priority) blocks.
local_start = tl.maximum(0, num_blocks - local_blocks)
# query vectors across all heads
q = tl.load(
q_ptr
+ pid_b * stride_q_n
+ tl.arange(0, num_idx_heads) * stride_q_h
+ off_d[:, None] * stride_q_d,
) # [D,H]
for blk in tl.range(chunk_start_block, chunk_end_block):
page = tl.load(bt_row + blk).to(tl.int64)
pos = blk * BLOCK_SIZE_K + off_k
pos_mask = pos < kv_len
# we don't need masked load for K, because KV cache ensures
# allocation is multiple of BLOCK_SIZE_K.
# for tokens beyond seqlen, they will be masked in qk later.
k = tl.load(
ik_cache_ptr
+ page * stride_ik_blk
+ off_k[:, None] * stride_ik_pos
+ off_d * stride_ik_d,
) # [N,D]
kq = tl.dot(k, q) * sm_scale_log2e # [N,H]
kq = tl.where(pos_mask[:, None], kq, float("-inf"))
score = tl.max(kq, axis=0) # [H]
is_init = blk < init_blocks
is_local = (blk >= local_start) & (blk < num_blocks)
score = tl.where(is_local, 1e29, tl.where(is_init, 1e30, score))
tl.store(
score_ptr
+ tl.arange(0, num_idx_heads) * stride_s_h
+ pid_b * stride_s_n
+ blk * stride_s_k,
score,
)
# ---------------------------------------------------------------------------
# Decode top-k (split-K): per-chunk partial top-k + merge. Forced init/local
# blocks are already encoded in the scores.
# ---------------------------------------------------------------------------
@triton.heuristics({"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"])})
@triton.autotune(
configs=[
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2),
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3),
triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),
],
key=["topk"],
)
@triton.jit(do_not_specialize=["chunk_blocks", "decode_query_len"])
def _topk_index_partial_kernel(
s_ptr, # score: [num_idx_heads, total_q, max_block]
ts_partial_ptr, # partial scores out: [NUM_TOPK_CHUNKS, num_idx_heads, total_q, T]
ti_partial_ptr, # partial idx out (1-indexed global, 0=invalid): same shape
seq_lens, # [num_reqs]
block_size: tl.constexpr, # sparse block size (128)
topk: tl.constexpr,
chunk_blocks, # how many score-blocks each chunk owns
decode_query_len,
stride_s_h,
stride_s_b,
stride_s_k,
stride_ts_c,
stride_ts_h,
stride_ts_b,
stride_ts_t,
stride_ti_c,
stride_ti_h,
stride_ti_b,
stride_ti_t,
BLOCK_SIZE_K: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
USE_PDL: tl.constexpr,
):
tl.static_assert(topk < BLOCK_SIZE_K)
pid_b = tl.program_id(0) # flattened query-token id
pid_h = tl.program_id(1)
pid_chunk = tl.program_id(2)
req_id = pid_b // decode_query_len
q_offset = pid_b - req_id * decode_query_len
if USE_PDL:
tl.extra.cuda.gdc_wait()
seq_len = tl.load(seq_lens + req_id)
query_pos = seq_len - decode_query_len + q_offset
# Full-CG padding uses zero-length request rows. Clamp to an empty
# attention range instead of letting padded rows produce negative lengths.
kv_len = tl.maximum(query_pos + 1, 0)
num_blocks = (kv_len + block_size - 1) // block_size
# Slice this chunk owns within [0, num_blocks).
chunk_start = pid_chunk * chunk_blocks
chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks)
chunk_actual = tl.maximum(chunk_end - chunk_start, 0)
off_k = tl.arange(0, BLOCK_SIZE_K)
off_t = tl.arange(0, BLOCK_SIZE_T)
s_ptrs = (
s_ptr
+ pid_b * stride_s_b
+ pid_h * stride_s_h
+ (chunk_start + off_k) * stride_s_k
)
topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
# Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty
# chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0.
for i in tl.range(0, chunk_actual, BLOCK_SIZE_K):
mask = off_k < chunk_actual - i
score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32)
score = tl.where(score != score, -1e30, score)
s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
topk_score, last_topk_score = score, topk_score
topk_idx, last_topk_idx = (
tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global
topk_idx,
)
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
for j in tl.static_range(1, n_dims):
topk_score, topk_idx = _bitonic_merge(
topk_score, topk_idx.to(tl.int32), j, 2, n_dims
)
if i != 0:
topk_score, topk_idx = _bitonic_merge(
topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
)
topk_score_new = last_topk_score * left_half_mask + topk_score * (
1 - left_half_mask
)
topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
1 - left_half_mask
)
topk_score, topk_idx = _bitonic_merge(
topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
)
else:
topk_score, topk_idx = _bitonic_merge(
topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
)
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
# Extract first BLOCK_SIZE_T entries (top-K of this chunk after the sort).
topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
final_score = tl.sum(
topk_mask_extract[:, None]
* tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
axis=0,
)
final_idx = tl.sum(
topk_mask_extract[:, None]
* tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
axis=0,
)
# Always write all BLOCK_SIZE_T slots — invalid slots carry -1e30 / 0
# sentinels and lose to real scores in the merge stage.
ts_ptrs = (
ts_partial_ptr
+ pid_chunk * stride_ts_c
+ pid_b * stride_ts_b
+ pid_h * stride_ts_h
+ off_t * stride_ts_t
)
ti_ptrs = (
ti_partial_ptr
+ pid_chunk * stride_ti_c
+ pid_b * stride_ti_b
+ pid_h * stride_ti_h
+ off_t * stride_ti_t
)
tl.store(ts_ptrs, final_score)
tl.store(ti_ptrs, final_idx)
@triton.heuristics(
{
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
"BLOCK_SIZE_K": lambda args: triton.next_power_of_2(
args["num_topk_chunks"] * triton.next_power_of_2(args["topk"])
),
}
)
@triton.jit(do_not_specialize=["num_topk_chunks", "decode_query_len"])
def _topk_index_merge_kernel(
ts_partial_ptr, # partial scores: [NUM_TOPK_CHUNKS, num_idx_heads, total_q, T]
ti_partial_ptr, # partial idx (1-indexed global, 0=invalid): same shape
ti_final_ptr, # final idx (0-indexed, -1=invalid): [num_idx_heads, total_q, topk]
seq_lens, # [num_reqs]
block_size: tl.constexpr, # sparse block size (128)
topk: tl.constexpr,
decode_query_len,
stride_ts_c,
stride_ts_h,
stride_ts_b,
stride_ts_t,
stride_ti_c,
stride_ti_h,
stride_ti_b,
stride_ti_t,
stride_tif_h,
stride_tif_b,
stride_tif_t,
num_topk_chunks,
BLOCK_SIZE_K: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
USE_PDL: tl.constexpr,
):
pid_b = tl.program_id(0) # flattened query-token id
pid_h = tl.program_id(1)
req_id = pid_b // decode_query_len
q_offset = pid_b - req_id * decode_query_len
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
seq_len = tl.load(seq_lens + req_id)
query_pos = seq_len - decode_query_len + q_offset
# Full-CG padding uses zero-length request rows. Clamp to an empty
# attention range instead of letting padded rows produce negative lengths.
kv_len = tl.maximum(query_pos + 1, 0)
num_blocks = (kv_len + block_size - 1) // block_size
# Load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K.
# Candidate at flat position p comes from chunk = p // BLOCK_SIZE_T,
# in_chunk = p % BLOCK_SIZE_T.
off = tl.arange(0, BLOCK_SIZE_K)
chunk_idx = off // BLOCK_SIZE_T
in_chunk_idx = off % BLOCK_SIZE_T
valid = chunk_idx < num_topk_chunks
score_offset = (
chunk_idx * stride_ts_c
+ pid_h * stride_ts_h
+ pid_b * stride_ts_b
+ in_chunk_idx * stride_ts_t
)
idx_offset = (
chunk_idx * stride_ti_c
+ pid_h * stride_ti_h
+ pid_b * stride_ti_b
+ in_chunk_idx * stride_ti_t
)
score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to(
tl.float32
)
score = tl.where(score != score, -1e30, score)
idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32)
# Full bitonic descending sort of BLOCK_SIZE_K items.
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
for j in tl.static_range(1, n_dims):
score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims)
score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims)
# Extract first BLOCK_SIZE_T positions — these are the global top-K.
extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
topk_idx_final = tl.sum(
extract_mask[:, None]
* tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
axis=0,
)
off_t = tl.arange(0, BLOCK_SIZE_T)
tif_ptrs = (
ti_final_ptr
+ pid_h * stride_tif_h
+ pid_b * stride_tif_b
+ off_t * stride_tif_t
)
store_mask = off_t < topk
topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1)
tl.store(
tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty), mask=store_mask
)
# ---------------------------------------------------------------------------
# Python wrappers
# ---------------------------------------------------------------------------
@torch.no_grad()
def minimax_m3_index_score(
idx_q: torch.Tensor, # [total_q, num_idx_heads, head_dim]
index_kv_cache: torch.Tensor, # [num_blocks, 128, head_dim]
block_table: torch.Tensor, # [batch, max_blocks]
cu_seqlens_q: torch.Tensor, # [batch+1] int32
seq_lens: torch.Tensor, # [batch] int32
prefix_lens: torch.Tensor, # [batch] int32
max_query_len: int,
max_seq_len: int,
num_kv_heads: int,
sm_scale: float,
) -> torch.Tensor:
"""Compute per-token index scores for each visible sparse block.
Returns score [num_kv_heads, total_q, max_block], where each score is the
max over a 128-token index-K block. M3 has num_idx_heads == num_kv_heads.
"""
total_q, num_idx_heads, head_dim = idx_q.shape
assert num_idx_heads == num_kv_heads, (
"M3 expects num_idx_heads == num_kv_heads (no topk index reduce)"
)
batch = cu_seqlens_q.shape[0] - 1
max_block = triton.cdiv(max_seq_len, SPARSE_BLOCK_SIZE)
# Keep score strides 16-divisible to avoid Triton recompiles.
score_block_stride = round_up(max_block, 16)
score = torch.empty(
(num_idx_heads, total_q, score_block_stride),
dtype=torch.float32,
device=idx_q.device,
)
BLOCK_SIZE_Q = 64
grid_score = (triton.cdiv(max_query_len, BLOCK_SIZE_Q), batch * num_idx_heads)
_index_block_score_kernel[grid_score](
idx_q,
index_kv_cache,
score,
block_table,
cu_seqlens_q,
seq_lens,
prefix_lens,
num_idx_heads,
head_dim,
sm_scale,
idx_q.stride(0),
idx_q.stride(1),
idx_q.stride(2),
index_kv_cache.stride(0),
index_kv_cache.stride(1),
index_kv_cache.stride(2),
score.stride(0),
score.stride(1),
score.stride(2),
block_table.stride(0),
BLOCK_SIZE_Q=BLOCK_SIZE_Q,
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
)
return score
@torch.no_grad()
def minimax_m3_index_topk(
score: torch.Tensor, # [num_idx_heads, total_q, max_block]
cu_seqlens_q: torch.Tensor, # [batch+1] int32
prefix_lens: torch.Tensor, # [batch] int32
max_query_len: int,
topk: int,
init_blocks: int,
local_blocks: int,
) -> torch.Tensor:
"""Select index top-k from a precomputed score tensor."""
num_idx_heads = score.shape[0]
batch = cu_seqlens_q.shape[0] - 1
total_q = score.shape[1]
topk_idx = torch.empty(
(num_idx_heads, total_q, topk),
dtype=torch.int32,
device=score.device,
)
# block_size_q == 1 -> query blocks coincide with query tokens.
grid_topk = (max_query_len, batch, num_idx_heads)
_topk_index_kernel[grid_topk](
score,
topk_idx,
1, # sample_interval (block_size_q)
SPARSE_BLOCK_SIZE,
cu_seqlens_q,
cu_seqlens_q, # cu_seqblocks_q == cu_seqlens_q when block_size_q == 1
prefix_lens,
topk,
init_blocks,
local_blocks,
score.stride(0),
score.stride(1),
score.stride(2),
topk_idx.stride(0),
topk_idx.stride(1),
topk_idx.stride(2),
MASK_INIT=False,
MASK_LOCAL=False,
)
return topk_idx
@torch.no_grad()
def minimax_m3_index_decode(
idx_q: torch.Tensor, # [total_q, num_idx_heads, head_dim]
index_kv_cache: torch.Tensor, # [num_blocks, 128, head_dim]
block_table: torch.Tensor, # [num_reqs, max_blocks]
seq_lens: torch.Tensor, # [num_reqs] int32
max_seq_len: int,
topk: int,
init_blocks: int,
local_blocks: int,
num_kv_heads: int,
sm_scale: float,
decode_query_len: int,
) -> torch.Tensor:
"""Decode index block-score + top-k, both split-K (cudagraph-safe).
Returns topk_idx [num_kv_heads, total_q, topk] (0-indexed block ids, -1 pad).
"""
total_q, num_idx_heads, head_dim = idx_q.shape
assert num_idx_heads == num_kv_heads, (
"M3 expects num_idx_heads == num_kv_heads (no topk index reduce)"
)
assert total_q == seq_lens.shape[0] * decode_query_len
batch = total_q
max_block = triton.cdiv(max_seq_len, SPARSE_BLOCK_SIZE)
use_pdl = current_platform.is_arch_support_pdl()
# `launch_pdl` is a Triton runtime kwarg only some backends accept (CUDA
# SM9+); this ROCm Triton rejects it even when False ("Keyword argument
# launch_pdl was specified but unrecognised"). Only pass it when PDL is
# actually supported -- on ROCm use_pdl is always False, so it's omitted.
pdl_launch = {"launch_pdl": True} if use_pdl else {}
# Keep score strides 16-divisible to avoid Triton recompiles.
score_block_stride = round_up(max_block, 16)
score = torch.empty(
(num_idx_heads, total_q, score_block_stride),
dtype=torch.float32,
device=idx_q.device,
)
# split-K over seq blocks; chunk count depends only on shape constants so
# the grid is fixed within a cuda graph.
TARGET_GRID = 4096
MAX_NUM_KV_CHUNKS = 256
target = max(
1, min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch * num_idx_heads))
)
num_kv_chunks = 1 << (target.bit_length() - 1)
grid_score = (batch, num_kv_chunks)
_decode_index_score_kernel[grid_score](
idx_q,
index_kv_cache,
score,
block_table,
seq_lens,
num_idx_heads,
head_dim,
init_blocks,
local_blocks,
sm_scale,
decode_query_len,
idx_q.stride(0),
idx_q.stride(1),
idx_q.stride(2),
index_kv_cache.stride(0),
index_kv_cache.stride(1),
index_kv_cache.stride(2),
score.stride(0),
score.stride(1),
score.stride(2),
block_table.stride(0),
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
num_kv_chunks=num_kv_chunks,
USE_PDL=use_pdl,
**pdl_launch,
)
topk_idx = torch.empty(
(num_idx_heads, total_q, topk),
dtype=torch.int32,
device=idx_q.device,
)
# Chunk count is shape-constant (cudagraph-safe), capped so the merge sorts
# pow2(num_topk_chunks * pow2(topk)) candidates.
TOPK_TARGET_GRID = 64
MAX_NUM_TOPK_CHUNKS = 16
topk_target = max(
1, min(MAX_NUM_TOPK_CHUNKS, TOPK_TARGET_GRID // max(1, batch * num_idx_heads))
)
num_topk_chunks = 1 << (topk_target.bit_length() - 1)
block_size_t = triton.next_power_of_2(topk)
chunk_blocks = (max_block + num_topk_chunks - 1) // num_topk_chunks
topk_score_partial = torch.empty(
num_topk_chunks,
num_idx_heads,
batch,
block_size_t,
dtype=torch.float32,
device=idx_q.device,
)
topk_idx_partial = torch.empty(
num_topk_chunks,
num_idx_heads,
batch,
block_size_t,
dtype=torch.int32,
device=idx_q.device,
)
_topk_index_partial_kernel[(batch, num_idx_heads, num_topk_chunks)](
score,
topk_score_partial,
topk_idx_partial,
seq_lens,
SPARSE_BLOCK_SIZE,
topk,
chunk_blocks,
decode_query_len,
score.stride(0),
score.stride(1),
score.stride(2),
topk_score_partial.stride(0),
topk_score_partial.stride(1),
topk_score_partial.stride(2),
topk_score_partial.stride(3),
topk_idx_partial.stride(0),
topk_idx_partial.stride(1),
topk_idx_partial.stride(2),
topk_idx_partial.stride(3),
USE_PDL=use_pdl,
**pdl_launch,
)
_topk_index_merge_kernel[(batch, num_idx_heads)](
topk_score_partial,
topk_idx_partial,
topk_idx,
seq_lens,
SPARSE_BLOCK_SIZE,
topk,
decode_query_len,
topk_score_partial.stride(0),
topk_score_partial.stride(1),
topk_score_partial.stride(2),
topk_score_partial.stride(3),
topk_idx_partial.stride(0),
topk_idx_partial.stride(1),
topk_idx_partial.stride(2),
topk_idx_partial.stride(3),
topk_idx.stride(0),
topk_idx.stride(1),
topk_idx.stride(2),
num_topk_chunks=num_topk_chunks,
USE_PDL=use_pdl,
**pdl_launch,
)
return topk_idx
@@ -0,0 +1,593 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton kernels for MiniMax M3 block-sparse GQA attention.
The main heads attend only to the blocks selected by the lightning indexer (see
``index_topk``). Adapted to vLLM's paged KV cache: the KV page size is forced to
equal the sparse block size (128), so one selected block maps to exactly one
page.
Main K/V cache layout (vLLM):
``(num_blocks, 2, 128, num_kv_heads, head_dim)`` K=[:,0] V=[:,1]
Only the paths MiniMax M3 uses are implemented: no attention sink, base-2
(exp2/log2) softmax. The decode kernels use split-K (flash-decoding) over the
selected blocks with a separate merge step, since one query token per request
leaves the prefill kernels (which parallelize over the query dim) idle.
"""
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
# One sparse block == one KV page.
SPARSE_BLOCK_SIZE = 128
_SPARSE_ATTN_NUM_STAGES_KWARG: dict | None = None
def _sparse_attn_num_stages_kwarg() -> dict:
"""Triton ``num_stages`` override for the sparse-attn GEMM kernels.
Forced only where required: CDNA3 (gfx942) caps LDS at
64 KB, and the default 2-stage pipeline double-buffers the 128x128 K/V tiles
to ~66 KB ("out of resource: shared memory"), so pin gfx942 to a single
stage (~32 KB, which fits). Everywhere else (NVIDIA, CDNA4 gfx950) return an
empty kwarg and let Triton keep its own default -- don't second-guess it.
Cached: the arch is fixed per process.
"""
global _SPARSE_ATTN_NUM_STAGES_KWARG
if _SPARSE_ATTN_NUM_STAGES_KWARG is None:
kwarg: dict = {}
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx942
if on_gfx942():
kwarg = {"num_stages": 1}
_SPARSE_ATTN_NUM_STAGES_KWARG = kwarg
return _SPARSE_ATTN_NUM_STAGES_KWARG
# ---------------------------------------------------------------------------
# GQA block-sparse attention (paged). Main heads attend only to the selected
# blocks. BLOCK_SIZE_K == 128 so each selected block is one page.
# ---------------------------------------------------------------------------
# since prefill metadata is sliced from mixed batch metadata, seq_lens and prefix_lens
# might lose pointer alignment, which trigger Triton recompiles. we don't actually
# need pointer alignment for those tensors anyway because we do scalar load.
@triton.heuristics(
{
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
"BLOCK_SIZE_H": lambda args: triton.next_power_of_2(args["gqa_group_size"]),
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
"BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"]
* triton.next_power_of_2(args["gqa_group_size"]),
}
)
@triton.jit(do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])
def _gqa_sparse_fwd_kernel(
q_ptr, # [total_q, num_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim]
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
o_ptr, # [total_q, num_heads, head_dim]
block_table_ptr, # [num_reqs, max_blocks]
cu_seqlens_q,
cu_seqblocks_q,
seq_lens,
prefix_lens,
num_kv_heads,
gqa_group_size,
head_dim,
max_topk,
num_q_loop,
sm_scale,
stride_qn,
stride_qh,
stride_qd,
stride_kv_blk,
stride_kv_kv,
stride_kv_pos,
stride_kv_h,
stride_kv_d,
stride_th,
stride_tn,
stride_tk,
stride_on,
stride_oh,
stride_od,
stride_bt_b,
BLOCK_SIZE_Q: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
BLOCK_SIZE_D: tl.constexpr,
BLOCK_SIZE_H: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
BLOCK_SIZE_QH: tl.constexpr,
USE_FP8: tl.constexpr, # fp8 KV cache: dequantize K/V to q.dtype on load
):
sm_scale_log2e = sm_scale * 1.4426950409
pid_q = tl.program_id(0)
pid_kh = tl.program_id(1)
pid_b = tl.program_id(2)
pid_h = pid_kh * gqa_group_size
q_start = tl.load(cu_seqlens_q + pid_b)
q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start
q_block_start = tl.load(cu_seqblocks_q + pid_b)
q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start
seq_len = tl.load(seq_lens + pid_b)
prefix_len = tl.load(prefix_lens + pid_b)
if pid_q * num_q_loop >= q_block_len:
return
real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop)
bt_row = block_table_ptr + pid_b * stride_bt_b
off_n = tl.arange(0, BLOCK_SIZE_K)
off_d = tl.arange(0, BLOCK_SIZE_D)
d_mask = off_d < head_dim
for j in range(real_q_loop):
pid_q_j = pid_q * num_q_loop + j
t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th
off_t = tl.arange(0, BLOCK_SIZE_T)
topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1)
real_topk = tl.sum((topk_idx >= 0).to(tl.int32), axis=0)
q_ptrs = tl.make_block_ptr(
base=q_ptr + q_start * stride_qn + pid_h * stride_qh,
shape=(q_len, gqa_group_size, head_dim),
strides=(stride_qn, stride_qh, stride_qd),
offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D),
order=(2, 1, 0),
)
q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero")
off_q = (
tl.arange(0, BLOCK_SIZE_Q)[:, None]
+ pid_q_j * BLOCK_SIZE_Q
+ prefix_len
- tl.arange(0, BLOCK_SIZE_K)[None, :]
)
m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
acc_o = tl.zeros((BLOCK_SIZE_QH, BLOCK_SIZE_D), dtype=tl.float32)
q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_D)
for _ in range(real_topk):
blk = tl.load(t_ptr_j).to(tl.int32)
t_ptr_j = t_ptr_j + stride_tk
c = blk * BLOCK_SIZE_K
page = tl.load(bt_row + blk).to(tl.int64)
pos = c + off_n
pos_mask = pos < seq_len
k = tl.load(
kv_cache_ptr
+ page * stride_kv_blk
+ 0 * stride_kv_kv
+ off_n[None, :] * stride_kv_pos
+ pid_kh * stride_kv_h
+ off_d[:, None] * stride_kv_d,
mask=d_mask[:, None] & pos_mask[None, :],
other=0.0,
)
if USE_FP8:
k = k.to(q.dtype)
qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
# causal: q_abs_pos - k_off >= block_start (c)
qk += tl.where(off_q[:, None, :] >= c, 0, float("-inf"))
qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
qk += tl.dot(q, k) * sm_scale_log2e
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
p = tl.exp2(qk - m_ij[:, None])
l_ij = tl.sum(p, axis=1)
acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]
v = tl.load(
kv_cache_ptr
+ page * stride_kv_blk
+ 1 * stride_kv_kv
+ off_n[:, None] * stride_kv_pos
+ pid_kh * stride_kv_h
+ off_d[None, :] * stride_kv_d,
mask=pos_mask[:, None] & d_mask[None, :],
other=0.0,
)
if USE_FP8:
v = v.to(q.dtype)
acc_o += tl.dot(p.to(v.dtype), v)
m_i = m_ij
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]
acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D)
o_ptrs = tl.make_block_ptr(
base=o_ptr + q_start * stride_on + pid_h * stride_oh,
shape=(q_len, gqa_group_size, head_dim),
strides=(stride_on, stride_oh, stride_od),
offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D),
order=(2, 1, 0),
)
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2))
# ---------------------------------------------------------------------------
# Decode kernels (split-K). Decode batches are flattened request-major, with a
# runtime query length used to map each query token back to its request metadata.
# This parallelizes over the selected top-k blocks, producing partials that the
# merge kernel combines (flash-decoding). All chunk counts depend only on shape
# constants so the grid is fixed within a cuda graph. Base-2 (exp2/log2)
# softmax matches the prefill kernel.
# ---------------------------------------------------------------------------
@triton.heuristics(
{
"BLOCK_SIZE_H": lambda args: max(
16, triton.next_power_of_2(args["gqa_group_size"])
),
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
}
)
@triton.jit(do_not_specialize=["decode_query_len"])
def _gqa_sparse_decode_kernel(
q_ptr, # [total_q, num_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim]
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
o_ptr, # partial out: [NUM_TOPK_CHUNKS, total_q, num_heads, head_dim]
lse_ptr, # partial lse (log2): [NUM_TOPK_CHUNKS, total_q, num_heads]
block_table_ptr, # [num_reqs, max_blocks]
seq_lens, # [num_reqs]
total_q,
gqa_group_size,
head_dim,
max_topk,
sm_scale,
decode_query_len,
stride_qn,
stride_qh,
stride_qd,
stride_kv_blk,
stride_kv_kv,
stride_kv_pos,
stride_kv_h,
stride_kv_d,
stride_th,
stride_tn,
stride_tk,
stride_o_c,
stride_o_b,
stride_o_h,
stride_o_d,
stride_l_c,
stride_l_b,
stride_l_h,
stride_bt_b,
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
NUM_TOPK_CHUNKS: tl.constexpr,
BLOCK_SIZE_H: tl.constexpr,
BLOCK_SIZE_D: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
USE_FP8: tl.constexpr, # fp8 KV cache: dequantize K/V to q.dtype on load
USE_PDL: tl.constexpr,
):
sm_scale_log2e = sm_scale * 1.4426950409
# split-K over the topk dimension: pid(0) folds (query-token, chunk).
pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
pid_b = pid_bc % total_q
pid_c = pid_bc // total_q
req_id = pid_b // decode_query_len
q_offset = pid_b - req_id * decode_query_len
pid_h = pid_kh * gqa_group_size
chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
chunk_start_topk = pid_c * chunk_size_topk
chunk_end_compiletime = chunk_start_topk + chunk_size_topk
if USE_PDL:
tl.extra.cuda.gdc_wait()
seq_len = tl.load(seq_lens + req_id)
query_pos = seq_len - decode_query_len + q_offset
# Full-CG padding uses zero-length request rows. Clamp to an empty
# attention range instead of letting padded rows produce negative lengths.
kv_len = tl.maximum(query_pos + 1, 0)
# number of valid (non-padded) selected blocks for this query token
off_t = tl.arange(0, BLOCK_SIZE_T)
idx_base = t_ptr + pid_kh * stride_th + pid_b * stride_tn
topk_idx = tl.load(idx_base + off_t * stride_tk, mask=off_t < max_topk, other=-1)
real_topk = tl.sum((topk_idx >= 0).to(tl.int32), axis=0)
chunk_end_topk = tl.minimum(chunk_end_compiletime, real_topk)
off_n = tl.arange(0, BLOCK_SIZE_K)
off_d = tl.arange(0, BLOCK_SIZE_D)
d_mask = off_d < head_dim
bt_row = block_table_ptr + req_id * stride_bt_b
m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
acc_o = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_D), dtype=tl.float32)
q_ptrs = tl.make_block_ptr(
base=q_ptr + pid_b * stride_qn + pid_h * stride_qh,
shape=(gqa_group_size, head_dim),
strides=(stride_qh, stride_qd),
offsets=(0, 0),
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
order=(1, 0),
)
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
cur_idx_ptr = idx_base + chunk_start_topk * stride_tk
for _ in tl.range(chunk_start_topk, chunk_end_topk):
blk = tl.load(cur_idx_ptr).to(tl.int32)
cur_idx_ptr = cur_idx_ptr + stride_tk
c = blk * BLOCK_SIZE_K
page = tl.load(bt_row + blk).to(tl.int64)
pos = c + off_n
pos_mask = pos < kv_len
k = tl.load(
kv_cache_ptr
+ page * stride_kv_blk
+ 0 * stride_kv_kv
+ off_n[None, :] * stride_kv_pos
+ pid_kh * stride_kv_h
+ off_d[:, None] * stride_kv_d,
mask=d_mask[:, None] & pos_mask[None, :],
other=0.0,
)
if USE_FP8:
k = k.to(q.dtype)
qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
qk += tl.dot(q, k) * sm_scale_log2e
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
p = tl.exp2(qk - m_ij[:, None])
l_ij = tl.sum(p, axis=1)
acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]
v = tl.load(
kv_cache_ptr
+ page * stride_kv_blk
+ 1 * stride_kv_kv
+ off_n[:, None] * stride_kv_pos
+ pid_kh * stride_kv_h
+ off_d[None, :] * stride_kv_d,
mask=pos_mask[:, None] & d_mask[None, :],
other=0.0,
)
if USE_FP8:
v = v.to(q.dtype)
acc_o += tl.dot(p.to(v.dtype), v)
m_i = m_ij
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
# Empty chunks for active rows must store zero output; otherwise the merge
# can hit 0 * NaN. All-empty padded rows may still produce NaNs in merge.
scale = tl.where(lse_i > float("-inf"), tl.exp2(m_i - lse_i), tl.zeros_like(lse_i))
acc_o = acc_o * scale[:, None]
o_ptrs = tl.make_block_ptr(
base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
shape=(gqa_group_size, head_dim),
strides=(stride_o_h, stride_o_d),
offsets=(0, 0),
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
order=(1, 0),
)
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
lse_ptrs = tl.make_block_ptr(
base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
shape=(gqa_group_size,),
strides=(stride_l_h,),
offsets=(0,),
block_shape=(BLOCK_SIZE_H,),
order=(0,),
)
tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
@triton.heuristics(
{"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"])}
)
@triton.jit
def _merge_topk_attn_out_kernel(
o_ptr, # partials: [NUM_TOPK_CHUNKS, total_q, num_heads, head_dim]
lse_ptr, # partials (log2): [NUM_TOPK_CHUNKS, total_q, num_heads]
out_ptr, # merged out: [total_q, num_heads, head_dim]
head_dim,
stride_o_c,
stride_o_b,
stride_o_h,
stride_o_d,
stride_l_c,
stride_l_b,
stride_l_h,
stride_out_n,
stride_out_h,
stride_out_d,
NUM_TOPK_CHUNKS: tl.constexpr,
BLOCK_SIZE_D: tl.constexpr,
USE_PDL: tl.constexpr,
):
pid_b, pid_h = tl.program_id(0), tl.program_id(1)
# NOTE: assume seq_lens is safe to load before gdc_wait()
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
off_c = tl.arange(0, NUM_TOPK_CHUNKS)
off_d = tl.arange(0, BLOCK_SIZE_D)
o_ptrs = tl.make_block_ptr(
base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
shape=(NUM_TOPK_CHUNKS, head_dim),
strides=(stride_o_c, stride_o_d),
offsets=(0, 0),
block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D),
order=(1, 0),
)
lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0
lse_max = tl.max(lse, axis=0)
weights = tl.exp2(lse - lse_max)
weights = weights / tl.sum(weights, axis=0)
o_merged = tl.sum(o * weights[:, None], axis=0)
out_ptrs = (
out_ptr + pid_b * stride_out_n + pid_h * stride_out_h + off_d * stride_out_d
)
tl.store(out_ptrs, o_merged.to(out_ptr.dtype.element_ty), mask=off_d < head_dim)
# ---------------------------------------------------------------------------
# Python wrappers
# ---------------------------------------------------------------------------
@torch.no_grad()
def minimax_m3_sparse_attn(
q: torch.Tensor, # [total_q, num_heads, head_dim]
kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim]
topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
block_table: torch.Tensor, # [batch, max_blocks]
cu_seqlens_q: torch.Tensor, # [batch+1] int32
seq_lens: torch.Tensor, # [batch] int32
prefix_lens: torch.Tensor, # [batch] int32
max_query_len: int,
num_kv_heads: int,
sm_scale: float,
output: torch.Tensor, # [total_q, num_heads, head_dim]
) -> None:
"""GQA block-sparse attention over the selected blocks. block_size_q == 1."""
total_q, num_heads, head_dim = q.shape
batch = cu_seqlens_q.shape[0] - 1
topk = topk_idx.shape[-1]
gqa_group_size = num_heads // num_kv_heads
use_fp8 = kv_cache.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
grid = (max_query_len, num_kv_heads, batch)
_gqa_sparse_fwd_kernel[grid](
q,
kv_cache,
topk_idx,
output,
block_table,
cu_seqlens_q,
cu_seqlens_q, # cu_seqblocks_q == cu_seqlens_q when block_size_q == 1
seq_lens,
prefix_lens,
num_kv_heads,
gqa_group_size,
head_dim,
topk,
1, # num_q_loop
sm_scale,
q.stride(0),
q.stride(1),
q.stride(2),
kv_cache.stride(0),
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
kv_cache.stride(4),
topk_idx.stride(0),
topk_idx.stride(1),
topk_idx.stride(2),
output.stride(0),
output.stride(1),
output.stride(2),
block_table.stride(0),
BLOCK_SIZE_Q=1,
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
USE_FP8=use_fp8,
**_sparse_attn_num_stages_kwarg(),
)
@torch.no_grad()
def minimax_m3_sparse_attn_decode(
q: torch.Tensor, # [total_q, num_heads, head_dim]
kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim]
topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
block_table: torch.Tensor, # [num_reqs, max_blocks]
seq_lens: torch.Tensor, # [num_reqs] int32
num_kv_heads: int,
sm_scale: float,
output: torch.Tensor, # [total_q, num_heads, head_dim]
decode_query_len: int,
) -> None:
"""GQA block-sparse attention for decode (split-K over the top-k blocks)."""
total_q, num_heads, head_dim = q.shape
assert total_q == seq_lens.shape[0] * decode_query_len
max_topk = topk_idx.shape[-1]
gqa_group_size = num_heads // num_kv_heads
use_fp8 = kv_cache.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
use_pdl = current_platform.is_arch_support_pdl()
# `launch_pdl` is a Triton runtime kwarg only some backends accept (CUDA
# SM9+); this ROCm Triton rejects it even when False ("Keyword argument
# launch_pdl was specified but unrecognised"). Only pass it when PDL is
# actually supported -- on ROCm use_pdl is always False, so it's omitted.
pdl_launch = {"launch_pdl": True} if use_pdl else {}
# split-K over the selected blocks; chunk count is shape-constant (cuda graph).
TARGET_GRID = 256
target = max(1, min(max_topk, TARGET_GRID // max(1, total_q * num_kv_heads)))
num_topk_chunks = 1 << (target.bit_length() - 1)
o_partial = torch.empty(
num_topk_chunks, total_q, num_heads, head_dim, dtype=q.dtype, device=q.device
)
lse_partial = torch.empty(
num_topk_chunks, total_q, num_heads, dtype=torch.float32, device=q.device
)
grid = (total_q * num_topk_chunks, num_kv_heads)
_gqa_sparse_decode_kernel[grid](
q,
kv_cache,
topk_idx,
o_partial,
lse_partial,
block_table,
seq_lens,
total_q,
gqa_group_size,
head_dim,
max_topk,
sm_scale,
decode_query_len,
q.stride(0),
q.stride(1),
q.stride(2),
kv_cache.stride(0),
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
kv_cache.stride(4),
topk_idx.stride(0),
topk_idx.stride(1),
topk_idx.stride(2),
o_partial.stride(0),
o_partial.stride(1),
o_partial.stride(2),
o_partial.stride(3),
lse_partial.stride(0),
lse_partial.stride(1),
lse_partial.stride(2),
block_table.stride(0),
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
NUM_TOPK_CHUNKS=num_topk_chunks,
USE_FP8=use_fp8,
USE_PDL=use_pdl,
**_sparse_attn_num_stages_kwarg(),
**pdl_launch,
)
merge_grid = (total_q, num_heads)
_merge_topk_attn_out_kernel[merge_grid](
o_partial,
lse_partial,
output,
head_dim,
o_partial.stride(0),
o_partial.stride(1),
o_partial.stride(2),
o_partial.stride(3),
lse_partial.stride(0),
lse_partial.stride(1),
lse_partial.stride(2),
output.stride(0),
output.stride(1),
output.stride(2),
NUM_TOPK_CHUNKS=num_topk_chunks,
USE_PDL=use_pdl,
**pdl_launch,
)
@@ -0,0 +1,398 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Main block-sparse GQA attention for MiniMax M3 sparse layers.
The lightning indexer (``indexer.py``) selects the top-k KV blocks; this module
holds the main attention that attends only to those blocks: the paged K/V cache
backend, its metadata + builder, and the impl that consumes the indexer's
``topk_idx``. The Triton attend kernel lives here; the SM100 (MSA)
``build_k2q_csr`` + ``sparse_atten_func`` attend lives in
``nvidia/sparse_attention_msa.py``.
``MiniMaxM3SparseBackend`` and ``MiniMaxM3SparseMetadata`` are referenced by the
attention-backend registry (by dotted path) and by spec-decode, so they must
keep these names and stay in this module.
"""
from dataclasses import dataclass
from typing import ClassVar
import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.models.minimax_m3.common.ops.sparse_attn import (
SPARSE_BLOCK_SIZE,
minimax_m3_sparse_attn,
minimax_m3_sparse_attn_decode,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionImplBase,
AttentionLayer,
AttentionMetadata,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.attention.backends.utils import (
get_kv_cache_layout,
split_decodes_and_prefills,
)
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
class MiniMaxM3SparseBackend(AttentionBackend):
"""Block-sparse GQA backend for MiniMax M3 sparse attention layers."""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16, torch.float16]
# bf16 or fp8 (e4m3/e5m2): the Triton kernels dequant fp8 before the dots.
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"bfloat16",
"fp8",
"fp8_e4m3",
"fp8_e5m2",
]
@staticmethod
def get_name() -> str:
return "MINIMAX_M3_SPARSE"
@staticmethod
def get_impl_cls() -> type["MiniMaxM3SparseImpl"]:
# Concrete impl chosen by select_main_impl_cls; base for introspection.
return MiniMaxM3SparseImpl
@staticmethod
def get_builder_cls() -> type["MiniMaxM3SparseMetadataBuilder"]:
return MiniMaxM3SparseMetadataBuilder
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
return [128]
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
# Page size == sparse block size (one sparse block per KV page).
return [128]
@classmethod
def is_sparse(cls) -> bool:
return True
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, 2, block_size, num_kv_heads, head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# Permutation from get_kv_cache_shape to the actual memory layout.
if include_num_layers_dimension:
raise NotImplementedError # no cross-layer KV blocks in M3
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD":
stride_order = (0, 1, 2, 3, 4)
elif cache_layout == "HND":
stride_order = (0, 1, 3, 2, 4)
else:
raise ValueError(f"Unknown cache layout format {cache_layout}.")
return stride_order
@dataclass
class MiniMaxM3SparsePrefillMetadata:
"""Per-prefill state; ``cu_seqlens_k``/``total_kv_blocks`` feed the MSA CSR."""
cu_seqlens_q: torch.Tensor # [num_prefills + 1] int32, rebased to 0
cu_seqlens_k: torch.Tensor # [num_prefills + 1] int32, cumulative KV lengths
seq_lens: torch.Tensor # [num_prefills] int32, total KV lengths
context_lens: torch.Tensor # [num_prefills] int32 (cached/context tokens)
block_table: torch.Tensor
max_query_len: int
max_seq_len: int
total_kv_blocks: int
@dataclass
class MiniMaxM3SparseDecodeMetadata:
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
per-request query length (1, or 1 + num_speculative_tokens)."""
seq_lens: torch.Tensor # [num_decodes] int32
block_table: torch.Tensor
decode_query_len: int
@dataclass
class MiniMaxM3SparseMetadata(AttentionMetadata):
"""Sparse-attention metadata, split into prefill and decode sub-metadata."""
seq_lens: torch.Tensor
max_seq_len: int
slot_mapping: torch.Tensor
num_actual_tokens: int # total query tokens (decode-first batch)
# Split counts (batch reordered decode-first).
num_decodes: int
num_decode_tokens: int
num_prefills: int
num_prefill_tokens: int
prefill: MiniMaxM3SparsePrefillMetadata | None = None
decode: MiniMaxM3SparseDecodeMetadata | None = None
class MiniMaxM3SparseMetadataBuilder(AttentionMetadataBuilder[MiniMaxM3SparseMetadata]):
# Full cudagraphs for uniform decode batches, incl. spec-decode verify
# batches with >1 query token/request.
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
# Raised to 1 + num_speculative_tokens by _init_reorder_batch_threshold when
# spec decode is on; must match the indexer builder so the splits agree.
reorder_batch_threshold: int = 1
def __init__(
self,
kv_cache_spec: AttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
# Stable context-length buffer for decode cudagraph replays.
self.context_len_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
dtype=torch.int32,
device=device,
)
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> MiniMaxM3SparseMetadata:
num_reqs = common_attn_metadata.num_reqs
num_tokens = common_attn_metadata.num_actual_tokens
query_start_loc = common_attn_metadata.query_start_loc
seq_lens = common_attn_metadata.seq_lens
block_table = common_attn_metadata.block_table_tensor
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
require_uniform=True,
)
)
assert num_decodes + num_prefills == num_reqs
assert num_decode_tokens + num_prefill_tokens == num_tokens
# Decode-first batch: context lengths into the stable cudagraph buffer.
context_lens = self.context_len_buffer[:num_reqs]
context_lens.copy_(
common_attn_metadata.compute_num_computed_tokens(), non_blocking=True
)
prefill_metadata: MiniMaxM3SparsePrefillMetadata | None = None
if num_prefills > 0:
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None
prefill_seq_lens_cpu = seq_lens_cpu[num_decodes:]
prefill_total_kv_blocks = (
((prefill_seq_lens_cpu + SPARSE_BLOCK_SIZE - 1) // SPARSE_BLOCK_SIZE)
.sum()
.item()
)
prefill_kv_lens = seq_lens[num_decodes:]
prefill_cu_seqlens_k = torch.empty(
num_prefills + 1, dtype=torch.int32, device=seq_lens.device
)
prefill_cu_seqlens_k[0] = 0
torch.cumsum(prefill_kv_lens, dim=0, out=prefill_cu_seqlens_k[1:])
prefill_metadata = MiniMaxM3SparsePrefillMetadata(
cu_seqlens_q=(query_start_loc[num_decodes:] - num_decode_tokens).to(
torch.int32
),
cu_seqlens_k=prefill_cu_seqlens_k,
seq_lens=prefill_kv_lens,
context_lens=context_lens[num_decodes:],
block_table=block_table[num_decodes:],
max_query_len=common_attn_metadata.max_query_len,
max_seq_len=common_attn_metadata.max_seq_len,
total_kv_blocks=prefill_total_kv_blocks,
)
decode_metadata: MiniMaxM3SparseDecodeMetadata | None = None
if num_decodes > 0:
qsl_cpu = common_attn_metadata.query_start_loc_cpu
query_lens_cpu = qsl_cpu[1 : num_decodes + 1] - qsl_cpu[:num_decodes]
decode_query_len = int(query_lens_cpu[0].item())
assert decode_query_len > 0
assert torch.all(
(query_lens_cpu == decode_query_len) | (query_lens_cpu == 0)
)
assert num_decode_tokens == num_decodes * decode_query_len
decode_metadata = MiniMaxM3SparseDecodeMetadata(
seq_lens=seq_lens[:num_decodes],
block_table=block_table[:num_decodes],
decode_query_len=decode_query_len,
)
return MiniMaxM3SparseMetadata(
seq_lens=seq_lens,
max_seq_len=common_attn_metadata.max_seq_len,
slot_mapping=common_attn_metadata.slot_mapping,
num_actual_tokens=num_tokens,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
num_prefills=num_prefills,
num_prefill_tokens=num_prefill_tokens,
prefill=prefill_metadata,
decode=decode_metadata,
)
class MiniMaxM3SparseImpl(AttentionImplBase[MiniMaxM3SparseMetadata]):
"""Abstract base for block-sparse GQA over the indexer-selected blocks.
Inherits ``AttentionImplBase`` for a custom forward signature (the layer
pre-inserts K/V and runs the indexer, so forward takes the queries +
``topk_idx``). The Triton and MSA subclasses each own a full ``forward`` --
no shared forward code.
"""
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int | None = None,
kv_cache_dtype: str = "auto",
*,
topk_blocks: int,
sparse_block_size: int,
) -> None:
self.num_heads = num_heads
self.head_size = head_size
self.scale = scale
self.num_kv_heads = num_kv_heads if num_kv_heads is not None else num_heads
self.kv_cache_dtype = kv_cache_dtype
self.use_fp8_kv = is_quantized_kv_cache(kv_cache_dtype)
self.kv_cache_fp8_dtype = (
torch.float8_e5m2 if "e5m2" in kv_cache_dtype else torch.float8_e4m3fn
)
# Sparse selection parameters (block_size == page size == SPARSE_BLOCK_SIZE).
self.topk_blocks = topk_blocks
self.block_size = sparse_block_size
def forward(
self,
layer: AttentionLayer,
query: torch.Tensor,
kv_cache: torch.Tensor,
topk_idx: tuple[torch.Tensor | None, torch.Tensor | None],
output: torch.Tensor,
) -> torch.Tensor:
"""Attend the queries to the indexer-selected blocks. Per kernel."""
raise NotImplementedError
class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
"""Triton block-sparse attend (``minimax_m3_sparse_attn``) + Triton decode."""
def forward(
self,
layer: AttentionLayer,
query: torch.Tensor,
kv_cache: torch.Tensor,
topk_idx: tuple[torch.Tensor | None, torch.Tensor | None],
output: torch.Tensor,
) -> torch.Tensor:
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
return output # profiling run; caches unbound
main_md = attn_metadata[layer.layer_name] # type: ignore[attr-defined]
assert isinstance(main_md, MiniMaxM3SparseMetadata)
decode_topk, prefill_topk = topk_idx
nd = main_md.num_decode_tokens
num_tokens = main_md.num_actual_tokens
hd = self.head_size
q = query[:num_tokens].view(-1, self.num_heads, hd)
out = output[:num_tokens].view(-1, self.num_heads, hd)
kv_cache = (
kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache
)
# Decode [:nd]: split-K over the selected blocks (request-major chunks).
if main_md.num_decodes > 0:
d = main_md.decode
assert d is not None and decode_topk is not None
minimax_m3_sparse_attn_decode(
q[:nd],
kv_cache,
decode_topk,
d.block_table,
d.seq_lens,
self.num_kv_heads,
self.scale,
out[:nd],
d.decode_query_len,
)
# Prefill [nd:]: cu_seqlens_q already rebased to 0.
if main_md.num_prefills > 0:
p = main_md.prefill
assert p is not None and prefill_topk is not None
minimax_m3_sparse_attn(
q[nd:],
kv_cache,
prefill_topk,
p.block_table,
p.cu_seqlens_q,
p.seq_lens,
p.context_lens,
p.max_query_len,
self.num_kv_heads,
self.scale,
out[nd:],
)
return output
def select_main_impl_cls(
*,
topk_blocks: int,
kv_cache_dtype: str,
) -> type[MiniMaxM3SparseImpl]:
"""Pick the main attend impl off the main KV-cache dtype.
bf16 on Blackwell (SM100) uses the MSA attend; fp8 or non-Blackwell falls
back to Triton. The MSA module is imported lazily so AMD/non-SM100 never
import fmha_sm100.
"""
if (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and topk_blocks in (4, 8, 16, 32)
and not is_quantized_kv_cache(kv_cache_dtype)
):
from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
MiniMaxM3SparseMSAImpl,
)
return MiniMaxM3SparseMSAImpl
return MiniMaxM3SparseTritonImpl
@@ -0,0 +1,765 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
import numpy as np
import torch
import torch.nn as nn
from einops import rearrange
from transformers import PretrainedConfig
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.vision import (
get_vit_attn_backend,
is_vit_use_data_parallel,
)
from vllm.platforms import current_platform
# ROCm caps a kernel-launch gridDim.y at 65536. The HIP flash-attn Triton
# rotary kernel launches grid.y = cdiv(seqlen, BLOCK_M), so it fails with
# hipErrorInvalidValue once cdiv(seqlen, BLOCK_M) > 65536. Used below to decide
# when RoPE must be applied per video segment instead of in one launch.
_HIP_MAX_GRID_DIM_Y = 65536
class MiniMaxVLPatchEmbed(nn.Module):
"""Conv3d-based patch embedding.
Takes flat tokens of shape (N, C * temporal_patch_size * patch_size²)
and projects each to a hidden-size embedding.
"""
def __init__(self, config: PretrainedConfig) -> None:
super().__init__()
compression = config.img_token_compression_config
temporal_patch_size = compression.get("temporal_patch_size", 2)
patch_size = config.patch_size
num_channels = config.num_channels
self.patch_size = patch_size
self.temporal_patch_size = temporal_patch_size
self.num_channels = num_channels
self.hidden_size = config.hidden_size
self.patch_embedding = nn.Conv3d(
in_channels=num_channels,
out_channels=config.hidden_size,
kernel_size=(temporal_patch_size, patch_size, patch_size),
stride=(temporal_patch_size, patch_size, patch_size),
bias=False,
)
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
# pixel_values: (N, C * temporal_patch_size * patch_size²)
if self.patch_embedding.weight.dtype != pixel_values.dtype:
self.patch_embedding = self.patch_embedding.to(pixel_values.dtype)
x = pixel_values.reshape(
pixel_values.shape[0],
self.num_channels,
self.temporal_patch_size,
self.patch_size,
self.patch_size,
)
return self.patch_embedding(x).reshape(x.shape[0], -1)
class MiniMaxVLAttention(nn.Module):
"""Multi-head attention with MiniMax's partial 3D RoPE.
Partial means only the first ``rot_dim`` (< head_dim) dimensions of
Q and K are rotated; the remaining dims are passed through unchanged.
"""
def __init__(
self,
embed_dim: int,
num_heads: int,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
use_data_parallel = is_vit_use_data_parallel()
self.tp_size = (
1
if use_data_parallel
else parallel_state.get_tensor_model_parallel_world_size()
)
self.head_dim = embed_dim // num_heads
self.num_heads_per_partition = dist_utils.divide(num_heads, self.tp_size)
self.qkv_proj = QKVParallelLinear(
hidden_size=embed_dim,
head_size=self.head_dim,
total_num_heads=num_heads,
total_num_kv_heads=num_heads,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
disable_tp=use_data_parallel,
)
self.out_proj = RowParallelLinear(
input_size=embed_dim,
output_size=embed_dim,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.out_proj",
disable_tp=use_data_parallel,
)
self.attn = MMEncoderAttention(
num_heads=self.num_heads_per_partition,
head_size=self.head_dim,
prefix=f"{prefix}.attn",
)
# ApplyRotaryEmb handles the internal cos/sin repeat and partial
# rotation (ro_dim = half_rot_dim * 2 < head_dim for MiniMax).
# enable_fp32_compute=True runs the rotation in fp32 (q/k upcast,
# fp32 cos/sin), matching the reference ``_minimax_rope_applier``.
self.apply_rotary_emb = ApplyRotaryEmb(
enforce_enable=True, enable_fp32_compute=True
)
def _apply_rotary_emb(
self,
qk_reshaped: torch.Tensor,
rotary_cos: torch.Tensor,
rotary_sin: torch.Tensor,
seq_len: int,
rotary_segment_lengths: list[int] | None,
) -> torch.Tensor:
# Default fast path (all NVIDIA inputs, and ROCm short clips/images):
# a single rotary kernel launch. ``rotary_segment_lengths`` is only
# populated on ROCm (see ``MiniMaxVLVisionTransformer.forward``), so
# the per-segment path below is ROCm-only and never touches the
# NVIDIA/CUDA code path.
if not current_platform.is_rocm() or rotary_segment_lengths is None:
return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin)
# ROCm only: the HIP flash-attn Triton rotary kernel fails with
# hipErrorInvalidValue once grid.y = cdiv(seqlen, BLOCK_M) exceeds
# _HIP_MAX_GRID_DIM_Y (65536). BLOCK_M is 8 for rotary_dim <= 128
# (MiniMax-M3 vision: rotary_dim=78), giving a hard limit of
# 65536 * BLOCK_M tokens — measured exactly as 524288 OK / 524289 fail.
# Only long videos cross it; since vision_segment_max_frames caps each
# segment at a few frames (<< limit), applying RoPE per segment keeps
# every sub-call in range. Splitting on segment boundaries is
# mathematically exact because rotary_cos/sin are precomputed per token.
# Images and short clips stay on the single-kernel fast path above.
rotary_dim = rotary_cos.shape[-1] * 2
block_m = 8 if rotary_dim <= 128 else 4
hip_rotary_max_seqlen = _HIP_MAX_GRID_DIM_Y * block_m
if seq_len <= hip_rotary_max_seqlen or len(rotary_segment_lengths) <= 1:
return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin)
qk_segments = qk_reshaped.split(rotary_segment_lengths, dim=1)
cos_segments = rotary_cos.split(rotary_segment_lengths, dim=0)
sin_segments = rotary_sin.split(rotary_segment_lengths, dim=0)
return torch.cat(
[
self.apply_rotary_emb(qk_s, cos_s, sin_s)
for qk_s, cos_s, sin_s in zip(qk_segments, cos_segments, sin_segments)
],
dim=1,
)
def forward(
self,
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_cos: torch.Tensor,
rotary_sin: torch.Tensor,
max_seqlen: torch.Tensor,
rotary_segment_lengths: list[int] | None = None,
sequence_lengths: torch.Tensor | None = None,
) -> torch.Tensor:
# x: (N, 1, embed_dim) [seq=N, batch=1, chan=embed_dim]
x_qkv, _ = self.qkv_proj(x) # (N, 1, 3 * heads_per_part * head_dim)
seq_len, batch_size, _ = x_qkv.shape
# Rearrange to (b=1, N, 3, heads, head_dim) — same as Qwen2_5_VisionAttention
qkv = rearrange(
x_qkv,
"s b (three head d) -> b s three head d",
three=3,
head=self.num_heads_per_partition,
)
qk, v = qkv[:, :, :2], qkv[:, :, 2] # (b,N,2,h,d) and (b,N,h,d)
# Stack q/k → (2*b, N, heads, head_dim) for joint RoPE application.
# rotary_cos/sin: (N, half_rot_dim) — ApplyRotaryEmb expands internally
# and rotates only the first 2*half_rot_dim dims, passing the rest through.
qk_reshaped = rearrange(qk, "b s two h d -> (two b) s h d", two=2).contiguous()
qk_rotated = self._apply_rotary_emb(
qk_reshaped, rotary_cos, rotary_sin, seq_len, rotary_segment_lengths
)
qk_rotated = qk_rotated.view(
2, batch_size, seq_len, self.num_heads_per_partition, self.head_dim
)
q, k = qk_rotated.unbind(dim=0) # each (b=1, N, heads, head_dim)
# Flash attention → (b, N, heads, head_dim)
context = self.attn(
query=q,
key=k,
value=v,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
# Back to (N, 1, embed_dim)
context = rearrange(context, "b s h d -> s b (h d)", b=batch_size)
output, _ = self.out_proj(context)
return output
class MiniMaxVLEncoderLayer(nn.Module):
"""Single CLIP-style transformer block."""
def __init__(
self,
config: PretrainedConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
embed_dim = config.hidden_size
self.layer_norm1 = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.self_attn = MiniMaxVLAttention(
embed_dim=embed_dim,
num_heads=config.num_attention_heads,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
)
self.layer_norm2 = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
use_data_parallel = is_vit_use_data_parallel()
self.fc1 = ColumnParallelLinear(
config.hidden_size,
config.intermediate_size,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.fc1",
disable_tp=use_data_parallel,
)
self.act = get_act_fn(getattr(config, "hidden_act", "gelu"))
self.fc2 = RowParallelLinear(
config.intermediate_size,
config.hidden_size,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.fc2",
disable_tp=use_data_parallel,
)
def forward(
self,
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_cos: torch.Tensor,
rotary_sin: torch.Tensor,
max_seqlen: torch.Tensor,
rotary_segment_lengths: list[int] | None = None,
sequence_lengths: torch.Tensor | None = None,
) -> torch.Tensor:
# x: (N, 1, hidden_size)
x = x + self.self_attn(
self.layer_norm1(x),
cu_seqlens,
rotary_cos,
rotary_sin,
max_seqlen,
rotary_segment_lengths,
sequence_lengths,
)
residual = x
x, _ = self.fc1(self.layer_norm2(x))
x = self.act(x)
x, _ = self.fc2(x)
return residual + x
class MiniMaxVLEncoder(nn.Module):
def __init__(
self,
config: PretrainedConfig,
num_hidden_layers_override: int | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
n = (
config.num_hidden_layers
if num_hidden_layers_override is None
else num_hidden_layers_override
)
self.layers = nn.ModuleList(
[
MiniMaxVLEncoderLayer(
config=config,
quant_config=quant_config,
prefix=f"{prefix}.layers.{i}",
)
for i in range(n)
]
)
def forward(
self,
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_cos: torch.Tensor,
rotary_sin: torch.Tensor,
max_seqlen: torch.Tensor,
rotary_segment_lengths: list[int] | None = None,
sequence_lengths: torch.Tensor | None = None,
) -> torch.Tensor:
for layer in self.layers:
x = layer(
x,
cu_seqlens,
rotary_cos,
rotary_sin,
max_seqlen,
rotary_segment_lengths,
sequence_lengths,
)
return x
class MiniMaxVLVisionTransformer(nn.Module):
"""CLIP-based ViT with 3D RoPE (t/h/w decomposed).
Faithfully mirrors the reference ``MiniMaxVLVisionTransformer``.
FLASHINFER backend is not supported; standard flash-attn is used.
"""
def __init__(
self,
config: PretrainedConfig,
num_hidden_layers_override: int | None = None,
require_post_norm: bool | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
compression = config.img_token_compression_config
self.spatial_merge_size: int = compression.get("spatial_merge_size", 2)
self.temporal_patch_size: int = compression.get("temporal_patch_size", 2)
self.vision_segment_max_frames: int | None = getattr(
config, "vision_segment_max_frames", None
)
self.use_data_parallel = is_vit_use_data_parallel()
embed_dim = config.hidden_size
head_dim = embed_dim // config.num_attention_heads
# Backend selection + sharding info for building encoder metadata.
# Defaults to FLASH_ATTN on SM80+; --mm-encoder-attn-backend FLASHINFER
# selects the cuDNN ViT prefill path.
self.hidden_size = embed_dim
self.tp_size = (
1
if self.use_data_parallel
else parallel_state.get_tensor_model_parallel_world_size()
)
self.attn_backend = get_vit_attn_backend(
head_size=head_dim, dtype=torch.get_default_dtype()
)
rope_dims = 2 * (head_dim // 2)
# Split rope dims evenly across t/h/w (same formula as the reference)
self.t_dim = int(2 * ((rope_dims // 3) // 2))
self.h_dim = int(2 * ((rope_dims // 3) // 2))
self.w_dim = int(2 * ((rope_dims // 3) // 2))
# rot_dim = t_dim + h_dim + w_dim (may be < head_dim)
rope_theta: float = getattr(config, "rope_theta", 10000.0)
inv_freq_t = 1.0 / (
rope_theta
** (torch.arange(0, self.t_dim, 2, dtype=torch.float32) / self.t_dim)
)
inv_freq_h = 1.0 / (
rope_theta
** (torch.arange(0, self.h_dim, 2, dtype=torch.float32) / self.h_dim)
)
inv_freq_w = 1.0 / (
rope_theta
** (torch.arange(0, self.w_dim, 2, dtype=torch.float32) / self.w_dim)
)
self.register_buffer("inv_freq_t", inv_freq_t, persistent=False)
self.register_buffer("inv_freq_h", inv_freq_h, persistent=False)
self.register_buffer("inv_freq_w", inv_freq_w, persistent=False)
self.embeddings = MiniMaxVLPatchEmbed(config)
self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
n_layers = config.num_hidden_layers
if num_hidden_layers_override is None:
num_hidden_layers_override = n_layers
self.encoder = MiniMaxVLEncoder(
config=config,
num_hidden_layers_override=num_hidden_layers_override,
quant_config=quant_config,
prefix=f"{prefix}.encoder",
)
if require_post_norm is None:
require_post_norm = num_hidden_layers_override == n_layers
self.post_layernorm = (
nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
if require_post_norm
else None
)
# out_hidden_size needed by run_dp_sharded_mrope_vision_model
self.out_hidden_size = embed_dim
# ── RoPE helpers ─────────────────────────────────────────────────────
def _get_3d_rope_embed(
self, grid_t: int, grid_h: int, grid_w: int, spatial_merge_size: int
) -> torch.Tensor:
"""Compute 3D RoPE frequencies for a single (T, H, W) grid.
Returns (T*H*W, half_rot_dim) on the same device as inv_freq buffers.
Mirrors the reference ``_get_3d_rope_embed`` exactly.
"""
tokens_per_frame = grid_h * grid_w
tpos_ids = (
torch.arange(grid_t, device=self.inv_freq_t.device)
.unsqueeze(1)
.expand(-1, tokens_per_frame)
.flatten()
)
hpos_ids = (
torch.arange(grid_h, device=self.inv_freq_h.device)
.unsqueeze(1)
.expand(-1, grid_w)
.reshape(
grid_h // spatial_merge_size,
spatial_merge_size,
grid_w // spatial_merge_size,
spatial_merge_size,
)
.permute(0, 2, 1, 3)
.unsqueeze(0)
.expand(grid_t, -1, -1, -1, -1)
.flatten()
)
wpos_ids = (
torch.arange(grid_w, device=self.inv_freq_w.device)
.unsqueeze(0)
.expand(grid_h, -1)
.reshape(
grid_h // spatial_merge_size,
spatial_merge_size,
grid_w // spatial_merge_size,
spatial_merge_size,
)
.permute(0, 2, 1, 3)
.unsqueeze(0)
.expand(grid_t, -1, -1, -1, -1)
.flatten()
)
max_t = max(grid_t, 1)
max_hw = max(grid_h, grid_w)
seq_t = torch.arange(
max_t, device=self.inv_freq_t.device, dtype=self.inv_freq_t.dtype
)
seq_hw = torch.arange(
max_hw, device=self.inv_freq_h.device, dtype=self.inv_freq_h.dtype
)
freqs_t = torch.outer(seq_t, self.inv_freq_t) # (max_t, t_dim/2)
freqs_h = torch.outer(seq_hw, self.inv_freq_h) # (max_hw, h_dim/2)
freqs_w = torch.outer(seq_hw, self.inv_freq_w) # (max_hw, w_dim/2)
return torch.cat(
[freqs_t[tpos_ids], freqs_h[hpos_ids], freqs_w[wpos_ids]], dim=-1
) # (T*H*W, half_rot_dim)
def _get_rope_embed_3d(
self, grid_thw: list[list[int]], spatial_merge_size: int
) -> torch.Tensor:
embeds = [
self._get_3d_rope_embed(t, h, w, spatial_merge_size) for t, h, w in grid_thw
]
return torch.cat(embeds, dim=0) # (total_N, half_rot_dim)
# ── Frame-limit helper (mirrors the reference) ───────────────────────
def _apply_max_frames_limit(self, grid_thw: list[list[int]]) -> list[list[int]]:
if self.vision_segment_max_frames is None:
return grid_thw
max_f = self.vision_segment_max_frames
out: list[list[int]] = []
for t, h, w in grid_thw:
if t <= max_f:
out.append([t, h, w])
else:
for i in range(0, t, max_f):
out.append([min(max_f, t - i), h, w])
return out
# ── Forward ──────────────────────────────────────────────────────────
def forward(
self,
pixel_values: torch.Tensor,
grid_thw: list[list[int]],
) -> torch.Tensor:
# pixel_values: (total_N, C * temporal_patch_size * patch_size²)
# Output: (total_N, hidden_size)
hidden = self.embeddings(pixel_values) # (total_N, hidden_size)
hidden = self.pre_layrnorm(hidden)
limited = self._apply_max_frames_limit(grid_thw)
# Token-level cumulative sequence lengths (one segment per limited grid).
lens = [t * h * w for t, h, w in limited]
cu_seqlens_np = np.zeros(len(lens) + 1, dtype=np.int32)
np.cumsum(np.array(lens, dtype=np.int32), out=cu_seqlens_np[1:])
# Backend-specific encoder metadata. For FLASH_ATTN this returns the raw
# token cu_seqlens, the max segment length, and sequence_lengths=None;
# for FLASHINFER (cuDNN) it repacks cu_seqlens into element-offset
# indptrs, buckets max_seqlen, and builds padded per-sequence lengths.
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
self.attn_backend, cu_seqlens_np, hidden.device
)
max_seqlen = torch.tensor(
MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens_np),
dtype=torch.int32,
)
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
self.attn_backend,
cu_seqlens_np,
self.hidden_size,
self.tp_size,
hidden.device,
)
# 3D RoPE: (total_N, half_rot_dim); ApplyRotaryEmb expands internally
freqs = self._get_rope_embed_3d(limited, self.spatial_merge_size)
freqs = freqs.to(device=hidden.device)
# Keep cos/sin in fp32; ApplyRotaryEmb(enable_fp32_compute=True) runs the
# rotation in fp32 to match the reference precision.
rotary_cos, rotary_sin = freqs.cos(), freqs.sin()
# Encoder expects (N, 1, hidden_size) — add batch dim
hidden = hidden.unsqueeze(1)
# On ROCm, the flash_attn Triton rotary kernel can fail with
# hipErrorInvalidValue when seqlen is very large, e.g. 192k video
# tokens; pass per-segment lengths so RoPE can be applied in chunks.
# On other platforms leave it None -> single-kernel fast path, so the
# NVIDIA/CUDA code path is unchanged.
rotary_segment_lengths = lens if current_platform.is_rocm() else None
hidden = self.encoder(
hidden,
cu_seqlens,
rotary_cos,
rotary_sin,
max_seqlen,
rotary_segment_lengths,
sequence_lengths=sequence_lengths,
)
hidden = hidden.squeeze(1) # back to (total_N, hidden_size)
if self.post_layernorm is not None:
hidden = self.post_layernorm(hidden)
return hidden
class MiniMaxVLMultiModalProjector(nn.Module):
"""Two-layer MLP projector: vision_hidden → text_hidden."""
def __init__(
self,
vision_hidden_size: int,
text_hidden_size: int,
projector_hidden_size: int | None,
multimodal_projector_bias: bool,
projector_hidden_act: str = "gelu",
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
mid = projector_hidden_size if projector_hidden_size else text_hidden_size
use_dp = is_vit_use_data_parallel()
self.linear_1 = ColumnParallelLinear(
vision_hidden_size,
mid,
bias=multimodal_projector_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_1",
disable_tp=use_dp,
)
self.act = get_act_fn(projector_hidden_act)
self.linear_2 = RowParallelLinear(
mid,
text_hidden_size,
bias=multimodal_projector_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
disable_tp=use_dp,
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x, _ = self.linear_1(x)
x = self.act(x)
x, _ = self.linear_2(x)
return x
class MiniMaxVLPatchMerger(nn.Module):
def __init__(
self,
spatial_merge_size: int,
text_hidden_size: int,
projector_hidden_size: int | None,
patch_merge_bias: bool,
projector_hidden_act: str = "gelu",
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.spatial_merge_size = spatial_merge_size
mid = projector_hidden_size if projector_hidden_size else text_hidden_size
merge_in = text_hidden_size * spatial_merge_size**2
use_dp = is_vit_use_data_parallel()
self.linear_1 = ColumnParallelLinear(
merge_in,
mid,
bias=patch_merge_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_1",
disable_tp=use_dp,
)
self.act = get_act_fn(projector_hidden_act)
self.linear_2 = RowParallelLinear(
mid,
text_hidden_size,
bias=patch_merge_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
disable_tp=use_dp,
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
# x: (N, text_hidden_size) → (N // merge_size², text_hidden_size)
x = x.reshape(x.shape[0] // (self.spatial_merge_size**2), -1)
x, _ = self.linear_1(x)
x = self.act(x)
x, _ = self.linear_2(x)
return x
class MiniMaxVLVisionModel(nn.Module):
"""Full vision model: ViT → projector → patch merger."""
def __init__(
self,
config: PretrainedConfig,
text_hidden_size: int,
projector_hidden_size: int | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
compression = config.img_token_compression_config
spatial_merge_size: int = compression.get("spatial_merge_size", 2)
self.spatial_merge_size = spatial_merge_size
self.use_data_parallel = is_vit_use_data_parallel()
# The released checkpoint ships no ``post_layernorm`` weights and
# uses ``vision_feature_layer=-1`` with ``vision_feature_select_strategy
# ="full"``, i.e. the raw last encoder hidden state (CLIP's
# ``last_hidden_state`` is taken before the post layernorm). Applying an
# untrained post layernorm here would corrupt the visual features.
self.vision_model = MiniMaxVLVisionTransformer(
config=config,
require_post_norm=False,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "vision_model"),
)
self.multi_modal_projector = MiniMaxVLMultiModalProjector(
vision_hidden_size=config.hidden_size,
text_hidden_size=text_hidden_size,
projector_hidden_size=projector_hidden_size,
multimodal_projector_bias=getattr(
config, "multimodal_projector_bias", True
),
projector_hidden_act=getattr(config, "projector_hidden_act", "gelu"),
quant_config=quant_config,
prefix=maybe_prefix(prefix, "multi_modal_projector"),
)
self.patch_merge_mlp = MiniMaxVLPatchMerger(
spatial_merge_size=spatial_merge_size,
text_hidden_size=text_hidden_size,
projector_hidden_size=projector_hidden_size,
patch_merge_bias=getattr(config, "patch_merge_bias", True),
projector_hidden_act=getattr(config, "projector_hidden_act", "gelu"),
quant_config=quant_config,
prefix=maybe_prefix(prefix, "patch_merge_mlp"),
)
self.dtype = self.vision_model.embeddings.patch_embedding.weight.dtype
self.out_hidden_size = text_hidden_size
def forward(
self,
pixel_values: torch.Tensor,
grid_thw: list[list[int]],
) -> torch.Tensor:
hidden = self.vision_model(pixel_values=pixel_values, grid_thw=grid_thw)
if hidden.dim() == 3:
hidden = hidden.squeeze(0)
hidden = self.multi_modal_projector(hidden)
hidden = self.patch_merge_mlp(hidden)
return hidden
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj.", "q_proj.", "q"),
("qkv_proj.", "k_proj.", "k"),
("qkv_proj.", "v_proj.", "v"),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
File diff suppressed because it is too large Load Diff
+312
View File
@@ -0,0 +1,312 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
import regex as re
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.linear import (
ReplicatedLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from .model import (
MiniMAXGemmaRMSNorm,
MiniMaxM3DecoderLayer,
)
class MiniMaxM3MultiTokenPredictorLayer(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
quant_config = vllm_config.quant_config
self.enorm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = ReplicatedLinear(
config.hidden_size * 2,
config.hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.eh_proj",
)
self.transformer_layer = MiniMaxM3DecoderLayer(
vllm_config=vllm_config,
prefix=prefix,
force_sparse_attn=True,
force_moe=True,
is_mtp_block=True,
)
self.final_layernorm = MiniMAXGemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None
# Mask out inputs at position 0, as not needed by MTP.
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
# Combine the normalized token embeddings with the normalized
# previous hidden states.
inputs_embeds = self.enorm(inputs_embeds)
previous_hidden_states = self.hnorm(previous_hidden_states)
hidden_states, _ = self.eh_proj(
torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
)
# Apply transformer layer.
hidden_states, residual = self.transformer_layer(
positions=positions,
hidden_states=hidden_states,
residual=None,
)
hidden_states += residual
return hidden_states
class MiniMaxM3MultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
assert vllm_config.speculative_config is not None
# Use the draft (MTP) config, not the target model's. This is flat for a
# standalone checkpoint, and the promoted text_config for a bundled one.
config = vllm_config.speculative_config.draft_model_config.hf_config
self.num_mtp_layers = config.num_mtp_modules
self.layers = torch.nn.ModuleDict(
{
str(idx): MiniMaxM3MultiTokenPredictorLayer(
vllm_config, f"{prefix}.layers.{idx}"
)
for idx in range(self.num_mtp_layers)
}
)
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "embed_tokens"),
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
current_step_idx = spec_step_idx % self.num_mtp_layers
return self.layers[str(current_step_idx)](
input_ids,
positions,
previous_hidden_states,
inputs_embeds,
current_step_idx,
)
class MiniMaxM3MTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
self.quant_config = vllm_config.quant_config
self.model = MiniMaxM3MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.lm_head = ParallelLMHead(
self.config.vocab_size,
self.config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(self.config.vocab_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
return self.model(
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor | None:
current_step_idx = spec_step_idx % self.model.num_mtp_layers
mtp_layer = self.model.layers[str(current_step_idx)]
return self.logits_processor(
self.lm_head, mtp_layer.final_layernorm(hidden_states)
)
def _get_mtp_layer_idx_from_weight_name(self, name: str) -> int | None:
"""Return the MTP layer index in *.mtp.layers.{idx}.*, else None."""
match = re.search(r"\.mtp\.layers\.(\d+)\.", name)
return int(match.group(1)) if match else None
def _map_checkpoint_name(self, name: str) -> str | None:
"""Map a full checkpoint key to this MTP module's parameter name.
The MTP module only owns the *.mtp.layers.* weights plus the token
embedding and LM head, which the checkpoint shares with the main model.
Everything else belongs to other modules and is ignored here by returning
None.
"""
# In the bundled checkpoint, the MTP weights are prefixed with
# "language_model". The standalone MTP checkpoint has no such prefix.
# Strip it if present.
name = name.removeprefix("language_model.")
if name == "model.embed_tokens.weight":
return "model.embed_tokens.weight"
if name == "lm_head.weight":
return "lm_head.weight"
if "model.mtp.layers" in name:
if "weight_scale_inv" in name:
# The checkpoint stores block scales as "weight_scale_inv".
# The ModelOpt MXFP8 layers expose them as "weight_scale".
name = name.replace("weight_scale_inv", "weight_scale")
# Strip "mtp" from prefix.
return name.replace(".mtp.", ".")
return None
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Map q/k/v projections to qkv_proj, and gate/up projections to gate_up_proj.
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".qkv_proj", ".index_q_proj", "index_q"),
(".qkv_proj", ".index_k_proj", "index_k"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
# Map expert weights w1/w2/w3 to gate/down/up.
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_local_experts,
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
loaded_mtp_layers: set[int] = set()
for name, loaded_weight in weights:
mtp_layer = self._get_mtp_layer_idx_from_weight_name(name)
mapped_name = self._map_checkpoint_name(name)
if mapped_name is None:
# This weight does not belong to the MTP module, so skip it.
continue
name = mapped_name
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
# Routed experts (w1/w2/w3) are handled below. Don't let the
# stacked mapping rewrite them.
if ("block_sparse_moe.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for (
param_name,
weight_name,
expert_id,
expert_shard_id,
) in expert_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=expert_shard_id,
expert_id=expert_id,
)
break
else:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None or remapped_name not in params_dict:
continue
name = remapped_name
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
if mtp_layer is not None:
loaded_mtp_layers.add(mtp_layer)
# Validate that weights were loaded for each MTP layer.
for layer_idx in range(self.model.num_mtp_layers):
if layer_idx not in loaded_mtp_layers:
raise ValueError(
f"Failed to load MTP layer {layer_idx} weights from checkpoint."
)
return loaded_params
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MSA (SM100/Blackwell) block-sparse attend for MiniMax M3.
Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``);
decode falls back to the Triton split-K kernel (no MSA decode yet). ``fmha_sm100``
imports are function-local, so this module is import-safe on AMD/non-SM100.
"""
import torch
from vllm.forward_context import get_forward_context
from vllm.models.minimax_m3.common.ops.sparse_attn import (
SPARSE_BLOCK_SIZE,
minimax_m3_sparse_attn_decode,
)
from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseImpl,
MiniMaxM3SparseMetadata,
)
from vllm.v1.attention.backend import AttentionLayer
class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
"""MSA block-sparse attend (``fmha_sm100``); Triton split-K decode."""
def forward(
self,
layer: AttentionLayer,
query: torch.Tensor,
kv_cache: torch.Tensor,
topk_idx: tuple[torch.Tensor | None, torch.Tensor | None],
output: torch.Tensor,
) -> torch.Tensor:
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
return output # profiling run; caches unbound
main_md = attn_metadata[layer.layer_name] # type: ignore[attr-defined]
assert isinstance(main_md, MiniMaxM3SparseMetadata)
decode_topk, prefill_topk = topk_idx
nd = main_md.num_decode_tokens
num_tokens = main_md.num_actual_tokens
hd = self.head_size
q = query[:num_tokens].view(-1, self.num_heads, hd)
out = output[:num_tokens].view(-1, self.num_heads, hd)
kv_cache = (
kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache
)
# Decode [:nd]: Triton split-K placeholder (no MSA decode yet).
if main_md.num_decodes > 0:
d = main_md.decode
assert d is not None and decode_topk is not None
minimax_m3_sparse_attn_decode(
q[:nd],
kv_cache,
decode_topk,
d.block_table,
d.seq_lens,
self.num_kv_heads,
self.scale,
out[:nd],
d.decode_query_len,
)
# Prefill [nd:]: MSA sparse FMHA over the selected blocks.
if main_md.num_prefills > 0:
from vllm.third_party.fmha_sm100.sparse import (
build_k2q_csr,
sparse_atten_func,
)
p = main_md.prefill
assert p is not None and prefill_topk is not None
qp = q[nd:]
k_cache = kv_cache[:, 0].transpose(1, 2)
v_cache = kv_cache[:, 1].transpose(1, 2)
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
prefill_topk,
p.cu_seqlens_q,
p.cu_seqlens_k,
SPARSE_BLOCK_SIZE,
total_k=0,
max_seqlen_k=p.max_seq_len,
max_seqlen_q=p.max_query_len,
total_rows=p.total_kv_blocks,
qhead_per_kv=qp.shape[1] // self.num_kv_heads,
return_schedule=True,
)
sparse_atten_func(
qp,
k_cache,
v_cache,
k2q_row_ptr,
k2q_q_indices,
topK=self.topk_blocks,
blk_kv=SPARSE_BLOCK_SIZE,
causal=True,
softmax_scale=self.scale,
cu_seqlens_q=p.cu_seqlens_q,
cu_seqlens_k=p.cu_seqlens_k,
max_seqlen_q=p.max_query_len,
max_seqlen_k=p.max_seq_len,
page_table=p.block_table,
seqused_k=p.seq_lens,
schedule=schedule,
out=out[nd:],
)
return output