[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniMax M3 model — hardware-isolated entry point.
|
||||
|
||||
The implementation lives under ``nvidia/`` and ``amd/``; this module picks the
|
||||
right one for the current platform and re-exports the public classes used by
|
||||
the model registry. (Mirrors ``vllm.models.deepseek_v4``.)
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# The NVIDIA branch is the static default that type-checkers see; the ROCm
|
||||
# branch overrides it at runtime (kept type-compatible via type: ignore).
|
||||
if TYPE_CHECKING or not current_platform.is_rocm():
|
||||
from .nvidia.model import (
|
||||
MiniMaxM3SparseForCausalLM,
|
||||
MiniMaxM3SparseForConditionalGeneration,
|
||||
)
|
||||
from .nvidia.mtp import MiniMaxM3MTP
|
||||
else:
|
||||
from .amd.model import ( # type: ignore[assignment]
|
||||
MiniMaxM3SparseForCausalLM,
|
||||
MiniMaxM3SparseForConditionalGeneration,
|
||||
)
|
||||
from .amd.mtp import MiniMaxM3MTP # type: ignore[assignment]
|
||||
|
||||
__all__ = [
|
||||
"MiniMaxM3MTP",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
"MiniMaxM3SparseForConditionalGeneration",
|
||||
]
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,330 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniMax M3 MTP (multi-token prediction) draft model -- ROCm/AMD variant.
|
||||
|
||||
Byte-identical to ``nvidia/mtp.py`` except this file lives under ``amd/`` so its
|
||||
``from .model import ...`` resolves to the self-contained AMD model (native Gemma
|
||||
RMSNorm, native MXFP8 MoE, Triton sparse attention). The MTP logic is
|
||||
platform-agnostic. (Mirrors ``vllm.models.deepseek_v4.amd.mtp``.)
|
||||
|
||||
TODO(future, separate diff): since this is byte-identical to ``nvidia/mtp.py``,
|
||||
both copies could be consolidated into a single ``common/mtp.py`` that dispatches
|
||||
its model import (``..amd.model`` vs ``..nvidia.model``) via
|
||||
``current_platform.is_rocm()`` -- the same dispatch ``minimax_m3/__init__.py``
|
||||
uses. This was prototyped and VERIFIED working (``MiniMaxM3MTP`` resolves through
|
||||
``common.mtp`` to the AMD decoder layer / RMSNorm on ROCm), but it deletes the
|
||||
upstream ``nvidia/mtp.py`` and touches the NVIDIA load path, so it is deferred to
|
||||
a dedicated refactor diff to keep this AMD-enablement change NVIDIA-untouched.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .model import (
|
||||
MiniMAXGemmaRMSNorm,
|
||||
MiniMaxM3DecoderLayer,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.enorm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = ReplicatedLinear(
|
||||
config.hidden_size * 2,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.eh_proj",
|
||||
)
|
||||
self.transformer_layer = MiniMaxM3DecoderLayer(
|
||||
config=config,
|
||||
prefix=prefix,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
force_sparse_attn=True,
|
||||
force_moe=True,
|
||||
)
|
||||
self.final_layernorm = MiniMAXGemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
# Mask out inputs at position 0, as not needed by MTP.
|
||||
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
|
||||
|
||||
# Combine the normalized token embeddings with the normalized
|
||||
# previous hidden states.
|
||||
inputs_embeds = self.enorm(inputs_embeds)
|
||||
previous_hidden_states = self.hnorm(previous_hidden_states)
|
||||
hidden_states, _ = self.eh_proj(
|
||||
torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
|
||||
)
|
||||
|
||||
# Apply transformer layer.
|
||||
hidden_states, residual = self.transformer_layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
|
||||
hidden_states += residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class MiniMaxM3MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
# Use the draft (MTP) config, not the target model's. This is flat for a
|
||||
# standalone checkpoint, and the promoted text_config for a bundled one.
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.num_mtp_layers = config.num_mtp_modules
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): MiniMaxM3MultiTokenPredictorLayer(
|
||||
vllm_config, f"{prefix}.layers.{idx}"
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = MiniMaxM3MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.vocab_size,
|
||||
self.config.hidden_size,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
current_step_idx = spec_step_idx % self.model.num_mtp_layers
|
||||
mtp_layer = self.model.layers[str(current_step_idx)]
|
||||
return self.logits_processor(
|
||||
self.lm_head, mtp_layer.final_layernorm(hidden_states)
|
||||
)
|
||||
|
||||
def _get_mtp_layer_idx_from_weight_name(self, name: str) -> int | None:
|
||||
"""Return the MTP layer index in *.mtp.layers.{idx}.*, else None."""
|
||||
match = re.search(r"\.mtp\.layers\.(\d+)\.", name)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
def _map_checkpoint_name(self, name: str) -> str | None:
|
||||
"""Map a full checkpoint key to this MTP module's parameter name.
|
||||
|
||||
The MTP module only owns the *.mtp.layers.* weights plus the token
|
||||
embedding and LM head, which the checkpoint shares with the main model.
|
||||
Everything else belongs to other modules and is ignored here by returning
|
||||
None.
|
||||
"""
|
||||
# In the bundled checkpoint, the MTP weights are prefixed with
|
||||
# "language_model". The standalone MTP checkpoint has no such prefix.
|
||||
# Strip it if present.
|
||||
name = name.removeprefix("language_model.")
|
||||
|
||||
if name == "model.embed_tokens.weight":
|
||||
return "model.embed_tokens.weight"
|
||||
if name == "lm_head.weight":
|
||||
return "lm_head.weight"
|
||||
if "model.mtp.layers" in name:
|
||||
if "weight_scale_inv" in name:
|
||||
# The checkpoint stores block scales as "weight_scale_inv".
|
||||
# The ModelOpt MXFP8 layers expose them as "weight_scale".
|
||||
name = name.replace("weight_scale_inv", "weight_scale")
|
||||
# Strip "mtp" from prefix.
|
||||
return name.replace(".mtp.", ".")
|
||||
return None
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Map q/k/v projections to qkv_proj, and gate/up projections to gate_up_proj.
|
||||
stacked_params_mapping: list[tuple[str, str, int | str]] = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".qkv_proj", ".index_q_proj", "index_q"),
|
||||
(".qkv_proj", ".index_k_proj", "index_k"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
|
||||
# Map expert weights w1/w2/w3 to gate/down/up.
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_local_experts,
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
loaded_mtp_layers: set[int] = set()
|
||||
for name, loaded_weight in weights:
|
||||
mtp_layer = self._get_mtp_layer_idx_from_weight_name(name)
|
||||
mapped_name = self._map_checkpoint_name(name)
|
||||
if mapped_name is None:
|
||||
# This weight does not belong to the MTP module, so skip it.
|
||||
continue
|
||||
name = mapped_name
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
# Routed experts (w1/w2/w3) are handled below. Don't let the
|
||||
# stacked mapping rewrite them.
|
||||
if ("block_sparse_moe.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
expert_id,
|
||||
expert_shard_id,
|
||||
) in expert_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None or remapped_name not in params_dict:
|
||||
continue
|
||||
name = remapped_name
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
loaded_params.add(name)
|
||||
if mtp_layer is not None:
|
||||
loaded_mtp_layers.add(mtp_layer)
|
||||
|
||||
# Validate that weights were loaded for each MTP layer.
|
||||
for layer_idx in range(self.model.num_mtp_layers):
|
||||
if layer_idx not in loaded_mtp_layers:
|
||||
raise ValueError(
|
||||
f"Failed to load MTP layer {layer_idx} weights from checkpoint."
|
||||
)
|
||||
|
||||
return loaded_params
|
||||
@@ -0,0 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AMD/ROCm fused Triton ops for MiniMax-M3.
|
||||
|
||||
These replace per-element PyTorch fallbacks (FlashInfer / fused HIP kernels are
|
||||
unavailable on ROCm) with single-pass Triton kernels to cut launch overhead and
|
||||
intermediate-tensor traffic during decode.
|
||||
"""
|
||||
|
||||
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import (
|
||||
gemma_fused_add_rmsnorm,
|
||||
gemma_rmsnorm,
|
||||
)
|
||||
from vllm.models.minimax_m3.amd.ops.swiglu_oai import (
|
||||
swiglu_oai_quantize_mxfp8,
|
||||
swiglu_oai_split,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"gemma_rmsnorm",
|
||||
"gemma_fused_add_rmsnorm",
|
||||
"swiglu_oai_split",
|
||||
"swiglu_oai_quantize_mxfp8",
|
||||
]
|
||||
@@ -0,0 +1,155 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused Gemma-style RMSNorm for AMD ROCm via Triton.
|
||||
|
||||
Gemma RMSNorm = normalize(x) * (1 + weight), computed in fp32. FlashInfer's
|
||||
``gemma_rmsnorm`` / ``gemma_fused_add_rmsnorm`` CUDA kernels are unavailable on
|
||||
ROCm, so the AMD path previously used a ~8-op PyTorch sequence (float cast, add,
|
||||
pow, mean, rsqrt, two muls, cast) — each a separate kernel launch materializing
|
||||
fp32 intermediates. These kernels collapse that into a single pass per row.
|
||||
|
||||
Two entry points:
|
||||
* ``gemma_rmsnorm(x, w, eps)`` -> normalized tensor
|
||||
* ``gemma_fused_add_rmsnorm(x, res, w, eps)`` -> (normalized, x + res)
|
||||
|
||||
Both normalize over the last dim and broadcast ``weight`` (shape [N]) over it,
|
||||
so they serve both the full-hidden norms (input/post-attn/final) and the
|
||||
per-head q_norm/k_norm (N == head_dim). Inputs may be non-contiguous views
|
||||
(e.g. ``qkv.split`` slices); strides are passed through and outputs are written
|
||||
contiguous.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gemma_rmsnorm_kernel(
|
||||
x_ptr,
|
||||
w_ptr,
|
||||
out_ptr,
|
||||
n_cols,
|
||||
stride_row,
|
||||
stride_col,
|
||||
eps,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
cols = tl.arange(0, BLOCK_N)
|
||||
mask = cols < n_cols
|
||||
x = tl.load(x_ptr + row * stride_row + cols * stride_col, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
var = tl.sum(x * x, axis=0) / n_cols
|
||||
rstd = 1.0 / tl.sqrt(var + eps)
|
||||
w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
out = x * rstd * (1.0 + w)
|
||||
tl.store(
|
||||
out_ptr + row * n_cols + cols,
|
||||
out.to(out_ptr.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gemma_fused_add_rmsnorm_kernel(
|
||||
x_ptr,
|
||||
res_ptr,
|
||||
w_ptr,
|
||||
out_ptr,
|
||||
res_out_ptr,
|
||||
n_cols,
|
||||
stride_xrow,
|
||||
stride_xcol,
|
||||
stride_rrow,
|
||||
stride_rcol,
|
||||
eps,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
cols = tl.arange(0, BLOCK_N)
|
||||
mask = cols < n_cols
|
||||
x = tl.load(
|
||||
x_ptr + row * stride_xrow + cols * stride_xcol, mask=mask, other=0.0
|
||||
).to(tl.float32)
|
||||
r = tl.load(
|
||||
res_ptr + row * stride_rrow + cols * stride_rcol, mask=mask, other=0.0
|
||||
).to(tl.float32)
|
||||
s = x + r
|
||||
# residual_out is the pre-norm sum (consumed by the next layer's add).
|
||||
tl.store(
|
||||
res_out_ptr + row * n_cols + cols,
|
||||
s.to(res_out_ptr.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
var = tl.sum(s * s, axis=0) / n_cols
|
||||
rstd = 1.0 / tl.sqrt(var + eps)
|
||||
w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
out = s * rstd * (1.0 + w)
|
||||
tl.store(
|
||||
out_ptr + row * n_cols + cols,
|
||||
out.to(out_ptr.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
def _num_warps(block_n: int) -> int:
|
||||
if block_n >= 4096:
|
||||
return 16
|
||||
if block_n >= 1024:
|
||||
return 8
|
||||
return 4
|
||||
|
||||
|
||||
def gemma_rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
orig_shape = x.shape
|
||||
n = orig_shape[-1]
|
||||
x2 = x.reshape(-1, n)
|
||||
m = x2.shape[0]
|
||||
out = torch.empty((m, n), dtype=x.dtype, device=x.device)
|
||||
block_n = triton.next_power_of_2(n)
|
||||
_gemma_rmsnorm_kernel[(m,)](
|
||||
x2,
|
||||
weight,
|
||||
out,
|
||||
n,
|
||||
x2.stride(0),
|
||||
x2.stride(1),
|
||||
eps,
|
||||
BLOCK_N=block_n,
|
||||
num_warps=_num_warps(block_n),
|
||||
)
|
||||
return out.reshape(orig_shape)
|
||||
|
||||
|
||||
def gemma_fused_add_rmsnorm(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
orig_shape = x.shape
|
||||
n = orig_shape[-1]
|
||||
x2 = x.reshape(-1, n)
|
||||
r2 = residual.reshape(-1, n)
|
||||
m = x2.shape[0]
|
||||
out = torch.empty((m, n), dtype=x.dtype, device=x.device)
|
||||
res_out = torch.empty((m, n), dtype=x.dtype, device=x.device)
|
||||
block_n = triton.next_power_of_2(n)
|
||||
_gemma_fused_add_rmsnorm_kernel[(m,)](
|
||||
x2,
|
||||
r2,
|
||||
weight,
|
||||
out,
|
||||
res_out,
|
||||
n,
|
||||
x2.stride(0),
|
||||
x2.stride(1),
|
||||
r2.stride(0),
|
||||
r2.stride(1),
|
||||
eps,
|
||||
BLOCK_N=block_n,
|
||||
num_warps=_num_warps(block_n),
|
||||
)
|
||||
return out.reshape(orig_shape), res_out.reshape(orig_shape)
|
||||
@@ -0,0 +1,221 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused SwiGLU-OAI activation (split layout) for AMD ROCm via Triton.
|
||||
|
||||
SwiGLU-OAI on a ``[*, 2I]`` split-layout input (gate = first half, up = second
|
||||
half):
|
||||
|
||||
gate = clamp(gate, max=limit)
|
||||
up = clamp(up, -limit, +limit)
|
||||
out = gate * sigmoid(alpha * gate) * (up + beta)
|
||||
|
||||
On ROCm the dense MLP and the native MXFP8 MoE (between its two GEMMs) fell back
|
||||
to a chain of elementwise PyTorch ops with fp32 intermediates: vLLM's shared
|
||||
``SiluAndMulWithClamp`` blanket-routes ROCm to ``forward_native``, and the MoE
|
||||
applies the activation inline in PyTorch. This Triton kernel collapses that into
|
||||
a single pass producing the ``[*, I]`` output directly, and computes in fp32
|
||||
(rel ~1e-6 vs reference).
|
||||
|
||||
Note: the vectorized ``torch.ops._C.silu_and_mul_with_clamp`` op IS built on
|
||||
ROCm and is ~1.2-2.2x faster in isolation, but the win is launch overhead that
|
||||
HIP graphs already eliminate — measured end-to-end throughput is identical
|
||||
(within noise), so we keep the fp32-accurate Triton kernel.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _swiglu_oai_kernel(
|
||||
g_ptr,
|
||||
out_ptr,
|
||||
n_inter,
|
||||
stride_gm,
|
||||
stride_gn,
|
||||
stride_om,
|
||||
stride_on,
|
||||
alpha,
|
||||
beta,
|
||||
limit,
|
||||
HAS_LIMIT: tl.constexpr,
|
||||
BLOCK_I: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
pid_i = tl.program_id(1)
|
||||
cols = pid_i * BLOCK_I + tl.arange(0, BLOCK_I)
|
||||
mask = cols < n_inter
|
||||
gate = tl.load(g_ptr + row * stride_gm + cols * stride_gn, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
up = tl.load(
|
||||
g_ptr + row * stride_gm + (n_inter + cols) * stride_gn,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
if HAS_LIMIT:
|
||||
gate = tl.minimum(gate, limit)
|
||||
up = tl.minimum(tl.maximum(up, -limit), limit)
|
||||
out = gate * tl.sigmoid(alpha * gate) * (up + beta)
|
||||
tl.store(
|
||||
out_ptr + row * stride_om + cols * stride_on,
|
||||
out.to(out_ptr.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _swiglu_oai_quant_kernel(
|
||||
g_ptr,
|
||||
aq_ptr,
|
||||
as_ptr,
|
||||
M,
|
||||
n_inter,
|
||||
stride_gm,
|
||||
stride_gn,
|
||||
stride_qm,
|
||||
stride_qn,
|
||||
stride_sm,
|
||||
stride_sk,
|
||||
alpha,
|
||||
beta,
|
||||
limit,
|
||||
HAS_LIMIT: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
):
|
||||
"""SwiGLU-OAI (split layout) fused with per-32-block MXFP8 (E4M3 + E8M0)
|
||||
quant. Each program handles ``[BLOCK_M, 32]`` of the ``[M, I]`` output (one
|
||||
MX block): it reads the matching gate/up columns from ``g1`` (``[M, 2I]``),
|
||||
computes the SwiGLU in fp32, then derives the block E8M0 scale and emits the
|
||||
FP8 values + scale in a single pass — no bf16 ``act`` round-trip to HBM.
|
||||
"""
|
||||
pid_m = tl.program_id(0)
|
||||
pid_b = tl.program_id(1) # which 32-element block along I
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_c = pid_b * 32 + tl.arange(0, 32)
|
||||
m_mask = offs_m < M
|
||||
gate = tl.load(
|
||||
g_ptr + offs_m[:, None] * stride_gm + offs_c[None, :] * stride_gn,
|
||||
mask=m_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
up = tl.load(
|
||||
g_ptr + offs_m[:, None] * stride_gm + (n_inter + offs_c)[None, :] * stride_gn,
|
||||
mask=m_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
if HAS_LIMIT:
|
||||
gate = tl.minimum(gate, limit)
|
||||
up = tl.minimum(tl.maximum(up, -limit), limit)
|
||||
act = gate * tl.sigmoid(alpha * gate) * (up + beta) # [BLOCK_M, 32] fp32
|
||||
amax = tl.maximum(tl.max(tl.abs(act), axis=1), 1e-30) # [BLOCK_M]
|
||||
sb = tl.minimum(tl.maximum(tl.floor(tl.log2(amax)) + 127.0, 0.0), 254.0)
|
||||
descale = tl.exp2(sb - 127.0)
|
||||
aq = (act / descale[:, None]).to(aq_ptr.dtype.element_ty)
|
||||
tl.store(
|
||||
aq_ptr + offs_m[:, None] * stride_qm + offs_c[None, :] * stride_qn,
|
||||
aq,
|
||||
mask=m_mask[:, None],
|
||||
)
|
||||
tl.store(
|
||||
as_ptr + offs_m * stride_sm + pid_b * stride_sk, sb.to(tl.uint8), mask=m_mask
|
||||
)
|
||||
|
||||
|
||||
def swiglu_oai_quantize_mxfp8(
|
||||
gate_up: torch.Tensor,
|
||||
alpha: float,
|
||||
beta: float,
|
||||
limit: float | None,
|
||||
block_m: int = 64,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""SwiGLU-OAI on split-layout ``[M, 2I]`` fused with MXFP8 activation-quant.
|
||||
|
||||
Returns ``(act_q [M, I] float8_e4m3fn, act_scale [M, I//32] uint8 E8M0)``,
|
||||
identical to ``mxfp8_e4m3_quantize(swiglu_oai_split(gate_up))`` but in a
|
||||
single Triton pass (no bf16 intermediate). Used between the two GEMMs of the
|
||||
native MXFP8 MoE. Numerically equivalent to the unfused chain (bit-exact on
|
||||
measured MoE shapes); marginally more accurate (fp32 act, no bf16 round-trip).
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
|
||||
two_i = gate_up.shape[-1]
|
||||
n_inter = two_i // 2
|
||||
assert n_inter % MXFP8_BLOCK_SIZE == 0, (
|
||||
f"fused swiglu+quant needs I % {MXFP8_BLOCK_SIZE} == 0, got I={n_inter}"
|
||||
)
|
||||
g1 = gate_up.reshape(-1, two_i).contiguous()
|
||||
M = g1.shape[0]
|
||||
aq = torch.empty((M, n_inter), dtype=MXFP8_VALUE_DTYPE, device=g1.device)
|
||||
asc = torch.empty(
|
||||
(M, n_inter // MXFP8_BLOCK_SIZE), dtype=MXFP8_SCALE_DTYPE, device=g1.device
|
||||
)
|
||||
grid = (triton.cdiv(M, block_m), n_inter // MXFP8_BLOCK_SIZE)
|
||||
_swiglu_oai_quant_kernel[grid](
|
||||
g1,
|
||||
aq,
|
||||
asc,
|
||||
M,
|
||||
n_inter,
|
||||
g1.stride(0),
|
||||
g1.stride(1),
|
||||
aq.stride(0),
|
||||
aq.stride(1),
|
||||
asc.stride(0),
|
||||
asc.stride(1),
|
||||
float(alpha),
|
||||
float(beta),
|
||||
0.0 if limit is None else float(limit),
|
||||
HAS_LIMIT=limit is not None,
|
||||
BLOCK_M=block_m,
|
||||
num_warps=4,
|
||||
)
|
||||
return aq, asc
|
||||
|
||||
|
||||
def swiglu_oai_split(
|
||||
gate_up: torch.Tensor,
|
||||
alpha: float,
|
||||
beta: float,
|
||||
limit: float | None,
|
||||
out_dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""SwiGLU-OAI on a split-layout ``[*, 2I]`` tensor -> ``[*, I]``."""
|
||||
orig_shape = gate_up.shape
|
||||
two_i = orig_shape[-1]
|
||||
n_inter = two_i // 2
|
||||
x2 = gate_up.reshape(-1, two_i)
|
||||
m = x2.shape[0]
|
||||
dt = out_dtype if out_dtype is not None else gate_up.dtype
|
||||
out = torch.empty((m, n_inter), dtype=dt, device=gate_up.device)
|
||||
# Tile tuned on gfx950. The SwiGLU intermediate is sharded across tensor
|
||||
# parallel ranks (per-rank n_inter = I / tp: dense I=12288, MoE I=3072), and
|
||||
# a 512-wide tile (4 warps, ~2 elems/lane) only helps once the per-rank slice
|
||||
# is large enough to be bandwidth-bound — at TP=1 prefill that is ~1.25-1.35x
|
||||
# faster than 256. For small sharded slices (high TP) the kernel is launch-
|
||||
# bound (~12us) and a wide tile can slightly regress, so fall back to 256.
|
||||
# Decode is launch-bound at every TP. num_warps=8 underfills this tile, so it
|
||||
# is pinned to 4.
|
||||
block_i = 512 if n_inter >= 2048 else 256
|
||||
grid = (m, triton.cdiv(n_inter, block_i))
|
||||
_swiglu_oai_kernel[grid](
|
||||
x2,
|
||||
out,
|
||||
n_inter,
|
||||
x2.stride(0),
|
||||
x2.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
float(alpha),
|
||||
float(beta),
|
||||
0.0 if limit is None else float(limit),
|
||||
HAS_LIMIT=limit is not None,
|
||||
BLOCK_I=block_i,
|
||||
num_warps=4,
|
||||
)
|
||||
return out.reshape(*orig_shape[:-1], n_inter)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,512 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniMax M3 lightning indexer: side cache, metadata, and impl.
|
||||
|
||||
The indexer scores KV blocks with the index heads and selects the top-k blocks
|
||||
(plus fixed init/local blocks) that the main block-sparse attention
|
||||
(``sparse_attention.py``) then attends to. It owns its own side cache
|
||||
(``MiniMaxM3IndexerCache``, one index-key vector per token), metadata, and
|
||||
metadata builder, mirroring how DeepSeek V4 keeps the indexer separate from the
|
||||
main attention.
|
||||
|
||||
``MiniMaxM3Indexer`` is the ``nn.Module`` the attention layer holds (like
|
||||
``DeepseekV4Indexer``); it picks a kernel impl in ``__init__`` (via
|
||||
``select_indexer_impl_cls``) and delegates ``forward`` to it.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.attention import IndexerKVDType
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.models.minimax_m3.common.ops.index_topk import (
|
||||
minimax_m3_index_decode,
|
||||
minimax_m3_index_score,
|
||||
minimax_m3_index_topk,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionMetadata,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3IndexerBackend(AttentionBackend):
|
||||
"""Indexer side-cache backend (key-only)."""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16, torch.float16]
|
||||
# bf16 today; mirrors the main backend to keep spec validation permissive.
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
"fp8_e5m2",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "MINIMAX_M3_SPARSE_INDEXER"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["MiniMaxM3IndexerImpl"]:
|
||||
# Concrete impl chosen by select_indexer_impl_cls; base for introspection.
|
||||
return MiniMaxM3IndexerImpl
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["MiniMaxM3IndexerMetadataBuilder"]:
|
||||
return MiniMaxM3IndexerTritonMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [128]
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [128]
|
||||
|
||||
@classmethod
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
if include_num_layers_dimension:
|
||||
# M3 does not use cross-layer (per-layer-stacked) KV blocks.
|
||||
raise NotImplementedError
|
||||
return (0, 1, 2)
|
||||
|
||||
|
||||
class MiniMaxM3IndexerCache(nn.Module, AttentionLayerBase):
|
||||
"""Side KV cache for the indexer's per-token index keys (key-only).
|
||||
|
||||
Registers itself in the static forward context so the KV-cache manager
|
||||
allocates it (like ``DeepseekV32IndexerCache``).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_dim: int,
|
||||
prefix: str,
|
||||
cache_config: CacheConfig | None = None,
|
||||
indexer_kv_dtype: IndexerKVDType = "bf16",
|
||||
backend_cls: type[AttentionBackend] = MiniMaxM3IndexerBackend,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if indexer_kv_dtype != "bf16":
|
||||
raise NotImplementedError(
|
||||
f"indexer_kv_dtype={indexer_kv_dtype!r} is not supported yet "
|
||||
"for the MiniMax M3 indexer cache (only 'bf16')."
|
||||
)
|
||||
self.kv_cache = torch.tensor([])
|
||||
self.head_dim = head_dim
|
||||
self.indexer_kv_dtype = indexer_kv_dtype
|
||||
# Storage dtype for the side cache (bf16 today; quantized layouts later).
|
||||
self.dtype = torch.bfloat16
|
||||
self.prefix = prefix
|
||||
self.cache_config = cache_config
|
||||
# Impl-chosen backend -> each impl gets its own builder (get_attn_backend).
|
||||
self.backend_cls = backend_cls
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# Key-only: MLAAttentionSpec budgets one vector/token (not 2x for K+V).
|
||||
return MLAAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
|
||||
def forward(self) -> None: ...
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.backend_cls
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxM3IndexerPrefillMetadata:
|
||||
"""Per-prefill index-scoring state."""
|
||||
|
||||
cu_seqlens_q: torch.Tensor # [num_prefills + 1] int32, rebased to 0
|
||||
seq_lens: torch.Tensor # [num_prefills] int32, total KV lengths
|
||||
context_lens: torch.Tensor # [num_prefills] int32 (cached/context tokens)
|
||||
block_table: torch.Tensor
|
||||
max_query_len: int
|
||||
max_seq_len: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxM3IndexerDecodeMetadata:
|
||||
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
|
||||
per-request query length (1, or 1 + num_speculative_tokens)."""
|
||||
|
||||
seq_lens: torch.Tensor # [num_decodes] int32
|
||||
block_table: torch.Tensor
|
||||
max_seq_len: int
|
||||
decode_query_len: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxM3IndexerMetadata(AttentionMetadata):
|
||||
"""Indexer metadata, split into prefill and decode sub-metadata."""
|
||||
|
||||
seq_lens: torch.Tensor
|
||||
max_seq_len: int
|
||||
slot_mapping: torch.Tensor
|
||||
|
||||
num_actual_tokens: int # total query tokens (decode-first batch)
|
||||
|
||||
# Split counts; identical to the main metadata's (same reorder threshold).
|
||||
num_decodes: int
|
||||
num_decode_tokens: int
|
||||
num_prefills: int
|
||||
num_prefill_tokens: int
|
||||
|
||||
prefill: MiniMaxM3IndexerPrefillMetadata | None = None
|
||||
decode: MiniMaxM3IndexerDecodeMetadata | None = None
|
||||
|
||||
|
||||
class MiniMaxM3IndexerMetadataBuilder(
|
||||
AttentionMetadataBuilder[MiniMaxM3IndexerMetadata]
|
||||
):
|
||||
"""Abstract base: shared setup only. The Triton and MSA builders are
|
||||
parallel subclasses that each own their full ``build`` (no shared code)."""
|
||||
|
||||
# Full cudagraphs for uniform decode batches (incl. spec-decode verify).
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
# Raised to 1 + num_speculative_tokens by _init_reorder_batch_threshold when
|
||||
# spec decode is on; matches the main builder so the splits agree.
|
||||
reorder_batch_threshold: int = 1
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
text_config = getattr(hf_config, "text_config", hf_config)
|
||||
sparse_cfg = text_config.sparse_attention_config
|
||||
# Index-query head count from model config (cache spec has 1 vec/token).
|
||||
total_index_heads = sparse_cfg["sparse_num_index_heads"]
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if total_index_heads >= tp_size:
|
||||
assert total_index_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % total_index_heads == 0
|
||||
self.num_index_heads = max(1, total_index_heads // tp_size)
|
||||
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
||||
|
||||
# Stable context-length buffer for decode cudagraph replays.
|
||||
self.context_len_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3IndexerTritonMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
|
||||
"""Triton indexer metadata: no SM100 fmha_sm100 plan."""
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> MiniMaxM3IndexerMetadata:
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
query_start_loc = common_attn_metadata.query_start_loc
|
||||
seq_lens = common_attn_metadata.seq_lens
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
require_uniform=True,
|
||||
)
|
||||
)
|
||||
assert num_decodes + num_prefills == num_reqs
|
||||
assert num_decode_tokens + num_prefill_tokens == num_tokens
|
||||
|
||||
# Decode-first batch: context lengths into the stable cudagraph buffer.
|
||||
context_lens = self.context_len_buffer[:num_reqs]
|
||||
context_lens.copy_(
|
||||
common_attn_metadata.compute_num_computed_tokens(), non_blocking=True
|
||||
)
|
||||
|
||||
prefill_metadata: MiniMaxM3IndexerPrefillMetadata | None = None
|
||||
if num_prefills > 0:
|
||||
prefill_metadata = MiniMaxM3IndexerPrefillMetadata(
|
||||
cu_seqlens_q=(query_start_loc[num_decodes:] - num_decode_tokens).to(
|
||||
torch.int32
|
||||
),
|
||||
seq_lens=seq_lens[num_decodes:],
|
||||
context_lens=context_lens[num_decodes:],
|
||||
block_table=block_table[num_decodes:],
|
||||
max_query_len=common_attn_metadata.max_query_len,
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
)
|
||||
|
||||
decode_metadata: MiniMaxM3IndexerDecodeMetadata | None = None
|
||||
if num_decodes > 0:
|
||||
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
query_lens_cpu = qsl_cpu[1 : num_decodes + 1] - qsl_cpu[:num_decodes]
|
||||
decode_query_len = int(query_lens_cpu[0].item())
|
||||
assert decode_query_len > 0
|
||||
assert torch.all(
|
||||
(query_lens_cpu == decode_query_len) | (query_lens_cpu == 0)
|
||||
)
|
||||
assert num_decode_tokens == num_decodes * decode_query_len
|
||||
decode_metadata = MiniMaxM3IndexerDecodeMetadata(
|
||||
seq_lens=seq_lens[:num_decodes],
|
||||
block_table=block_table[:num_decodes],
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
decode_query_len=decode_query_len,
|
||||
)
|
||||
|
||||
return MiniMaxM3IndexerMetadata(
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
slot_mapping=common_attn_metadata.slot_mapping,
|
||||
num_actual_tokens=num_tokens,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefills=num_prefills,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
prefill=prefill_metadata,
|
||||
decode=decode_metadata,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3IndexerImpl(nn.Module):
|
||||
"""Abstract base for the indexer kernel impls.
|
||||
|
||||
Each impl owns its side cache and reports its backend via
|
||||
``indexer_backend_cls`` (so each gets its own builder). The Triton and MSA
|
||||
subclasses each own a full ``forward`` returning ``(decode_topk,
|
||||
prefill_topk)`` -- no shared forward code.
|
||||
"""
|
||||
|
||||
# Set by each impl so the side cache reports the matching backend + builder.
|
||||
indexer_backend_cls: ClassVar[type[AttentionBackend]] = MiniMaxM3IndexerBackend
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
num_kv_heads: int,
|
||||
scale: float,
|
||||
topk_blocks: int,
|
||||
sparse_block_size: int,
|
||||
num_index_heads: int,
|
||||
index_head_dim: int,
|
||||
prefix: str,
|
||||
init_blocks: int = 0,
|
||||
local_blocks: int = 0,
|
||||
score_type: str = "max",
|
||||
cache_config: CacheConfig | None = None,
|
||||
indexer_kv_dtype: IndexerKVDType = "bf16",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.scale = scale
|
||||
self.topk_blocks = topk_blocks
|
||||
self.block_size = sparse_block_size
|
||||
self.init_blocks = init_blocks
|
||||
self.local_blocks = local_blocks
|
||||
self.score_type = score_type
|
||||
self.num_index_heads = num_index_heads
|
||||
self.index_head_dim = index_head_dim
|
||||
self.indexer_kv_dtype = indexer_kv_dtype
|
||||
# Owns the side cache (registers itself in the static forward context).
|
||||
self.index_cache = MiniMaxM3IndexerCache(
|
||||
head_dim=index_head_dim,
|
||||
prefix=f"{prefix}.index_cache",
|
||||
cache_config=cache_config,
|
||||
indexer_kv_dtype=indexer_kv_dtype,
|
||||
backend_cls=type(self).indexer_backend_cls,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
index_query: torch.Tensor,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
"""Return ``(decode_topk, prefill_topk)``; implemented per kernel impl."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MiniMaxM3IndexerTritonImpl(MiniMaxM3IndexerImpl):
|
||||
"""Triton indexer score + top-k for both prefill and decode."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
index_query: torch.Tensor,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return None, None # profiling run; caches unbound
|
||||
index_md = attn_metadata[self.index_cache.prefix]
|
||||
assert isinstance(index_md, MiniMaxM3IndexerMetadata)
|
||||
num_tokens = index_md.num_actual_tokens
|
||||
nd = index_md.num_decode_tokens
|
||||
iq = index_query[:num_tokens].view(
|
||||
-1, self.num_index_heads, self.index_head_dim
|
||||
)
|
||||
kv = self.index_cache.kv_cache
|
||||
|
||||
decode_topk: torch.Tensor | None = None
|
||||
prefill_topk: torch.Tensor | None = None
|
||||
if index_md.num_decodes > 0:
|
||||
d = index_md.decode
|
||||
assert d is not None
|
||||
decode_topk = minimax_m3_index_decode(
|
||||
iq[:nd],
|
||||
kv,
|
||||
d.block_table,
|
||||
d.seq_lens,
|
||||
d.max_seq_len,
|
||||
self.topk_blocks,
|
||||
self.init_blocks,
|
||||
self.local_blocks,
|
||||
self.num_kv_heads,
|
||||
self.scale,
|
||||
d.decode_query_len,
|
||||
)
|
||||
if index_md.num_prefills > 0:
|
||||
p = index_md.prefill
|
||||
assert p is not None
|
||||
score = minimax_m3_index_score(
|
||||
iq[nd:],
|
||||
kv,
|
||||
p.block_table,
|
||||
p.cu_seqlens_q,
|
||||
p.seq_lens,
|
||||
p.context_lens,
|
||||
p.max_query_len,
|
||||
p.max_seq_len,
|
||||
self.num_kv_heads,
|
||||
self.scale,
|
||||
)
|
||||
prefill_topk = minimax_m3_index_topk(
|
||||
score,
|
||||
p.cu_seqlens_q,
|
||||
p.context_lens,
|
||||
p.max_query_len,
|
||||
self.topk_blocks,
|
||||
self.init_blocks,
|
||||
self.local_blocks,
|
||||
)
|
||||
return decode_topk, prefill_topk
|
||||
|
||||
|
||||
def select_indexer_impl_cls(
|
||||
*,
|
||||
indexer_kv_dtype: IndexerKVDType = "bf16",
|
||||
) -> type[MiniMaxM3IndexerImpl]:
|
||||
"""Pick the indexer impl off the index-cache dtype.
|
||||
|
||||
The SM100 MSA indexer score path is disabled for now; use the local Triton
|
||||
indexer. If re-enabled, add a NVIDIA-specific ``MiniMaxM3IndexerImpl`` here.
|
||||
"""
|
||||
if indexer_kv_dtype in ("mxfp4", "nvfp4"):
|
||||
raise NotImplementedError(
|
||||
f"indexer_kv_dtype={indexer_kv_dtype!r} needs the (not-yet-added) "
|
||||
"CuteDSL indexer impl."
|
||||
)
|
||||
if indexer_kv_dtype != "bf16":
|
||||
raise NotImplementedError(
|
||||
f"indexer_kv_dtype={indexer_kv_dtype!r} is not supported by the "
|
||||
"Triton indexer impl."
|
||||
)
|
||||
return MiniMaxM3IndexerTritonImpl
|
||||
|
||||
|
||||
class MiniMaxM3Indexer(nn.Module):
|
||||
"""Indexer module held by the attention layer (like ``DeepseekV4Indexer``).
|
||||
|
||||
Picks the kernel impl in ``__init__`` (``select_indexer_impl_cls``) and
|
||||
delegates ``forward``; exposes the impl's side cache via ``index_cache``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
num_kv_heads: int,
|
||||
scale: float,
|
||||
topk_blocks: int,
|
||||
sparse_block_size: int,
|
||||
num_index_heads: int,
|
||||
index_head_dim: int,
|
||||
prefix: str,
|
||||
init_blocks: int = 0,
|
||||
local_blocks: int = 0,
|
||||
score_type: str = "max",
|
||||
cache_config: CacheConfig | None = None,
|
||||
indexer_kv_dtype: IndexerKVDType = "bf16",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
impl_cls = select_indexer_impl_cls(
|
||||
indexer_kv_dtype=indexer_kv_dtype,
|
||||
)
|
||||
self.impl = impl_cls(
|
||||
num_kv_heads=num_kv_heads,
|
||||
scale=scale,
|
||||
topk_blocks=topk_blocks,
|
||||
sparse_block_size=sparse_block_size,
|
||||
num_index_heads=num_index_heads,
|
||||
index_head_dim=index_head_dim,
|
||||
prefix=prefix,
|
||||
init_blocks=init_blocks,
|
||||
local_blocks=local_blocks,
|
||||
score_type=score_type,
|
||||
cache_config=cache_config,
|
||||
indexer_kv_dtype=indexer_kv_dtype,
|
||||
)
|
||||
|
||||
@property
|
||||
def index_cache(self) -> MiniMaxM3IndexerCache:
|
||||
return self.impl.index_cache
|
||||
|
||||
@property
|
||||
def num_index_heads(self) -> int:
|
||||
return self.impl.num_index_heads
|
||||
|
||||
def forward(
|
||||
self,
|
||||
index_query: torch.Tensor,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
return self.impl(index_query)
|
||||
@@ -0,0 +1,514 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
from transformers import BatchFeature
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.config.multimodal import (
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
ImageSize,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.video import (
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
VideoBackend,
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
)
|
||||
from vllm.transformers_utils.configs.minimax_m3 import MiniMaxM3Config
|
||||
from vllm.transformers_utils.processors.minimax_m3 import (
|
||||
MIN_SHORT_SIDE_PIXEL,
|
||||
MiniMaxM3VLImageProcessor,
|
||||
MiniMaxM3VLVideoProcessor,
|
||||
MiniMaxVLProcessor,
|
||||
smart_resize,
|
||||
)
|
||||
|
||||
# Upper bound on the number of frames used to build the dummy video during
|
||||
# memory profiling. Sized to the worst-case video the processor accepts:
|
||||
# ``max_total_pixels // max_pixels_per_frame`` = 301,056,000 // 602,112 = 500
|
||||
# frames, each at the video processor's per-frame ``max_pixels`` (768 * 28 * 28
|
||||
# = 602,112). This reaches the true worst-case ~192,000 vision tokens, but only
|
||||
# because the dummy video is sized via ``get_video_size_with_most_features()``
|
||||
# (the video ``max_pixels`` bound), not the smaller image bound. Without a cap,
|
||||
# ``_get_max_video_frames(seq_len)`` with M3's large ``max_model_len`` yields
|
||||
# ~1400 frames, producing a multi-GB dummy tensor that overflows the
|
||||
# multimodal encoder cache.
|
||||
_MAX_FRAMES_PER_VIDEO = 500
|
||||
|
||||
|
||||
class MiniMaxM3VLProcessingInfo(BaseProcessingInfo):
|
||||
IMAGE_TOKEN = "]<]image[>["
|
||||
VIDEO_TOKEN = "]<]video[>["
|
||||
VISION_START_TOKEN = "]<]start of image[>["
|
||||
VISION_END_TOKEN = "]<]end of image[>["
|
||||
|
||||
def get_hf_config(self) -> MiniMaxM3Config:
|
||||
return self.ctx.get_hf_config(MiniMaxM3Config)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> MiniMaxVLProcessor:
|
||||
# The released checkpoint only ships the processor as remote code
|
||||
# (via ``auto_map``). Construct the vendored processor directly so the
|
||||
# model loads without ``--trust-remote-code``.
|
||||
return self.ctx.get_hf_processor(MiniMaxVLProcessor, **kwargs)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None, "video": None}
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> Mapping[str, int]:
|
||||
return {
|
||||
"image": self.get_max_image_tokens(),
|
||||
"video": self.get_max_video_tokens(seq_len, mm_counts),
|
||||
}
|
||||
|
||||
def get_image_processor(self, **kwargs: object) -> MiniMaxM3VLImageProcessor:
|
||||
return self.get_hf_processor(**kwargs).image_processor
|
||||
|
||||
def get_video_processor(self, **kwargs: object) -> MiniMaxM3VLVideoProcessor:
|
||||
return self.get_hf_processor(**kwargs).video_processor
|
||||
|
||||
def _get_vision_info(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
num_frames: int,
|
||||
image_processor,
|
||||
) -> tuple[ImageSize, int]:
|
||||
"""Compute resized image size and number of vision tokens.
|
||||
|
||||
Mirrors the processor's Qwen-style ``smart_resize`` (area bound by
|
||||
``max_pixels``) so token counts match the actual processor output.
|
||||
"""
|
||||
patch_size: int = image_processor.patch_size
|
||||
merge_size: int = image_processor.merge_size
|
||||
temporal_patch_size: int = image_processor.temporal_patch_size
|
||||
factor = patch_size * merge_size
|
||||
max_pixels: int = image_processor.max_pixels
|
||||
# Long-side resize spec (opt-in). ``image_processor`` is the *video*
|
||||
# processor when counting video tokens, so read the bounds off it.
|
||||
max_long_side_pixel = getattr(image_processor, "max_long_side_pixel", None)
|
||||
min_short_side_pixel = getattr(
|
||||
image_processor, "min_short_side_pixel", MIN_SHORT_SIDE_PIXEL
|
||||
)
|
||||
|
||||
new_h, new_w = smart_resize(
|
||||
image_height,
|
||||
image_width,
|
||||
factor=factor,
|
||||
max_pixels=max_pixels,
|
||||
max_long_side_pixel=max_long_side_pixel,
|
||||
min_short_side_pixel=min_short_side_pixel,
|
||||
# Token counting must not raise; the volumetric/area cap is enforced
|
||||
# in the processor's _preprocess on the real inputs.
|
||||
max_total_pixels=None,
|
||||
)
|
||||
grid_h = new_h // patch_size
|
||||
grid_w = new_w // patch_size
|
||||
|
||||
# Pad frames to be divisible by temporal_patch_size
|
||||
padded_frames = num_frames + (-num_frames % temporal_patch_size)
|
||||
grid_t = max(padded_frames // temporal_patch_size, 1)
|
||||
|
||||
num_tokens = grid_t * grid_h * grid_w // (merge_size**2)
|
||||
return ImageSize(width=new_w, height=new_h), num_tokens
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
image_processor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> int:
|
||||
_, n = self._get_vision_info(
|
||||
image_width=image_width,
|
||||
image_height=image_height,
|
||||
num_frames=1,
|
||||
image_processor=image_processor,
|
||||
)
|
||||
return n
|
||||
|
||||
def get_num_video_tokens(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
num_frames: int,
|
||||
image_processor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> int:
|
||||
_, n = self._get_vision_info(
|
||||
image_width=image_width,
|
||||
image_height=image_height,
|
||||
num_frames=num_frames,
|
||||
image_processor=image_processor,
|
||||
)
|
||||
return n
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
# Largest square (a multiple of patch_size*merge_size) whose area is
|
||||
# within the image processor's bound — this yields the most vision
|
||||
# tokens for one image. With the long-side spec the square side is
|
||||
# capped by ``max_long_side_pixel`` (and the fixed ``max_total_pixels``);
|
||||
# otherwise it is bound by the ``max_pixels`` area.
|
||||
image_processor = self.get_image_processor()
|
||||
factor = image_processor.patch_size * image_processor.merge_size
|
||||
max_long_side_pixel = getattr(image_processor, "max_long_side_pixel", None)
|
||||
if max_long_side_pixel is not None:
|
||||
side_px = min(
|
||||
max_long_side_pixel,
|
||||
math.isqrt(image_processor.max_total_pixels),
|
||||
)
|
||||
else:
|
||||
side_px = math.isqrt(image_processor.max_pixels)
|
||||
side = max(factor, (side_px // factor) * factor)
|
||||
return ImageSize(width=side, height=side)
|
||||
|
||||
def get_video_size_with_most_features(self) -> ImageSize:
|
||||
# Per-frame size that yields the most vision tokens, bound by the
|
||||
# *video* processor's ``max_pixels`` (which differs from the image
|
||||
# bound). Token count depends only on area, so maximize the area
|
||||
# achievable with both sides a multiple of patch_size*merge_size rather
|
||||
# than picking the largest square — a square (e.g. 756x756 for M3's
|
||||
# 602,112 bound) leaves area on the table, undercounting frames.
|
||||
video_processor = self.get_video_processor()
|
||||
factor = video_processor.patch_size * video_processor.merge_size
|
||||
per_frame_pixels = video_processor.max_pixels
|
||||
max_long_side_pixel = getattr(video_processor, "max_long_side_pixel", None)
|
||||
if max_long_side_pixel is not None:
|
||||
# Long-side spec: a frame's worst case is a square capped by
|
||||
# ``max_long_side_pixel`` (per-frame area, not the volumetric cap).
|
||||
per_frame_pixels = min(per_frame_pixels, max_long_side_pixel**2)
|
||||
units = per_frame_pixels // (factor * factor) # h_u * w_u
|
||||
h_u = math.isqrt(units)
|
||||
while units % h_u:
|
||||
h_u -= 1
|
||||
return ImageSize(width=(units // h_u) * factor, height=h_u * factor)
|
||||
|
||||
def get_max_image_tokens(self) -> int:
|
||||
image_processor = self.get_image_processor()
|
||||
size = self.get_image_size_with_most_features()
|
||||
return self.get_num_image_tokens(
|
||||
image_width=size.width,
|
||||
image_height=size.height,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
)
|
||||
|
||||
def _get_max_video_frames(self, max_tokens: int) -> int:
|
||||
video_processor = self.get_video_processor()
|
||||
size = self.get_video_size_with_most_features()
|
||||
num_frames = 1
|
||||
while True:
|
||||
next_n = self.get_num_video_tokens(
|
||||
image_width=size.width,
|
||||
image_height=size.height,
|
||||
num_frames=num_frames + 1,
|
||||
image_processor=video_processor,
|
||||
mm_kwargs={},
|
||||
)
|
||||
if next_n > max_tokens:
|
||||
break
|
||||
num_frames += 1
|
||||
return num_frames
|
||||
|
||||
def get_num_frames_with_most_features(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
max_frames_per_video: int = _MAX_FRAMES_PER_VIDEO,
|
||||
) -> int:
|
||||
max_videos = mm_counts.get("video", 0)
|
||||
max_total_frames = self._get_max_video_frames(seq_len)
|
||||
max_frames_per_video = min(
|
||||
max_total_frames // max(max_videos, 1), max_frames_per_video
|
||||
)
|
||||
return max(max_frames_per_video, 1)
|
||||
|
||||
def get_max_video_tokens(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> int:
|
||||
video_processor = self.get_video_processor()
|
||||
size = self.get_video_size_with_most_features()
|
||||
return self.get_num_video_tokens(
|
||||
image_width=size.width,
|
||||
image_height=size.height,
|
||||
num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
|
||||
image_processor=video_processor,
|
||||
mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
image_token: str = self.info.IMAGE_TOKEN
|
||||
video_token: str = self.info.VIDEO_TOKEN
|
||||
return image_token * num_images + video_token * num_videos
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
) -> MultiModalDataDict:
|
||||
size = self.info.get_image_size_with_most_features()
|
||||
video_size = self.info.get_video_size_with_most_features()
|
||||
num_frames = self.info.get_num_frames_with_most_features(seq_len, mm_counts)
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
width=size.width,
|
||||
height=size.height,
|
||||
num_images=mm_counts.get("image", 0),
|
||||
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
|
||||
),
|
||||
"video": self._get_dummy_videos(
|
||||
width=video_size.width,
|
||||
height=video_size.height,
|
||||
num_frames=num_frames,
|
||||
num_videos=mm_counts.get("video", 0),
|
||||
overrides=cast(VideoDummyOptions | None, mm_options.get("video")),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class MiniMaxM3VLMultiModalProcessor(
|
||||
BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]
|
||||
):
|
||||
def _get_data_parser(self) -> MultiModalDataParser:
|
||||
# Request video metadata (fps + sampled frame indices) so the HF
|
||||
# processor can emit per-frame ``]<]X.X seconds[>[`` timestamp markers,
|
||||
# matching MiniMax's reference video token stream. ``_get_prompt_updates``
|
||||
# reconstructs the same markers from the metadata to keep the prompt
|
||||
# replacement aligned with the processor output.
|
||||
return MultiModalDataParser(video_needs_metadata=True)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
# With ``video_needs_metadata=True`` each video arrives as a
|
||||
# ``(frames, metadata)`` tuple. Split the frames back out and forward the
|
||||
# metadata as ``VideoMetadata`` so the processor emits timestamps.
|
||||
videos = cast(list | None, mm_data.get("videos"))
|
||||
video_metadata: list[VideoMetadata] | None = None
|
||||
if videos:
|
||||
frames_only = []
|
||||
video_metadata = []
|
||||
for item in videos:
|
||||
if isinstance(item, tuple) and len(item) == 2:
|
||||
frames, meta = item
|
||||
else:
|
||||
frames, meta = item, {}
|
||||
frames_only.append(frames)
|
||||
meta = {
|
||||
k: v for k, v in (meta or {}).items() if k != "do_sample_frames"
|
||||
}
|
||||
# VideoMetadata requires total_num_frames; derive it for
|
||||
# dummy/profiling videos whose metadata omits it. fps and
|
||||
# frames_indices default to None there → no timestamps, which
|
||||
# stays consistent with _get_prompt_updates.
|
||||
meta.setdefault("total_num_frames", len(frames))
|
||||
video_metadata.append(VideoMetadata(**meta))
|
||||
mm_data["videos"] = frames_only
|
||||
|
||||
# Override the video processor's default do_resize=False (set for a
|
||||
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
|
||||
merged = dict(do_resize=True, **mm_kwargs, **tok_kwargs)
|
||||
data = dict(text=prompt, **mm_data)
|
||||
if video_metadata is not None:
|
||||
data["video_metadata"] = video_metadata
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
data,
|
||||
merged,
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
image_grid_thw = hf_inputs.get("image_grid_thw")
|
||||
video_grid_thw = hf_inputs.get("video_grid_thw")
|
||||
|
||||
# Total patches per item (grid_t * grid_h * grid_w)
|
||||
image_grid_sizes = (
|
||||
image_grid_thw.prod(-1)
|
||||
if image_grid_thw is not None
|
||||
else torch.empty(0, dtype=torch.long)
|
||||
)
|
||||
video_grid_sizes = (
|
||||
video_grid_thw.prod(-1)
|
||||
if video_grid_thw is not None
|
||||
else torch.empty(0, dtype=torch.long)
|
||||
)
|
||||
|
||||
return {
|
||||
"pixel_values": MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", image_grid_sizes
|
||||
),
|
||||
"image_grid_thw": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
"pixel_values_videos": MultiModalFieldConfig.flat_from_sizes(
|
||||
"video", video_grid_sizes
|
||||
),
|
||||
"video_grid_thw": MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||
}
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
image_token_id: int = vocab[self.info.IMAGE_TOKEN]
|
||||
video_token_id: int = vocab[self.info.VIDEO_TOKEN]
|
||||
start_token_id: int = vocab[self.info.VISION_START_TOKEN]
|
||||
end_token_id: int = vocab[self.info.VISION_END_TOKEN]
|
||||
merge_length: int = hf_processor.image_processor.merge_size**2
|
||||
|
||||
def get_image_replacement(item_idx: int):
|
||||
grid_thw: torch.Tensor = out_mm_kwargs["image"][item_idx][
|
||||
"image_grid_thw"
|
||||
].data
|
||||
# grid_thw shape: (3,) = [1, grid_h, grid_w]
|
||||
N = int(grid_thw.prod().item()) // merge_length
|
||||
full = [start_token_id] + [image_token_id] * N + [end_token_id]
|
||||
return PromptUpdateDetails.select_token_id(full, image_token_id)
|
||||
|
||||
# Per-video metadata (fps + sampled frame indices) is carried on the
|
||||
# parsed video items; used to reproduce the HF processor's timestamps.
|
||||
video_items = mm_items.get("video")
|
||||
video_metadata = getattr(video_items, "metadata", None)
|
||||
temporal_patch_size: int = hf_processor.video_processor.temporal_patch_size
|
||||
|
||||
def get_video_replacement(item_idx: int):
|
||||
grid_thw: torch.Tensor = out_mm_kwargs["video"][item_idx][
|
||||
"video_grid_thw"
|
||||
].data
|
||||
# grid_thw shape: (3,) = [grid_t, grid_h, grid_w]
|
||||
# HF model uses VIDEO_TOKEN (not IMAGE_TOKEN) for video frame content:
|
||||
# processing_minimax.py L245: replace(placeholder, self.VIDEO_TOKEN)
|
||||
T = int(grid_thw[0].item())
|
||||
M = int(grid_thw[1].item() * grid_thw[2].item()) // merge_length
|
||||
|
||||
# Reproduce the HF processor's per-frame timestamp markers
|
||||
# (processing_minimax.py: ts = frames_indices[frame*tps] / fps,
|
||||
# rendered as "]<]X.X seconds[>["). Falls back to no timestamps when
|
||||
# metadata is unavailable (keeping the replacement aligned with the
|
||||
# processor output in both cases).
|
||||
meta = (
|
||||
video_metadata[item_idx]
|
||||
if video_metadata is not None and item_idx < len(video_metadata)
|
||||
else None
|
||||
)
|
||||
fps = meta.get("fps") if meta else None
|
||||
frames_indices = meta.get("frames_indices") if meta else None
|
||||
|
||||
full: list[int] = []
|
||||
for frame_idx in range(T):
|
||||
if fps is not None and frames_indices is not None:
|
||||
idx = min(frame_idx * temporal_patch_size, len(frames_indices) - 1)
|
||||
ts = frames_indices[idx] / fps
|
||||
full += tokenizer.encode(
|
||||
f"]<]{ts:.1f} seconds[>[", add_special_tokens=False
|
||||
)
|
||||
full += [start_token_id] + [video_token_id] * M + [end_token_id]
|
||||
return PromptUpdateDetails.select_token_id(full, video_token_id)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=[image_token_id],
|
||||
replacement=get_image_replacement,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=[video_token_id],
|
||||
replacement=get_video_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# TODO(Isotr0py): Tie with MinimaxVideoProcessor
|
||||
# after https://github.com/vllm-project/vllm/pull/44126
|
||||
@VIDEO_LOADER_REGISTRY.register("minimax_m3_vl")
|
||||
class MiniMaxM3VideoBackend(VideoBackend):
|
||||
@classmethod
|
||||
def compute_frames_index_to_sample(
|
||||
cls,
|
||||
source: VideoSourceMetadata,
|
||||
target: VideoTargetMetadata,
|
||||
**kwargs,
|
||||
) -> list[int]:
|
||||
total_frames = source.total_frames_num
|
||||
video_fps = source.original_fps
|
||||
fps = target.fps
|
||||
|
||||
if total_frames <= 0 or video_fps <= 0 or fps <= 0:
|
||||
return [0] if total_frames > 0 else []
|
||||
|
||||
read_time_interval = 1.0 / fps
|
||||
eps = 1e-4
|
||||
|
||||
indices: list[int] = []
|
||||
prev_kept_ts = -float("inf")
|
||||
while True:
|
||||
if not indices:
|
||||
target_frame = 0
|
||||
else:
|
||||
target_ts = prev_kept_ts + read_time_interval - eps
|
||||
target_frame = math.ceil(target_ts * video_fps)
|
||||
target_frame = max(target_frame, indices[-1] + 1)
|
||||
if target_frame >= total_frames:
|
||||
break
|
||||
indices.append(target_frame)
|
||||
prev_kept_ts = target_frame / video_fps
|
||||
|
||||
last_frame_idx = total_frames - 1
|
||||
last_ts = last_frame_idx / video_fps
|
||||
if indices and indices[-1] != last_frame_idx and last_ts - prev_kept_ts > eps:
|
||||
indices.append(last_frame_idx)
|
||||
|
||||
if not indices:
|
||||
indices = [0]
|
||||
return indices
|
||||
@@ -0,0 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cross-platform (Triton) kernels for MiniMax M3 sparse attention."""
|
||||
|
||||
from .index_topk import (
|
||||
minimax_m3_index_decode,
|
||||
minimax_m3_index_score,
|
||||
minimax_m3_index_topk,
|
||||
)
|
||||
from .sparse_attn import minimax_m3_sparse_attn, minimax_m3_sparse_attn_decode
|
||||
|
||||
__all__ = [
|
||||
"minimax_m3_index_decode",
|
||||
"minimax_m3_index_score",
|
||||
"minimax_m3_index_topk",
|
||||
"minimax_m3_sparse_attn",
|
||||
"minimax_m3_sparse_attn_decode",
|
||||
]
|
||||
@@ -0,0 +1,898 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton kernels for MiniMax M3 lightning-indexer block scoring + top-k.
|
||||
|
||||
Index queries score each 128-token block of index keys (max over the block),
|
||||
then the top-k blocks (plus forced init/local blocks) are selected per query
|
||||
token. Adapted to vLLM's paged KV cache: the KV page size is forced to equal the
|
||||
sparse block size (128), so one sparse block maps to exactly one page.
|
||||
|
||||
Index-K cache layout (vLLM): ``(num_blocks, 128, idx_head_dim)`` (single head).
|
||||
|
||||
Only the paths MiniMax M3 uses are implemented: score_type="max", index value
|
||||
disabled (score-only indexer), single shared index head. The selected block ids
|
||||
feed the block-sparse attention kernels in ``sparse_attn``.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
# One sparse block == one KV page.
|
||||
SPARSE_BLOCK_SIZE = 128
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bitonic top-k helpers (layout-agnostic).
|
||||
# ---------------------------------------------------------------------------
|
||||
@triton.jit
|
||||
def _compare_and_swap(x, ids, flip, i: tl.constexpr, n_dims: tl.constexpr):
|
||||
n_outer: tl.constexpr = x.numel >> n_dims
|
||||
shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)]
|
||||
y = tl.reshape(x, shape)
|
||||
mask = tl.arange(0, 2)[None, :, None]
|
||||
left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype)
|
||||
right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype)
|
||||
left = tl.reshape(left, x.shape)
|
||||
right = tl.reshape(right, x.shape)
|
||||
y_idx = tl.reshape(ids, shape)
|
||||
left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape)
|
||||
right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape)
|
||||
left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype)
|
||||
right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype)
|
||||
idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True)
|
||||
ileft = left.to(idtype, bitcast=True)
|
||||
iright = right.to(idtype, bitcast=True)
|
||||
ix = x.to(idtype, bitcast=True)
|
||||
cond = (left > right) != flip
|
||||
ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix))
|
||||
new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids))
|
||||
return ret.to(x.dtype, bitcast=True), new_ids
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _bitonic_merge(
|
||||
x, ids, stage: tl.constexpr, order: tl.constexpr, n_dims: tl.constexpr
|
||||
):
|
||||
n_outer: tl.constexpr = x.numel >> n_dims
|
||||
tl.static_assert(stage <= n_dims)
|
||||
if order == 2:
|
||||
shape: tl.constexpr = [n_outer * 2 ** (n_dims - 1 - stage), 2, 2**stage]
|
||||
flip = tl.reshape(
|
||||
tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape
|
||||
)
|
||||
else:
|
||||
flip = order
|
||||
for i in tl.static_range(stage):
|
||||
x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims)
|
||||
return x, ids
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Index block-score kernel (paged). score[h, token, block] = max over the
|
||||
# 128-token block of (idx_q . index_k), causal-masked. BLOCK_SIZE_K == 128 so
|
||||
# each K-tile is exactly one page (BLOCKS_PER_K_BLOCK == 1).
|
||||
# ---------------------------------------------------------------------------
|
||||
# since prefill metadata is sliced from mixed batch metadata, seq_lens and prefix_lens
|
||||
# might lose pointer alignment, which trigger Triton recompiles. we don't actually
|
||||
# need pointer alignment for those tensors anyway because we do scalar load.
|
||||
@triton.jit(do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])
|
||||
def _index_block_score_kernel(
|
||||
q_ptr, # idx_q: [total_q, num_idx_heads, head_dim]
|
||||
ik_cache_ptr, # index-K cache: [num_blocks, 128, head_dim]
|
||||
score_ptr, # [num_idx_heads, total_q, max_block]
|
||||
block_table_ptr, # [num_reqs, max_blocks]
|
||||
cu_seqlens, # [batch+1] query start offsets
|
||||
seq_lens, # [batch] total K length
|
||||
prefix_lens, # [batch] context length before this chunk's queries
|
||||
num_idx_heads,
|
||||
head_dim: tl.constexpr,
|
||||
sm_scale,
|
||||
stride_q_n,
|
||||
stride_q_h,
|
||||
stride_q_d,
|
||||
stride_ik_blk,
|
||||
stride_ik_pos,
|
||||
stride_ik_d,
|
||||
stride_s_h,
|
||||
stride_s_n,
|
||||
stride_s_k,
|
||||
stride_bt_b,
|
||||
BLOCK_SIZE_Q: tl.constexpr,
|
||||
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
|
||||
):
|
||||
sm_scale_log2e = sm_scale * 1.4426950409
|
||||
pid_q = tl.program_id(0)
|
||||
pid_bh = tl.program_id(1)
|
||||
pid_b = pid_bh // num_idx_heads
|
||||
pid_h = pid_bh % num_idx_heads
|
||||
|
||||
seq_start = tl.load(cu_seqlens + pid_b)
|
||||
q_len = tl.load(cu_seqlens + pid_b + 1) - seq_start
|
||||
seq_len = tl.load(seq_lens + pid_b)
|
||||
prefix_len = tl.load(prefix_lens + pid_b)
|
||||
if BLOCK_SIZE_Q * pid_q >= q_len:
|
||||
return
|
||||
|
||||
q_ptrs = tl.make_block_ptr(
|
||||
base=q_ptr + seq_start * stride_q_n + pid_h * stride_q_h,
|
||||
shape=(q_len, head_dim),
|
||||
strides=(stride_q_n, stride_q_d),
|
||||
offsets=(pid_q * BLOCK_SIZE_Q, 0),
|
||||
block_shape=(BLOCK_SIZE_Q, head_dim),
|
||||
order=(1, 0),
|
||||
)
|
||||
q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")
|
||||
q_start = prefix_len + pid_q * BLOCK_SIZE_Q
|
||||
|
||||
off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len
|
||||
off_k = tl.arange(0, BLOCK_SIZE_K)
|
||||
off_d = tl.arange(0, head_dim)
|
||||
# Block table row for this request.
|
||||
bt_row = block_table_ptr + pid_b * stride_bt_b
|
||||
# Causal window: only blocks up to the last query token's position.
|
||||
hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q)
|
||||
for i in tl.range(0, hi, BLOCK_SIZE_K):
|
||||
blk = i // BLOCK_SIZE_K
|
||||
page = tl.load(bt_row + blk).to(tl.int64)
|
||||
pos = i + off_k
|
||||
# index-K for this page: [BLOCK_SIZE_D, BLOCK_SIZE_K] (transposed)
|
||||
# we don't need masked load for K, because KV cache ensures
|
||||
# allocation is multiple of BLOCK_SIZE_K.
|
||||
# for tokens beyond seqlen, they will be masked in qk later.
|
||||
k = tl.load(
|
||||
ik_cache_ptr
|
||||
+ page * stride_ik_blk
|
||||
+ off_k[None, :] * stride_ik_pos
|
||||
+ off_d[:, None] * stride_ik_d,
|
||||
)
|
||||
qk = tl.dot(q, k) * sm_scale_log2e
|
||||
# apply causal mask as needed
|
||||
if q_start < i + BLOCK_SIZE_K:
|
||||
qk = tl.where(off_q[:, None] >= pos[None, :], qk, float("-inf"))
|
||||
# one sparse block per K-tile -> max over the 128 positions
|
||||
score = tl.max(qk, axis=1) # [BLOCK_SIZE_Q]
|
||||
s_ptrs = (
|
||||
score_ptr
|
||||
+ pid_h * stride_s_h
|
||||
+ (seq_start + pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q))
|
||||
* stride_s_n
|
||||
+ blk * stride_s_k
|
||||
)
|
||||
q_store_mask = (pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q)) < q_len
|
||||
tl.store(s_ptrs, score, mask=q_store_mask)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Top-k selection over per-token block scores (layout-agnostic). block_size_q
|
||||
# is 1 for M3, so top-k is computed per query token.
|
||||
# ---------------------------------------------------------------------------
|
||||
# since prefill metadata is sliced from mixed batch metadata, prefix_lens
|
||||
# might lose pointer alignment, which trigger Triton recompiles. we don't actually
|
||||
# need pointer alignment for those tensors anyway because we do scalar load.
|
||||
@triton.heuristics({"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"])})
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BLOCK_SIZE_K": 2048}, num_warps=8, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_K": 1024}, num_warps=8, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_K": 512}, num_warps=8, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),
|
||||
],
|
||||
key=["BLOCK_SIZE_T"],
|
||||
)
|
||||
@triton.jit(do_not_specialize_on_alignment=["prefix_lens"])
|
||||
def _topk_index_kernel(
|
||||
s_ptr, # [num_heads, total_q, max_block]
|
||||
ti_ptr, # [num_heads, total_q, topk]
|
||||
sample_interval: tl.constexpr, # block_size_q (1 for M3)
|
||||
block_size: tl.constexpr, # sparse block size (128)
|
||||
cu_seqlens,
|
||||
cu_seqblocks_q,
|
||||
prefix_lens,
|
||||
topk,
|
||||
init_blocks: tl.constexpr,
|
||||
local_blocks: tl.constexpr,
|
||||
stride_s_h,
|
||||
stride_s_n,
|
||||
stride_s_k,
|
||||
stride_ti_h,
|
||||
stride_ti_n,
|
||||
stride_ti_t,
|
||||
BLOCK_SIZE_K: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
MASK_INIT: tl.constexpr,
|
||||
MASK_LOCAL: tl.constexpr,
|
||||
):
|
||||
tl.static_assert(BLOCK_SIZE_K > BLOCK_SIZE_T)
|
||||
pid_q = tl.program_id(0)
|
||||
pid_b = tl.program_id(1)
|
||||
pid_h = tl.program_id(2)
|
||||
seq_start = tl.load(cu_seqlens + pid_b)
|
||||
block_start = tl.load(cu_seqblocks_q + pid_b)
|
||||
block_num = tl.load(cu_seqblocks_q + pid_b + 1) - block_start
|
||||
prefix_len = tl.load(prefix_lens + pid_b)
|
||||
if pid_q >= block_num:
|
||||
return
|
||||
off_k = tl.arange(0, BLOCK_SIZE_K)
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
s_ptrs = (
|
||||
s_ptr
|
||||
+ (seq_start + pid_q * sample_interval) * stride_s_n
|
||||
+ pid_h * stride_s_h
|
||||
+ off_k * stride_s_k
|
||||
)
|
||||
topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
|
||||
topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
|
||||
left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
|
||||
valid_blocks = (prefix_len + pid_q * sample_interval + block_size) // block_size
|
||||
for i in tl.range(0, valid_blocks, BLOCK_SIZE_K):
|
||||
causal_mask = i + off_k < valid_blocks
|
||||
local_mask = i + off_k >= max(0, valid_blocks - local_blocks)
|
||||
init_mask = i + off_k < init_blocks
|
||||
score = tl.load(s_ptrs, mask=causal_mask, other=-1e30).to(tl.float32)
|
||||
score = tl.where(score != score, -1e30, score)
|
||||
s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
|
||||
if MASK_INIT:
|
||||
score = tl.where(causal_mask & init_mask, score - 1e29, score)
|
||||
else:
|
||||
score = tl.where(causal_mask & init_mask, 1e30, score)
|
||||
if MASK_LOCAL:
|
||||
score = tl.where(causal_mask & local_mask, score - 1e28, score)
|
||||
else:
|
||||
score = tl.where(causal_mask & local_mask, 1e29, score)
|
||||
topk_score, last_topk_score = score, topk_score
|
||||
topk_idx, last_topk_idx = (tl.where(causal_mask, i + off_k + 1, 0), topk_idx)
|
||||
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
|
||||
for j in tl.static_range(1, n_dims):
|
||||
topk_score, topk_idx = _bitonic_merge(
|
||||
topk_score, topk_idx.to(tl.int32), j, 2, n_dims
|
||||
)
|
||||
if i != 0:
|
||||
topk_score, topk_idx = _bitonic_merge(
|
||||
topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
|
||||
)
|
||||
topk_score_new = last_topk_score * left_half_mask + topk_score * (
|
||||
1 - left_half_mask
|
||||
)
|
||||
topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
|
||||
1 - left_half_mask
|
||||
)
|
||||
topk_score, topk_idx = _bitonic_merge(
|
||||
topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
|
||||
)
|
||||
else:
|
||||
topk_score, topk_idx = _bitonic_merge(
|
||||
topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
|
||||
)
|
||||
topk_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
|
||||
topk_idx = tl.sum(
|
||||
topk_mask[:, None]
|
||||
* tl.reshape(topk_idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
||||
axis=0,
|
||||
)
|
||||
ti_ptrs = (
|
||||
ti_ptr
|
||||
+ (block_start + pid_q) * stride_ti_n
|
||||
+ pid_h * stride_ti_h
|
||||
+ off_t * stride_ti_t
|
||||
)
|
||||
store_mask = off_t < topk
|
||||
valid_mask = off_t < valid_blocks
|
||||
topk_idx = tl.where(store_mask & valid_mask, topk_idx, -1)
|
||||
tl.store(ti_ptrs, topk_idx.to(ti_ptrs.dtype.element_ty), mask=store_mask)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Decode index-score kernel (split-K over seq blocks). Decode batches are
|
||||
# flattened request-major, with a runtime query length used to map each query
|
||||
# token back to its request metadata. Chunk counts depend only on shape
|
||||
# constants so the grid is fixed within a cuda graph. Base-2 (exp2/log2)
|
||||
# softmax matches prefill.
|
||||
# ---------------------------------------------------------------------------
|
||||
@triton.jit(do_not_specialize=["num_kv_chunks", "decode_query_len"])
|
||||
def _decode_index_score_kernel(
|
||||
q_ptr, # idx_q: [total_q, num_idx_heads, head_dim]
|
||||
ik_cache_ptr, # index-K cache: [num_blocks, 128, head_dim]
|
||||
score_ptr, # [num_idx_heads, total_q, max_block]
|
||||
block_table_ptr, # [num_reqs, max_blocks]
|
||||
seq_lens, # [num_reqs]
|
||||
num_idx_heads: tl.constexpr,
|
||||
head_dim: tl.constexpr,
|
||||
init_blocks,
|
||||
local_blocks,
|
||||
sm_scale,
|
||||
decode_query_len,
|
||||
stride_q_n,
|
||||
stride_q_h,
|
||||
stride_q_d,
|
||||
stride_ik_blk,
|
||||
stride_ik_pos,
|
||||
stride_ik_d,
|
||||
stride_s_h,
|
||||
stride_s_n,
|
||||
stride_s_k,
|
||||
stride_bt_b,
|
||||
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
|
||||
num_kv_chunks,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
sm_scale_log2e = sm_scale * 1.4426950409
|
||||
pid_b = tl.program_id(0) # flattened query-token id
|
||||
pid_c = tl.program_id(1)
|
||||
req_id = pid_b // decode_query_len
|
||||
q_offset = pid_b - req_id * decode_query_len
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
seq_len = tl.load(seq_lens + req_id)
|
||||
query_pos = seq_len - decode_query_len + q_offset
|
||||
# Full-CG padding uses zero-length request rows. Clamp to an empty
|
||||
# attention range instead of letting padded rows produce negative lengths.
|
||||
kv_len = tl.maximum(query_pos + 1, 0)
|
||||
num_blocks = (kv_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K
|
||||
|
||||
# block-aligned fixed-count split: grid independent of seq_len (cuda graph).
|
||||
chunk_size_blocks = (num_blocks + num_kv_chunks - 1) // num_kv_chunks
|
||||
chunk_start_block = pid_c * chunk_size_blocks
|
||||
chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
|
||||
if chunk_start_block >= chunk_end_block:
|
||||
return
|
||||
off_k = tl.arange(0, BLOCK_SIZE_K) # positions within a 128-block
|
||||
off_d = tl.arange(0, head_dim)
|
||||
bt_row = block_table_ptr + req_id * stride_bt_b
|
||||
# Force-select init (1e30) and local (1e29, higher priority) blocks.
|
||||
local_start = tl.maximum(0, num_blocks - local_blocks)
|
||||
# query vectors across all heads
|
||||
q = tl.load(
|
||||
q_ptr
|
||||
+ pid_b * stride_q_n
|
||||
+ tl.arange(0, num_idx_heads) * stride_q_h
|
||||
+ off_d[:, None] * stride_q_d,
|
||||
) # [D,H]
|
||||
for blk in tl.range(chunk_start_block, chunk_end_block):
|
||||
page = tl.load(bt_row + blk).to(tl.int64)
|
||||
pos = blk * BLOCK_SIZE_K + off_k
|
||||
pos_mask = pos < kv_len
|
||||
# we don't need masked load for K, because KV cache ensures
|
||||
# allocation is multiple of BLOCK_SIZE_K.
|
||||
# for tokens beyond seqlen, they will be masked in qk later.
|
||||
k = tl.load(
|
||||
ik_cache_ptr
|
||||
+ page * stride_ik_blk
|
||||
+ off_k[:, None] * stride_ik_pos
|
||||
+ off_d * stride_ik_d,
|
||||
) # [N,D]
|
||||
kq = tl.dot(k, q) * sm_scale_log2e # [N,H]
|
||||
kq = tl.where(pos_mask[:, None], kq, float("-inf"))
|
||||
score = tl.max(kq, axis=0) # [H]
|
||||
is_init = blk < init_blocks
|
||||
is_local = (blk >= local_start) & (blk < num_blocks)
|
||||
score = tl.where(is_local, 1e29, tl.where(is_init, 1e30, score))
|
||||
tl.store(
|
||||
score_ptr
|
||||
+ tl.arange(0, num_idx_heads) * stride_s_h
|
||||
+ pid_b * stride_s_n
|
||||
+ blk * stride_s_k,
|
||||
score,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Decode top-k (split-K): per-chunk partial top-k + merge. Forced init/local
|
||||
# blocks are already encoded in the scores.
|
||||
# ---------------------------------------------------------------------------
|
||||
@triton.heuristics({"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"])})
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),
|
||||
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3),
|
||||
triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),
|
||||
],
|
||||
key=["topk"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["chunk_blocks", "decode_query_len"])
|
||||
def _topk_index_partial_kernel(
|
||||
s_ptr, # score: [num_idx_heads, total_q, max_block]
|
||||
ts_partial_ptr, # partial scores out: [NUM_TOPK_CHUNKS, num_idx_heads, total_q, T]
|
||||
ti_partial_ptr, # partial idx out (1-indexed global, 0=invalid): same shape
|
||||
seq_lens, # [num_reqs]
|
||||
block_size: tl.constexpr, # sparse block size (128)
|
||||
topk: tl.constexpr,
|
||||
chunk_blocks, # how many score-blocks each chunk owns
|
||||
decode_query_len,
|
||||
stride_s_h,
|
||||
stride_s_b,
|
||||
stride_s_k,
|
||||
stride_ts_c,
|
||||
stride_ts_h,
|
||||
stride_ts_b,
|
||||
stride_ts_t,
|
||||
stride_ti_c,
|
||||
stride_ti_h,
|
||||
stride_ti_b,
|
||||
stride_ti_t,
|
||||
BLOCK_SIZE_K: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
tl.static_assert(topk < BLOCK_SIZE_K)
|
||||
pid_b = tl.program_id(0) # flattened query-token id
|
||||
pid_h = tl.program_id(1)
|
||||
pid_chunk = tl.program_id(2)
|
||||
req_id = pid_b // decode_query_len
|
||||
q_offset = pid_b - req_id * decode_query_len
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
seq_len = tl.load(seq_lens + req_id)
|
||||
query_pos = seq_len - decode_query_len + q_offset
|
||||
# Full-CG padding uses zero-length request rows. Clamp to an empty
|
||||
# attention range instead of letting padded rows produce negative lengths.
|
||||
kv_len = tl.maximum(query_pos + 1, 0)
|
||||
num_blocks = (kv_len + block_size - 1) // block_size
|
||||
|
||||
# Slice this chunk owns within [0, num_blocks).
|
||||
chunk_start = pid_chunk * chunk_blocks
|
||||
chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks)
|
||||
chunk_actual = tl.maximum(chunk_end - chunk_start, 0)
|
||||
|
||||
off_k = tl.arange(0, BLOCK_SIZE_K)
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
|
||||
s_ptrs = (
|
||||
s_ptr
|
||||
+ pid_b * stride_s_b
|
||||
+ pid_h * stride_s_h
|
||||
+ (chunk_start + off_k) * stride_s_k
|
||||
)
|
||||
|
||||
topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
|
||||
topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
|
||||
left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
|
||||
|
||||
# Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty
|
||||
# chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0.
|
||||
for i in tl.range(0, chunk_actual, BLOCK_SIZE_K):
|
||||
mask = off_k < chunk_actual - i
|
||||
score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32)
|
||||
score = tl.where(score != score, -1e30, score)
|
||||
s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
|
||||
topk_score, last_topk_score = score, topk_score
|
||||
topk_idx, last_topk_idx = (
|
||||
tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global
|
||||
topk_idx,
|
||||
)
|
||||
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
|
||||
for j in tl.static_range(1, n_dims):
|
||||
topk_score, topk_idx = _bitonic_merge(
|
||||
topk_score, topk_idx.to(tl.int32), j, 2, n_dims
|
||||
)
|
||||
if i != 0:
|
||||
topk_score, topk_idx = _bitonic_merge(
|
||||
topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
|
||||
)
|
||||
topk_score_new = last_topk_score * left_half_mask + topk_score * (
|
||||
1 - left_half_mask
|
||||
)
|
||||
topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
|
||||
1 - left_half_mask
|
||||
)
|
||||
topk_score, topk_idx = _bitonic_merge(
|
||||
topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
|
||||
)
|
||||
else:
|
||||
topk_score, topk_idx = _bitonic_merge(
|
||||
topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
|
||||
)
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
# Extract first BLOCK_SIZE_T entries (top-K of this chunk after the sort).
|
||||
topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
|
||||
final_score = tl.sum(
|
||||
topk_mask_extract[:, None]
|
||||
* tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
||||
axis=0,
|
||||
)
|
||||
final_idx = tl.sum(
|
||||
topk_mask_extract[:, None]
|
||||
* tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
||||
axis=0,
|
||||
)
|
||||
|
||||
# Always write all BLOCK_SIZE_T slots — invalid slots carry -1e30 / 0
|
||||
# sentinels and lose to real scores in the merge stage.
|
||||
ts_ptrs = (
|
||||
ts_partial_ptr
|
||||
+ pid_chunk * stride_ts_c
|
||||
+ pid_b * stride_ts_b
|
||||
+ pid_h * stride_ts_h
|
||||
+ off_t * stride_ts_t
|
||||
)
|
||||
ti_ptrs = (
|
||||
ti_partial_ptr
|
||||
+ pid_chunk * stride_ti_c
|
||||
+ pid_b * stride_ti_b
|
||||
+ pid_h * stride_ti_h
|
||||
+ off_t * stride_ti_t
|
||||
)
|
||||
tl.store(ts_ptrs, final_score)
|
||||
tl.store(ti_ptrs, final_idx)
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
|
||||
"BLOCK_SIZE_K": lambda args: triton.next_power_of_2(
|
||||
args["num_topk_chunks"] * triton.next_power_of_2(args["topk"])
|
||||
),
|
||||
}
|
||||
)
|
||||
@triton.jit(do_not_specialize=["num_topk_chunks", "decode_query_len"])
|
||||
def _topk_index_merge_kernel(
|
||||
ts_partial_ptr, # partial scores: [NUM_TOPK_CHUNKS, num_idx_heads, total_q, T]
|
||||
ti_partial_ptr, # partial idx (1-indexed global, 0=invalid): same shape
|
||||
ti_final_ptr, # final idx (0-indexed, -1=invalid): [num_idx_heads, total_q, topk]
|
||||
seq_lens, # [num_reqs]
|
||||
block_size: tl.constexpr, # sparse block size (128)
|
||||
topk: tl.constexpr,
|
||||
decode_query_len,
|
||||
stride_ts_c,
|
||||
stride_ts_h,
|
||||
stride_ts_b,
|
||||
stride_ts_t,
|
||||
stride_ti_c,
|
||||
stride_ti_h,
|
||||
stride_ti_b,
|
||||
stride_ti_t,
|
||||
stride_tif_h,
|
||||
stride_tif_b,
|
||||
stride_tif_t,
|
||||
num_topk_chunks,
|
||||
BLOCK_SIZE_K: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
pid_b = tl.program_id(0) # flattened query-token id
|
||||
pid_h = tl.program_id(1)
|
||||
req_id = pid_b // decode_query_len
|
||||
q_offset = pid_b - req_id * decode_query_len
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
seq_len = tl.load(seq_lens + req_id)
|
||||
query_pos = seq_len - decode_query_len + q_offset
|
||||
# Full-CG padding uses zero-length request rows. Clamp to an empty
|
||||
# attention range instead of letting padded rows produce negative lengths.
|
||||
kv_len = tl.maximum(query_pos + 1, 0)
|
||||
num_blocks = (kv_len + block_size - 1) // block_size
|
||||
|
||||
# Load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K.
|
||||
# Candidate at flat position p comes from chunk = p // BLOCK_SIZE_T,
|
||||
# in_chunk = p % BLOCK_SIZE_T.
|
||||
off = tl.arange(0, BLOCK_SIZE_K)
|
||||
chunk_idx = off // BLOCK_SIZE_T
|
||||
in_chunk_idx = off % BLOCK_SIZE_T
|
||||
valid = chunk_idx < num_topk_chunks
|
||||
|
||||
score_offset = (
|
||||
chunk_idx * stride_ts_c
|
||||
+ pid_h * stride_ts_h
|
||||
+ pid_b * stride_ts_b
|
||||
+ in_chunk_idx * stride_ts_t
|
||||
)
|
||||
idx_offset = (
|
||||
chunk_idx * stride_ti_c
|
||||
+ pid_h * stride_ti_h
|
||||
+ pid_b * stride_ti_b
|
||||
+ in_chunk_idx * stride_ti_t
|
||||
)
|
||||
|
||||
score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to(
|
||||
tl.float32
|
||||
)
|
||||
score = tl.where(score != score, -1e30, score)
|
||||
idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32)
|
||||
|
||||
# Full bitonic descending sort of BLOCK_SIZE_K items.
|
||||
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
|
||||
for j in tl.static_range(1, n_dims):
|
||||
score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims)
|
||||
score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims)
|
||||
|
||||
# Extract first BLOCK_SIZE_T positions — these are the global top-K.
|
||||
extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
|
||||
topk_idx_final = tl.sum(
|
||||
extract_mask[:, None]
|
||||
* tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
||||
axis=0,
|
||||
)
|
||||
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
tif_ptrs = (
|
||||
ti_final_ptr
|
||||
+ pid_h * stride_tif_h
|
||||
+ pid_b * stride_tif_b
|
||||
+ off_t * stride_tif_t
|
||||
)
|
||||
store_mask = off_t < topk
|
||||
topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1)
|
||||
tl.store(
|
||||
tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty), mask=store_mask
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Python wrappers
|
||||
# ---------------------------------------------------------------------------
|
||||
@torch.no_grad()
|
||||
def minimax_m3_index_score(
|
||||
idx_q: torch.Tensor, # [total_q, num_idx_heads, head_dim]
|
||||
index_kv_cache: torch.Tensor, # [num_blocks, 128, head_dim]
|
||||
block_table: torch.Tensor, # [batch, max_blocks]
|
||||
cu_seqlens_q: torch.Tensor, # [batch+1] int32
|
||||
seq_lens: torch.Tensor, # [batch] int32
|
||||
prefix_lens: torch.Tensor, # [batch] int32
|
||||
max_query_len: int,
|
||||
max_seq_len: int,
|
||||
num_kv_heads: int,
|
||||
sm_scale: float,
|
||||
) -> torch.Tensor:
|
||||
"""Compute per-token index scores for each visible sparse block.
|
||||
|
||||
Returns score [num_kv_heads, total_q, max_block], where each score is the
|
||||
max over a 128-token index-K block. M3 has num_idx_heads == num_kv_heads.
|
||||
"""
|
||||
total_q, num_idx_heads, head_dim = idx_q.shape
|
||||
assert num_idx_heads == num_kv_heads, (
|
||||
"M3 expects num_idx_heads == num_kv_heads (no topk index reduce)"
|
||||
)
|
||||
batch = cu_seqlens_q.shape[0] - 1
|
||||
max_block = triton.cdiv(max_seq_len, SPARSE_BLOCK_SIZE)
|
||||
|
||||
# Keep score strides 16-divisible to avoid Triton recompiles.
|
||||
score_block_stride = round_up(max_block, 16)
|
||||
score = torch.empty(
|
||||
(num_idx_heads, total_q, score_block_stride),
|
||||
dtype=torch.float32,
|
||||
device=idx_q.device,
|
||||
)
|
||||
BLOCK_SIZE_Q = 64
|
||||
grid_score = (triton.cdiv(max_query_len, BLOCK_SIZE_Q), batch * num_idx_heads)
|
||||
_index_block_score_kernel[grid_score](
|
||||
idx_q,
|
||||
index_kv_cache,
|
||||
score,
|
||||
block_table,
|
||||
cu_seqlens_q,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
num_idx_heads,
|
||||
head_dim,
|
||||
sm_scale,
|
||||
idx_q.stride(0),
|
||||
idx_q.stride(1),
|
||||
idx_q.stride(2),
|
||||
index_kv_cache.stride(0),
|
||||
index_kv_cache.stride(1),
|
||||
index_kv_cache.stride(2),
|
||||
score.stride(0),
|
||||
score.stride(1),
|
||||
score.stride(2),
|
||||
block_table.stride(0),
|
||||
BLOCK_SIZE_Q=BLOCK_SIZE_Q,
|
||||
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
|
||||
)
|
||||
return score
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def minimax_m3_index_topk(
|
||||
score: torch.Tensor, # [num_idx_heads, total_q, max_block]
|
||||
cu_seqlens_q: torch.Tensor, # [batch+1] int32
|
||||
prefix_lens: torch.Tensor, # [batch] int32
|
||||
max_query_len: int,
|
||||
topk: int,
|
||||
init_blocks: int,
|
||||
local_blocks: int,
|
||||
) -> torch.Tensor:
|
||||
"""Select index top-k from a precomputed score tensor."""
|
||||
num_idx_heads = score.shape[0]
|
||||
batch = cu_seqlens_q.shape[0] - 1
|
||||
total_q = score.shape[1]
|
||||
topk_idx = torch.empty(
|
||||
(num_idx_heads, total_q, topk),
|
||||
dtype=torch.int32,
|
||||
device=score.device,
|
||||
)
|
||||
# block_size_q == 1 -> query blocks coincide with query tokens.
|
||||
grid_topk = (max_query_len, batch, num_idx_heads)
|
||||
_topk_index_kernel[grid_topk](
|
||||
score,
|
||||
topk_idx,
|
||||
1, # sample_interval (block_size_q)
|
||||
SPARSE_BLOCK_SIZE,
|
||||
cu_seqlens_q,
|
||||
cu_seqlens_q, # cu_seqblocks_q == cu_seqlens_q when block_size_q == 1
|
||||
prefix_lens,
|
||||
topk,
|
||||
init_blocks,
|
||||
local_blocks,
|
||||
score.stride(0),
|
||||
score.stride(1),
|
||||
score.stride(2),
|
||||
topk_idx.stride(0),
|
||||
topk_idx.stride(1),
|
||||
topk_idx.stride(2),
|
||||
MASK_INIT=False,
|
||||
MASK_LOCAL=False,
|
||||
)
|
||||
return topk_idx
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def minimax_m3_index_decode(
|
||||
idx_q: torch.Tensor, # [total_q, num_idx_heads, head_dim]
|
||||
index_kv_cache: torch.Tensor, # [num_blocks, 128, head_dim]
|
||||
block_table: torch.Tensor, # [num_reqs, max_blocks]
|
||||
seq_lens: torch.Tensor, # [num_reqs] int32
|
||||
max_seq_len: int,
|
||||
topk: int,
|
||||
init_blocks: int,
|
||||
local_blocks: int,
|
||||
num_kv_heads: int,
|
||||
sm_scale: float,
|
||||
decode_query_len: int,
|
||||
) -> torch.Tensor:
|
||||
"""Decode index block-score + top-k, both split-K (cudagraph-safe).
|
||||
|
||||
Returns topk_idx [num_kv_heads, total_q, topk] (0-indexed block ids, -1 pad).
|
||||
"""
|
||||
total_q, num_idx_heads, head_dim = idx_q.shape
|
||||
assert num_idx_heads == num_kv_heads, (
|
||||
"M3 expects num_idx_heads == num_kv_heads (no topk index reduce)"
|
||||
)
|
||||
assert total_q == seq_lens.shape[0] * decode_query_len
|
||||
batch = total_q
|
||||
max_block = triton.cdiv(max_seq_len, SPARSE_BLOCK_SIZE)
|
||||
use_pdl = current_platform.is_arch_support_pdl()
|
||||
# `launch_pdl` is a Triton runtime kwarg only some backends accept (CUDA
|
||||
# SM9+); this ROCm Triton rejects it even when False ("Keyword argument
|
||||
# launch_pdl was specified but unrecognised"). Only pass it when PDL is
|
||||
# actually supported -- on ROCm use_pdl is always False, so it's omitted.
|
||||
pdl_launch = {"launch_pdl": True} if use_pdl else {}
|
||||
|
||||
# Keep score strides 16-divisible to avoid Triton recompiles.
|
||||
score_block_stride = round_up(max_block, 16)
|
||||
score = torch.empty(
|
||||
(num_idx_heads, total_q, score_block_stride),
|
||||
dtype=torch.float32,
|
||||
device=idx_q.device,
|
||||
)
|
||||
# split-K over seq blocks; chunk count depends only on shape constants so
|
||||
# the grid is fixed within a cuda graph.
|
||||
TARGET_GRID = 4096
|
||||
MAX_NUM_KV_CHUNKS = 256
|
||||
target = max(
|
||||
1, min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch * num_idx_heads))
|
||||
)
|
||||
num_kv_chunks = 1 << (target.bit_length() - 1)
|
||||
grid_score = (batch, num_kv_chunks)
|
||||
_decode_index_score_kernel[grid_score](
|
||||
idx_q,
|
||||
index_kv_cache,
|
||||
score,
|
||||
block_table,
|
||||
seq_lens,
|
||||
num_idx_heads,
|
||||
head_dim,
|
||||
init_blocks,
|
||||
local_blocks,
|
||||
sm_scale,
|
||||
decode_query_len,
|
||||
idx_q.stride(0),
|
||||
idx_q.stride(1),
|
||||
idx_q.stride(2),
|
||||
index_kv_cache.stride(0),
|
||||
index_kv_cache.stride(1),
|
||||
index_kv_cache.stride(2),
|
||||
score.stride(0),
|
||||
score.stride(1),
|
||||
score.stride(2),
|
||||
block_table.stride(0),
|
||||
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
|
||||
num_kv_chunks=num_kv_chunks,
|
||||
USE_PDL=use_pdl,
|
||||
**pdl_launch,
|
||||
)
|
||||
|
||||
topk_idx = torch.empty(
|
||||
(num_idx_heads, total_q, topk),
|
||||
dtype=torch.int32,
|
||||
device=idx_q.device,
|
||||
)
|
||||
# Chunk count is shape-constant (cudagraph-safe), capped so the merge sorts
|
||||
# pow2(num_topk_chunks * pow2(topk)) candidates.
|
||||
TOPK_TARGET_GRID = 64
|
||||
MAX_NUM_TOPK_CHUNKS = 16
|
||||
topk_target = max(
|
||||
1, min(MAX_NUM_TOPK_CHUNKS, TOPK_TARGET_GRID // max(1, batch * num_idx_heads))
|
||||
)
|
||||
num_topk_chunks = 1 << (topk_target.bit_length() - 1)
|
||||
block_size_t = triton.next_power_of_2(topk)
|
||||
chunk_blocks = (max_block + num_topk_chunks - 1) // num_topk_chunks
|
||||
topk_score_partial = torch.empty(
|
||||
num_topk_chunks,
|
||||
num_idx_heads,
|
||||
batch,
|
||||
block_size_t,
|
||||
dtype=torch.float32,
|
||||
device=idx_q.device,
|
||||
)
|
||||
topk_idx_partial = torch.empty(
|
||||
num_topk_chunks,
|
||||
num_idx_heads,
|
||||
batch,
|
||||
block_size_t,
|
||||
dtype=torch.int32,
|
||||
device=idx_q.device,
|
||||
)
|
||||
_topk_index_partial_kernel[(batch, num_idx_heads, num_topk_chunks)](
|
||||
score,
|
||||
topk_score_partial,
|
||||
topk_idx_partial,
|
||||
seq_lens,
|
||||
SPARSE_BLOCK_SIZE,
|
||||
topk,
|
||||
chunk_blocks,
|
||||
decode_query_len,
|
||||
score.stride(0),
|
||||
score.stride(1),
|
||||
score.stride(2),
|
||||
topk_score_partial.stride(0),
|
||||
topk_score_partial.stride(1),
|
||||
topk_score_partial.stride(2),
|
||||
topk_score_partial.stride(3),
|
||||
topk_idx_partial.stride(0),
|
||||
topk_idx_partial.stride(1),
|
||||
topk_idx_partial.stride(2),
|
||||
topk_idx_partial.stride(3),
|
||||
USE_PDL=use_pdl,
|
||||
**pdl_launch,
|
||||
)
|
||||
_topk_index_merge_kernel[(batch, num_idx_heads)](
|
||||
topk_score_partial,
|
||||
topk_idx_partial,
|
||||
topk_idx,
|
||||
seq_lens,
|
||||
SPARSE_BLOCK_SIZE,
|
||||
topk,
|
||||
decode_query_len,
|
||||
topk_score_partial.stride(0),
|
||||
topk_score_partial.stride(1),
|
||||
topk_score_partial.stride(2),
|
||||
topk_score_partial.stride(3),
|
||||
topk_idx_partial.stride(0),
|
||||
topk_idx_partial.stride(1),
|
||||
topk_idx_partial.stride(2),
|
||||
topk_idx_partial.stride(3),
|
||||
topk_idx.stride(0),
|
||||
topk_idx.stride(1),
|
||||
topk_idx.stride(2),
|
||||
num_topk_chunks=num_topk_chunks,
|
||||
USE_PDL=use_pdl,
|
||||
**pdl_launch,
|
||||
)
|
||||
return topk_idx
|
||||
@@ -0,0 +1,593 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton kernels for MiniMax M3 block-sparse GQA attention.
|
||||
|
||||
The main heads attend only to the blocks selected by the lightning indexer (see
|
||||
``index_topk``). Adapted to vLLM's paged KV cache: the KV page size is forced to
|
||||
equal the sparse block size (128), so one selected block maps to exactly one
|
||||
page.
|
||||
|
||||
Main K/V cache layout (vLLM):
|
||||
``(num_blocks, 2, 128, num_kv_heads, head_dim)`` K=[:,0] V=[:,1]
|
||||
|
||||
Only the paths MiniMax M3 uses are implemented: no attention sink, base-2
|
||||
(exp2/log2) softmax. The decode kernels use split-K (flash-decoding) over the
|
||||
selected blocks with a separate merge step, since one query token per request
|
||||
leaves the prefill kernels (which parallelize over the query dim) idle.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
# One sparse block == one KV page.
|
||||
SPARSE_BLOCK_SIZE = 128
|
||||
|
||||
|
||||
_SPARSE_ATTN_NUM_STAGES_KWARG: dict | None = None
|
||||
|
||||
|
||||
def _sparse_attn_num_stages_kwarg() -> dict:
|
||||
"""Triton ``num_stages`` override for the sparse-attn GEMM kernels.
|
||||
|
||||
Forced only where required: CDNA3 (gfx942) caps LDS at
|
||||
64 KB, and the default 2-stage pipeline double-buffers the 128x128 K/V tiles
|
||||
to ~66 KB ("out of resource: shared memory"), so pin gfx942 to a single
|
||||
stage (~32 KB, which fits). Everywhere else (NVIDIA, CDNA4 gfx950) return an
|
||||
empty kwarg and let Triton keep its own default -- don't second-guess it.
|
||||
Cached: the arch is fixed per process.
|
||||
"""
|
||||
global _SPARSE_ATTN_NUM_STAGES_KWARG
|
||||
if _SPARSE_ATTN_NUM_STAGES_KWARG is None:
|
||||
kwarg: dict = {}
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
if on_gfx942():
|
||||
kwarg = {"num_stages": 1}
|
||||
_SPARSE_ATTN_NUM_STAGES_KWARG = kwarg
|
||||
return _SPARSE_ATTN_NUM_STAGES_KWARG
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GQA block-sparse attention (paged). Main heads attend only to the selected
|
||||
# blocks. BLOCK_SIZE_K == 128 so each selected block is one page.
|
||||
# ---------------------------------------------------------------------------
|
||||
# since prefill metadata is sliced from mixed batch metadata, seq_lens and prefix_lens
|
||||
# might lose pointer alignment, which trigger Triton recompiles. we don't actually
|
||||
# need pointer alignment for those tensors anyway because we do scalar load.
|
||||
@triton.heuristics(
|
||||
{
|
||||
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
||||
"BLOCK_SIZE_H": lambda args: triton.next_power_of_2(args["gqa_group_size"]),
|
||||
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
|
||||
"BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"]
|
||||
* triton.next_power_of_2(args["gqa_group_size"]),
|
||||
}
|
||||
)
|
||||
@triton.jit(do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])
|
||||
def _gqa_sparse_fwd_kernel(
|
||||
q_ptr, # [total_q, num_heads, head_dim]
|
||||
kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim]
|
||||
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
|
||||
o_ptr, # [total_q, num_heads, head_dim]
|
||||
block_table_ptr, # [num_reqs, max_blocks]
|
||||
cu_seqlens_q,
|
||||
cu_seqblocks_q,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
num_kv_heads,
|
||||
gqa_group_size,
|
||||
head_dim,
|
||||
max_topk,
|
||||
num_q_loop,
|
||||
sm_scale,
|
||||
stride_qn,
|
||||
stride_qh,
|
||||
stride_qd,
|
||||
stride_kv_blk,
|
||||
stride_kv_kv,
|
||||
stride_kv_pos,
|
||||
stride_kv_h,
|
||||
stride_kv_d,
|
||||
stride_th,
|
||||
stride_tn,
|
||||
stride_tk,
|
||||
stride_on,
|
||||
stride_oh,
|
||||
stride_od,
|
||||
stride_bt_b,
|
||||
BLOCK_SIZE_Q: tl.constexpr,
|
||||
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
|
||||
BLOCK_SIZE_D: tl.constexpr,
|
||||
BLOCK_SIZE_H: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
BLOCK_SIZE_QH: tl.constexpr,
|
||||
USE_FP8: tl.constexpr, # fp8 KV cache: dequantize K/V to q.dtype on load
|
||||
):
|
||||
sm_scale_log2e = sm_scale * 1.4426950409
|
||||
pid_q = tl.program_id(0)
|
||||
pid_kh = tl.program_id(1)
|
||||
pid_b = tl.program_id(2)
|
||||
pid_h = pid_kh * gqa_group_size
|
||||
q_start = tl.load(cu_seqlens_q + pid_b)
|
||||
q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start
|
||||
q_block_start = tl.load(cu_seqblocks_q + pid_b)
|
||||
q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start
|
||||
seq_len = tl.load(seq_lens + pid_b)
|
||||
prefix_len = tl.load(prefix_lens + pid_b)
|
||||
if pid_q * num_q_loop >= q_block_len:
|
||||
return
|
||||
real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop)
|
||||
bt_row = block_table_ptr + pid_b * stride_bt_b
|
||||
off_n = tl.arange(0, BLOCK_SIZE_K)
|
||||
off_d = tl.arange(0, BLOCK_SIZE_D)
|
||||
d_mask = off_d < head_dim
|
||||
for j in range(real_q_loop):
|
||||
pid_q_j = pid_q * num_q_loop + j
|
||||
t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1)
|
||||
real_topk = tl.sum((topk_idx >= 0).to(tl.int32), axis=0)
|
||||
q_ptrs = tl.make_block_ptr(
|
||||
base=q_ptr + q_start * stride_qn + pid_h * stride_qh,
|
||||
shape=(q_len, gqa_group_size, head_dim),
|
||||
strides=(stride_qn, stride_qh, stride_qd),
|
||||
offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
|
||||
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D),
|
||||
order=(2, 1, 0),
|
||||
)
|
||||
q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero")
|
||||
off_q = (
|
||||
tl.arange(0, BLOCK_SIZE_Q)[:, None]
|
||||
+ pid_q_j * BLOCK_SIZE_Q
|
||||
+ prefix_len
|
||||
- tl.arange(0, BLOCK_SIZE_K)[None, :]
|
||||
)
|
||||
m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
|
||||
lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
|
||||
acc_o = tl.zeros((BLOCK_SIZE_QH, BLOCK_SIZE_D), dtype=tl.float32)
|
||||
q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_D)
|
||||
for _ in range(real_topk):
|
||||
blk = tl.load(t_ptr_j).to(tl.int32)
|
||||
t_ptr_j = t_ptr_j + stride_tk
|
||||
c = blk * BLOCK_SIZE_K
|
||||
page = tl.load(bt_row + blk).to(tl.int64)
|
||||
pos = c + off_n
|
||||
pos_mask = pos < seq_len
|
||||
k = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
+ 0 * stride_kv_kv
|
||||
+ off_n[None, :] * stride_kv_pos
|
||||
+ pid_kh * stride_kv_h
|
||||
+ off_d[:, None] * stride_kv_d,
|
||||
mask=d_mask[:, None] & pos_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if USE_FP8:
|
||||
k = k.to(q.dtype)
|
||||
qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
|
||||
# causal: q_abs_pos - k_off >= block_start (c)
|
||||
qk += tl.where(off_q[:, None, :] >= c, 0, float("-inf"))
|
||||
qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
|
||||
qk += tl.dot(q, k) * sm_scale_log2e
|
||||
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
|
||||
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
|
||||
p = tl.exp2(qk - m_ij[:, None])
|
||||
l_ij = tl.sum(p, axis=1)
|
||||
acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]
|
||||
v = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
+ 1 * stride_kv_kv
|
||||
+ off_n[:, None] * stride_kv_pos
|
||||
+ pid_kh * stride_kv_h
|
||||
+ off_d[None, :] * stride_kv_d,
|
||||
mask=pos_mask[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if USE_FP8:
|
||||
v = v.to(q.dtype)
|
||||
acc_o += tl.dot(p.to(v.dtype), v)
|
||||
m_i = m_ij
|
||||
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
|
||||
acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]
|
||||
acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D)
|
||||
o_ptrs = tl.make_block_ptr(
|
||||
base=o_ptr + q_start * stride_on + pid_h * stride_oh,
|
||||
shape=(q_len, gqa_group_size, head_dim),
|
||||
strides=(stride_on, stride_oh, stride_od),
|
||||
offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
|
||||
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D),
|
||||
order=(2, 1, 0),
|
||||
)
|
||||
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Decode kernels (split-K). Decode batches are flattened request-major, with a
|
||||
# runtime query length used to map each query token back to its request metadata.
|
||||
# This parallelizes over the selected top-k blocks, producing partials that the
|
||||
# merge kernel combines (flash-decoding). All chunk counts depend only on shape
|
||||
# constants so the grid is fixed within a cuda graph. Base-2 (exp2/log2)
|
||||
# softmax matches the prefill kernel.
|
||||
# ---------------------------------------------------------------------------
|
||||
@triton.heuristics(
|
||||
{
|
||||
"BLOCK_SIZE_H": lambda args: max(
|
||||
16, triton.next_power_of_2(args["gqa_group_size"])
|
||||
),
|
||||
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
||||
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
|
||||
}
|
||||
)
|
||||
@triton.jit(do_not_specialize=["decode_query_len"])
|
||||
def _gqa_sparse_decode_kernel(
|
||||
q_ptr, # [total_q, num_heads, head_dim]
|
||||
kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim]
|
||||
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
|
||||
o_ptr, # partial out: [NUM_TOPK_CHUNKS, total_q, num_heads, head_dim]
|
||||
lse_ptr, # partial lse (log2): [NUM_TOPK_CHUNKS, total_q, num_heads]
|
||||
block_table_ptr, # [num_reqs, max_blocks]
|
||||
seq_lens, # [num_reqs]
|
||||
total_q,
|
||||
gqa_group_size,
|
||||
head_dim,
|
||||
max_topk,
|
||||
sm_scale,
|
||||
decode_query_len,
|
||||
stride_qn,
|
||||
stride_qh,
|
||||
stride_qd,
|
||||
stride_kv_blk,
|
||||
stride_kv_kv,
|
||||
stride_kv_pos,
|
||||
stride_kv_h,
|
||||
stride_kv_d,
|
||||
stride_th,
|
||||
stride_tn,
|
||||
stride_tk,
|
||||
stride_o_c,
|
||||
stride_o_b,
|
||||
stride_o_h,
|
||||
stride_o_d,
|
||||
stride_l_c,
|
||||
stride_l_b,
|
||||
stride_l_h,
|
||||
stride_bt_b,
|
||||
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
|
||||
NUM_TOPK_CHUNKS: tl.constexpr,
|
||||
BLOCK_SIZE_H: tl.constexpr,
|
||||
BLOCK_SIZE_D: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
USE_FP8: tl.constexpr, # fp8 KV cache: dequantize K/V to q.dtype on load
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
sm_scale_log2e = sm_scale * 1.4426950409
|
||||
# split-K over the topk dimension: pid(0) folds (query-token, chunk).
|
||||
pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
|
||||
pid_b = pid_bc % total_q
|
||||
pid_c = pid_bc // total_q
|
||||
req_id = pid_b // decode_query_len
|
||||
q_offset = pid_b - req_id * decode_query_len
|
||||
pid_h = pid_kh * gqa_group_size
|
||||
chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
|
||||
chunk_start_topk = pid_c * chunk_size_topk
|
||||
chunk_end_compiletime = chunk_start_topk + chunk_size_topk
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
seq_len = tl.load(seq_lens + req_id)
|
||||
query_pos = seq_len - decode_query_len + q_offset
|
||||
# Full-CG padding uses zero-length request rows. Clamp to an empty
|
||||
# attention range instead of letting padded rows produce negative lengths.
|
||||
kv_len = tl.maximum(query_pos + 1, 0)
|
||||
|
||||
# number of valid (non-padded) selected blocks for this query token
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
idx_base = t_ptr + pid_kh * stride_th + pid_b * stride_tn
|
||||
topk_idx = tl.load(idx_base + off_t * stride_tk, mask=off_t < max_topk, other=-1)
|
||||
real_topk = tl.sum((topk_idx >= 0).to(tl.int32), axis=0)
|
||||
chunk_end_topk = tl.minimum(chunk_end_compiletime, real_topk)
|
||||
|
||||
off_n = tl.arange(0, BLOCK_SIZE_K)
|
||||
off_d = tl.arange(0, BLOCK_SIZE_D)
|
||||
d_mask = off_d < head_dim
|
||||
bt_row = block_table_ptr + req_id * stride_bt_b
|
||||
|
||||
m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
||||
lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
||||
acc_o = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_D), dtype=tl.float32)
|
||||
q_ptrs = tl.make_block_ptr(
|
||||
base=q_ptr + pid_b * stride_qn + pid_h * stride_qh,
|
||||
shape=(gqa_group_size, head_dim),
|
||||
strides=(stride_qh, stride_qd),
|
||||
offsets=(0, 0),
|
||||
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
||||
order=(1, 0),
|
||||
)
|
||||
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
|
||||
|
||||
cur_idx_ptr = idx_base + chunk_start_topk * stride_tk
|
||||
for _ in tl.range(chunk_start_topk, chunk_end_topk):
|
||||
blk = tl.load(cur_idx_ptr).to(tl.int32)
|
||||
cur_idx_ptr = cur_idx_ptr + stride_tk
|
||||
c = blk * BLOCK_SIZE_K
|
||||
page = tl.load(bt_row + blk).to(tl.int64)
|
||||
pos = c + off_n
|
||||
pos_mask = pos < kv_len
|
||||
k = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
+ 0 * stride_kv_kv
|
||||
+ off_n[None, :] * stride_kv_pos
|
||||
+ pid_kh * stride_kv_h
|
||||
+ off_d[:, None] * stride_kv_d,
|
||||
mask=d_mask[:, None] & pos_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if USE_FP8:
|
||||
k = k.to(q.dtype)
|
||||
qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
|
||||
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
|
||||
qk += tl.dot(q, k) * sm_scale_log2e
|
||||
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
|
||||
p = tl.exp2(qk - m_ij[:, None])
|
||||
l_ij = tl.sum(p, axis=1)
|
||||
acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]
|
||||
v = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
+ 1 * stride_kv_kv
|
||||
+ off_n[:, None] * stride_kv_pos
|
||||
+ pid_kh * stride_kv_h
|
||||
+ off_d[None, :] * stride_kv_d,
|
||||
mask=pos_mask[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if USE_FP8:
|
||||
v = v.to(q.dtype)
|
||||
acc_o += tl.dot(p.to(v.dtype), v)
|
||||
m_i = m_ij
|
||||
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
# Empty chunks for active rows must store zero output; otherwise the merge
|
||||
# can hit 0 * NaN. All-empty padded rows may still produce NaNs in merge.
|
||||
scale = tl.where(lse_i > float("-inf"), tl.exp2(m_i - lse_i), tl.zeros_like(lse_i))
|
||||
acc_o = acc_o * scale[:, None]
|
||||
o_ptrs = tl.make_block_ptr(
|
||||
base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
|
||||
shape=(gqa_group_size, head_dim),
|
||||
strides=(stride_o_h, stride_o_d),
|
||||
offsets=(0, 0),
|
||||
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
||||
order=(1, 0),
|
||||
)
|
||||
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
|
||||
lse_ptrs = tl.make_block_ptr(
|
||||
base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
|
||||
shape=(gqa_group_size,),
|
||||
strides=(stride_l_h,),
|
||||
offsets=(0,),
|
||||
block_shape=(BLOCK_SIZE_H,),
|
||||
order=(0,),
|
||||
)
|
||||
tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"])}
|
||||
)
|
||||
@triton.jit
|
||||
def _merge_topk_attn_out_kernel(
|
||||
o_ptr, # partials: [NUM_TOPK_CHUNKS, total_q, num_heads, head_dim]
|
||||
lse_ptr, # partials (log2): [NUM_TOPK_CHUNKS, total_q, num_heads]
|
||||
out_ptr, # merged out: [total_q, num_heads, head_dim]
|
||||
head_dim,
|
||||
stride_o_c,
|
||||
stride_o_b,
|
||||
stride_o_h,
|
||||
stride_o_d,
|
||||
stride_l_c,
|
||||
stride_l_b,
|
||||
stride_l_h,
|
||||
stride_out_n,
|
||||
stride_out_h,
|
||||
stride_out_d,
|
||||
NUM_TOPK_CHUNKS: tl.constexpr,
|
||||
BLOCK_SIZE_D: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
pid_b, pid_h = tl.program_id(0), tl.program_id(1)
|
||||
|
||||
# NOTE: assume seq_lens is safe to load before gdc_wait()
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
off_c = tl.arange(0, NUM_TOPK_CHUNKS)
|
||||
off_d = tl.arange(0, BLOCK_SIZE_D)
|
||||
o_ptrs = tl.make_block_ptr(
|
||||
base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
|
||||
shape=(NUM_TOPK_CHUNKS, head_dim),
|
||||
strides=(stride_o_c, stride_o_d),
|
||||
offsets=(0, 0),
|
||||
block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D),
|
||||
order=(1, 0),
|
||||
)
|
||||
lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
|
||||
o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
|
||||
lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0
|
||||
lse_max = tl.max(lse, axis=0)
|
||||
weights = tl.exp2(lse - lse_max)
|
||||
weights = weights / tl.sum(weights, axis=0)
|
||||
o_merged = tl.sum(o * weights[:, None], axis=0)
|
||||
out_ptrs = (
|
||||
out_ptr + pid_b * stride_out_n + pid_h * stride_out_h + off_d * stride_out_d
|
||||
)
|
||||
tl.store(out_ptrs, o_merged.to(out_ptr.dtype.element_ty), mask=off_d < head_dim)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Python wrappers
|
||||
# ---------------------------------------------------------------------------
|
||||
@torch.no_grad()
|
||||
def minimax_m3_sparse_attn(
|
||||
q: torch.Tensor, # [total_q, num_heads, head_dim]
|
||||
kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim]
|
||||
topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
|
||||
block_table: torch.Tensor, # [batch, max_blocks]
|
||||
cu_seqlens_q: torch.Tensor, # [batch+1] int32
|
||||
seq_lens: torch.Tensor, # [batch] int32
|
||||
prefix_lens: torch.Tensor, # [batch] int32
|
||||
max_query_len: int,
|
||||
num_kv_heads: int,
|
||||
sm_scale: float,
|
||||
output: torch.Tensor, # [total_q, num_heads, head_dim]
|
||||
) -> None:
|
||||
"""GQA block-sparse attention over the selected blocks. block_size_q == 1."""
|
||||
total_q, num_heads, head_dim = q.shape
|
||||
batch = cu_seqlens_q.shape[0] - 1
|
||||
topk = topk_idx.shape[-1]
|
||||
gqa_group_size = num_heads // num_kv_heads
|
||||
use_fp8 = kv_cache.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
|
||||
grid = (max_query_len, num_kv_heads, batch)
|
||||
_gqa_sparse_fwd_kernel[grid](
|
||||
q,
|
||||
kv_cache,
|
||||
topk_idx,
|
||||
output,
|
||||
block_table,
|
||||
cu_seqlens_q,
|
||||
cu_seqlens_q, # cu_seqblocks_q == cu_seqlens_q when block_size_q == 1
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
num_kv_heads,
|
||||
gqa_group_size,
|
||||
head_dim,
|
||||
topk,
|
||||
1, # num_q_loop
|
||||
sm_scale,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
kv_cache.stride(0),
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
kv_cache.stride(3),
|
||||
kv_cache.stride(4),
|
||||
topk_idx.stride(0),
|
||||
topk_idx.stride(1),
|
||||
topk_idx.stride(2),
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
output.stride(2),
|
||||
block_table.stride(0),
|
||||
BLOCK_SIZE_Q=1,
|
||||
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
|
||||
USE_FP8=use_fp8,
|
||||
**_sparse_attn_num_stages_kwarg(),
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def minimax_m3_sparse_attn_decode(
|
||||
q: torch.Tensor, # [total_q, num_heads, head_dim]
|
||||
kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim]
|
||||
topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
|
||||
block_table: torch.Tensor, # [num_reqs, max_blocks]
|
||||
seq_lens: torch.Tensor, # [num_reqs] int32
|
||||
num_kv_heads: int,
|
||||
sm_scale: float,
|
||||
output: torch.Tensor, # [total_q, num_heads, head_dim]
|
||||
decode_query_len: int,
|
||||
) -> None:
|
||||
"""GQA block-sparse attention for decode (split-K over the top-k blocks)."""
|
||||
total_q, num_heads, head_dim = q.shape
|
||||
assert total_q == seq_lens.shape[0] * decode_query_len
|
||||
max_topk = topk_idx.shape[-1]
|
||||
gqa_group_size = num_heads // num_kv_heads
|
||||
use_fp8 = kv_cache.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
|
||||
use_pdl = current_platform.is_arch_support_pdl()
|
||||
# `launch_pdl` is a Triton runtime kwarg only some backends accept (CUDA
|
||||
# SM9+); this ROCm Triton rejects it even when False ("Keyword argument
|
||||
# launch_pdl was specified but unrecognised"). Only pass it when PDL is
|
||||
# actually supported -- on ROCm use_pdl is always False, so it's omitted.
|
||||
pdl_launch = {"launch_pdl": True} if use_pdl else {}
|
||||
# split-K over the selected blocks; chunk count is shape-constant (cuda graph).
|
||||
TARGET_GRID = 256
|
||||
target = max(1, min(max_topk, TARGET_GRID // max(1, total_q * num_kv_heads)))
|
||||
num_topk_chunks = 1 << (target.bit_length() - 1)
|
||||
o_partial = torch.empty(
|
||||
num_topk_chunks, total_q, num_heads, head_dim, dtype=q.dtype, device=q.device
|
||||
)
|
||||
lse_partial = torch.empty(
|
||||
num_topk_chunks, total_q, num_heads, dtype=torch.float32, device=q.device
|
||||
)
|
||||
grid = (total_q * num_topk_chunks, num_kv_heads)
|
||||
_gqa_sparse_decode_kernel[grid](
|
||||
q,
|
||||
kv_cache,
|
||||
topk_idx,
|
||||
o_partial,
|
||||
lse_partial,
|
||||
block_table,
|
||||
seq_lens,
|
||||
total_q,
|
||||
gqa_group_size,
|
||||
head_dim,
|
||||
max_topk,
|
||||
sm_scale,
|
||||
decode_query_len,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
kv_cache.stride(0),
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
kv_cache.stride(3),
|
||||
kv_cache.stride(4),
|
||||
topk_idx.stride(0),
|
||||
topk_idx.stride(1),
|
||||
topk_idx.stride(2),
|
||||
o_partial.stride(0),
|
||||
o_partial.stride(1),
|
||||
o_partial.stride(2),
|
||||
o_partial.stride(3),
|
||||
lse_partial.stride(0),
|
||||
lse_partial.stride(1),
|
||||
lse_partial.stride(2),
|
||||
block_table.stride(0),
|
||||
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
|
||||
NUM_TOPK_CHUNKS=num_topk_chunks,
|
||||
USE_FP8=use_fp8,
|
||||
USE_PDL=use_pdl,
|
||||
**_sparse_attn_num_stages_kwarg(),
|
||||
**pdl_launch,
|
||||
)
|
||||
merge_grid = (total_q, num_heads)
|
||||
_merge_topk_attn_out_kernel[merge_grid](
|
||||
o_partial,
|
||||
lse_partial,
|
||||
output,
|
||||
head_dim,
|
||||
o_partial.stride(0),
|
||||
o_partial.stride(1),
|
||||
o_partial.stride(2),
|
||||
o_partial.stride(3),
|
||||
lse_partial.stride(0),
|
||||
lse_partial.stride(1),
|
||||
lse_partial.stride(2),
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
output.stride(2),
|
||||
NUM_TOPK_CHUNKS=num_topk_chunks,
|
||||
USE_PDL=use_pdl,
|
||||
**pdl_launch,
|
||||
)
|
||||
@@ -0,0 +1,398 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Main block-sparse GQA attention for MiniMax M3 sparse layers.
|
||||
|
||||
The lightning indexer (``indexer.py``) selects the top-k KV blocks; this module
|
||||
holds the main attention that attends only to those blocks: the paged K/V cache
|
||||
backend, its metadata + builder, and the impl that consumes the indexer's
|
||||
``topk_idx``. The Triton attend kernel lives here; the SM100 (MSA)
|
||||
``build_k2q_csr`` + ``sparse_atten_func`` attend lives in
|
||||
``nvidia/sparse_attention_msa.py``.
|
||||
|
||||
``MiniMaxM3SparseBackend`` and ``MiniMaxM3SparseMetadata`` are referenced by the
|
||||
attention-backend registry (by dotted path) and by spec-decode, so they must
|
||||
keep these names and stay in this module.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
SPARSE_BLOCK_SIZE,
|
||||
minimax_m3_sparse_attn,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionImplBase,
|
||||
AttentionLayer,
|
||||
AttentionMetadata,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_kv_cache_layout,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
|
||||
|
||||
|
||||
class MiniMaxM3SparseBackend(AttentionBackend):
|
||||
"""Block-sparse GQA backend for MiniMax M3 sparse attention layers."""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16, torch.float16]
|
||||
# bf16 or fp8 (e4m3/e5m2): the Triton kernels dequant fp8 before the dots.
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
"fp8_e5m2",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "MINIMAX_M3_SPARSE"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["MiniMaxM3SparseImpl"]:
|
||||
# Concrete impl chosen by select_main_impl_cls; base for introspection.
|
||||
return MiniMaxM3SparseImpl
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["MiniMaxM3SparseMetadataBuilder"]:
|
||||
return MiniMaxM3SparseMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [128]
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
# Page size == sparse block size (one sparse block per KV page).
|
||||
return [128]
|
||||
|
||||
@classmethod
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# Permutation from get_kv_cache_shape to the actual memory layout.
|
||||
if include_num_layers_dimension:
|
||||
raise NotImplementedError # no cross-layer KV blocks in M3
|
||||
cache_layout = get_kv_cache_layout()
|
||||
if cache_layout == "NHD":
|
||||
stride_order = (0, 1, 2, 3, 4)
|
||||
elif cache_layout == "HND":
|
||||
stride_order = (0, 1, 3, 2, 4)
|
||||
else:
|
||||
raise ValueError(f"Unknown cache layout format {cache_layout}.")
|
||||
return stride_order
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxM3SparsePrefillMetadata:
|
||||
"""Per-prefill state; ``cu_seqlens_k``/``total_kv_blocks`` feed the MSA CSR."""
|
||||
|
||||
cu_seqlens_q: torch.Tensor # [num_prefills + 1] int32, rebased to 0
|
||||
cu_seqlens_k: torch.Tensor # [num_prefills + 1] int32, cumulative KV lengths
|
||||
seq_lens: torch.Tensor # [num_prefills] int32, total KV lengths
|
||||
context_lens: torch.Tensor # [num_prefills] int32 (cached/context tokens)
|
||||
block_table: torch.Tensor
|
||||
max_query_len: int
|
||||
max_seq_len: int
|
||||
total_kv_blocks: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxM3SparseDecodeMetadata:
|
||||
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
|
||||
per-request query length (1, or 1 + num_speculative_tokens)."""
|
||||
|
||||
seq_lens: torch.Tensor # [num_decodes] int32
|
||||
block_table: torch.Tensor
|
||||
decode_query_len: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxM3SparseMetadata(AttentionMetadata):
|
||||
"""Sparse-attention metadata, split into prefill and decode sub-metadata."""
|
||||
|
||||
seq_lens: torch.Tensor
|
||||
max_seq_len: int
|
||||
slot_mapping: torch.Tensor
|
||||
|
||||
num_actual_tokens: int # total query tokens (decode-first batch)
|
||||
|
||||
# Split counts (batch reordered decode-first).
|
||||
num_decodes: int
|
||||
num_decode_tokens: int
|
||||
num_prefills: int
|
||||
num_prefill_tokens: int
|
||||
|
||||
prefill: MiniMaxM3SparsePrefillMetadata | None = None
|
||||
decode: MiniMaxM3SparseDecodeMetadata | None = None
|
||||
|
||||
|
||||
class MiniMaxM3SparseMetadataBuilder(AttentionMetadataBuilder[MiniMaxM3SparseMetadata]):
|
||||
# Full cudagraphs for uniform decode batches, incl. spec-decode verify
|
||||
# batches with >1 query token/request.
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
# Raised to 1 + num_speculative_tokens by _init_reorder_batch_threshold when
|
||||
# spec decode is on; must match the indexer builder so the splits agree.
|
||||
reorder_batch_threshold: int = 1
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
||||
# Stable context-length buffer for decode cudagraph replays.
|
||||
self.context_len_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> MiniMaxM3SparseMetadata:
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
query_start_loc = common_attn_metadata.query_start_loc
|
||||
seq_lens = common_attn_metadata.seq_lens
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
require_uniform=True,
|
||||
)
|
||||
)
|
||||
assert num_decodes + num_prefills == num_reqs
|
||||
assert num_decode_tokens + num_prefill_tokens == num_tokens
|
||||
|
||||
# Decode-first batch: context lengths into the stable cudagraph buffer.
|
||||
context_lens = self.context_len_buffer[:num_reqs]
|
||||
context_lens.copy_(
|
||||
common_attn_metadata.compute_num_computed_tokens(), non_blocking=True
|
||||
)
|
||||
|
||||
prefill_metadata: MiniMaxM3SparsePrefillMetadata | None = None
|
||||
if num_prefills > 0:
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
prefill_seq_lens_cpu = seq_lens_cpu[num_decodes:]
|
||||
prefill_total_kv_blocks = (
|
||||
((prefill_seq_lens_cpu + SPARSE_BLOCK_SIZE - 1) // SPARSE_BLOCK_SIZE)
|
||||
.sum()
|
||||
.item()
|
||||
)
|
||||
prefill_kv_lens = seq_lens[num_decodes:]
|
||||
prefill_cu_seqlens_k = torch.empty(
|
||||
num_prefills + 1, dtype=torch.int32, device=seq_lens.device
|
||||
)
|
||||
prefill_cu_seqlens_k[0] = 0
|
||||
torch.cumsum(prefill_kv_lens, dim=0, out=prefill_cu_seqlens_k[1:])
|
||||
prefill_metadata = MiniMaxM3SparsePrefillMetadata(
|
||||
cu_seqlens_q=(query_start_loc[num_decodes:] - num_decode_tokens).to(
|
||||
torch.int32
|
||||
),
|
||||
cu_seqlens_k=prefill_cu_seqlens_k,
|
||||
seq_lens=prefill_kv_lens,
|
||||
context_lens=context_lens[num_decodes:],
|
||||
block_table=block_table[num_decodes:],
|
||||
max_query_len=common_attn_metadata.max_query_len,
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
total_kv_blocks=prefill_total_kv_blocks,
|
||||
)
|
||||
|
||||
decode_metadata: MiniMaxM3SparseDecodeMetadata | None = None
|
||||
if num_decodes > 0:
|
||||
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
query_lens_cpu = qsl_cpu[1 : num_decodes + 1] - qsl_cpu[:num_decodes]
|
||||
decode_query_len = int(query_lens_cpu[0].item())
|
||||
assert decode_query_len > 0
|
||||
assert torch.all(
|
||||
(query_lens_cpu == decode_query_len) | (query_lens_cpu == 0)
|
||||
)
|
||||
assert num_decode_tokens == num_decodes * decode_query_len
|
||||
decode_metadata = MiniMaxM3SparseDecodeMetadata(
|
||||
seq_lens=seq_lens[:num_decodes],
|
||||
block_table=block_table[:num_decodes],
|
||||
decode_query_len=decode_query_len,
|
||||
)
|
||||
|
||||
return MiniMaxM3SparseMetadata(
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
slot_mapping=common_attn_metadata.slot_mapping,
|
||||
num_actual_tokens=num_tokens,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefills=num_prefills,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
prefill=prefill_metadata,
|
||||
decode=decode_metadata,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3SparseImpl(AttentionImplBase[MiniMaxM3SparseMetadata]):
|
||||
"""Abstract base for block-sparse GQA over the indexer-selected blocks.
|
||||
|
||||
Inherits ``AttentionImplBase`` for a custom forward signature (the layer
|
||||
pre-inserts K/V and runs the indexer, so forward takes the queries +
|
||||
``topk_idx``). The Triton and MSA subclasses each own a full ``forward`` --
|
||||
no shared forward code.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int | None = None,
|
||||
kv_cache_dtype: str = "auto",
|
||||
*,
|
||||
topk_blocks: int,
|
||||
sparse_block_size: int,
|
||||
) -> None:
|
||||
self.num_heads = num_heads
|
||||
self.head_size = head_size
|
||||
self.scale = scale
|
||||
self.num_kv_heads = num_kv_heads if num_kv_heads is not None else num_heads
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.use_fp8_kv = is_quantized_kv_cache(kv_cache_dtype)
|
||||
self.kv_cache_fp8_dtype = (
|
||||
torch.float8_e5m2 if "e5m2" in kv_cache_dtype else torch.float8_e4m3fn
|
||||
)
|
||||
# Sparse selection parameters (block_size == page size == SPARSE_BLOCK_SIZE).
|
||||
self.topk_blocks = topk_blocks
|
||||
self.block_size = sparse_block_size
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
topk_idx: tuple[torch.Tensor | None, torch.Tensor | None],
|
||||
output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Attend the queries to the indexer-selected blocks. Per kernel."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
|
||||
"""Triton block-sparse attend (``minimax_m3_sparse_attn``) + Triton decode."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
topk_idx: tuple[torch.Tensor | None, torch.Tensor | None],
|
||||
output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return output # profiling run; caches unbound
|
||||
main_md = attn_metadata[layer.layer_name] # type: ignore[attr-defined]
|
||||
assert isinstance(main_md, MiniMaxM3SparseMetadata)
|
||||
decode_topk, prefill_topk = topk_idx
|
||||
|
||||
nd = main_md.num_decode_tokens
|
||||
num_tokens = main_md.num_actual_tokens
|
||||
hd = self.head_size
|
||||
q = query[:num_tokens].view(-1, self.num_heads, hd)
|
||||
out = output[:num_tokens].view(-1, self.num_heads, hd)
|
||||
kv_cache = (
|
||||
kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache
|
||||
)
|
||||
|
||||
# Decode [:nd]: split-K over the selected blocks (request-major chunks).
|
||||
if main_md.num_decodes > 0:
|
||||
d = main_md.decode
|
||||
assert d is not None and decode_topk is not None
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q[:nd],
|
||||
kv_cache,
|
||||
decode_topk,
|
||||
d.block_table,
|
||||
d.seq_lens,
|
||||
self.num_kv_heads,
|
||||
self.scale,
|
||||
out[:nd],
|
||||
d.decode_query_len,
|
||||
)
|
||||
|
||||
# Prefill [nd:]: cu_seqlens_q already rebased to 0.
|
||||
if main_md.num_prefills > 0:
|
||||
p = main_md.prefill
|
||||
assert p is not None and prefill_topk is not None
|
||||
minimax_m3_sparse_attn(
|
||||
q[nd:],
|
||||
kv_cache,
|
||||
prefill_topk,
|
||||
p.block_table,
|
||||
p.cu_seqlens_q,
|
||||
p.seq_lens,
|
||||
p.context_lens,
|
||||
p.max_query_len,
|
||||
self.num_kv_heads,
|
||||
self.scale,
|
||||
out[nd:],
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def select_main_impl_cls(
|
||||
*,
|
||||
topk_blocks: int,
|
||||
kv_cache_dtype: str,
|
||||
) -> type[MiniMaxM3SparseImpl]:
|
||||
"""Pick the main attend impl off the main KV-cache dtype.
|
||||
|
||||
bf16 on Blackwell (SM100) uses the MSA attend; fp8 or non-Blackwell falls
|
||||
back to Triton. The MSA module is imported lazily so AMD/non-SM100 never
|
||||
import fmha_sm100.
|
||||
"""
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and topk_blocks in (4, 8, 16, 32)
|
||||
and not is_quantized_kv_cache(kv_cache_dtype)
|
||||
):
|
||||
from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
|
||||
MiniMaxM3SparseMSAImpl,
|
||||
)
|
||||
|
||||
return MiniMaxM3SparseMSAImpl
|
||||
return MiniMaxM3SparseTritonImpl
|
||||
@@ -0,0 +1,765 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.distributed import parallel_state
|
||||
from vllm.distributed import utils as dist_utils
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.vision import (
|
||||
get_vit_attn_backend,
|
||||
is_vit_use_data_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# ROCm caps a kernel-launch gridDim.y at 65536. The HIP flash-attn Triton
|
||||
# rotary kernel launches grid.y = cdiv(seqlen, BLOCK_M), so it fails with
|
||||
# hipErrorInvalidValue once cdiv(seqlen, BLOCK_M) > 65536. Used below to decide
|
||||
# when RoPE must be applied per video segment instead of in one launch.
|
||||
_HIP_MAX_GRID_DIM_Y = 65536
|
||||
|
||||
|
||||
class MiniMaxVLPatchEmbed(nn.Module):
|
||||
"""Conv3d-based patch embedding.
|
||||
|
||||
Takes flat tokens of shape (N, C * temporal_patch_size * patch_size²)
|
||||
and projects each to a hidden-size embedding.
|
||||
"""
|
||||
|
||||
def __init__(self, config: PretrainedConfig) -> None:
|
||||
super().__init__()
|
||||
compression = config.img_token_compression_config
|
||||
temporal_patch_size = compression.get("temporal_patch_size", 2)
|
||||
patch_size = config.patch_size
|
||||
num_channels = config.num_channels
|
||||
|
||||
self.patch_size = patch_size
|
||||
self.temporal_patch_size = temporal_patch_size
|
||||
self.num_channels = num_channels
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.patch_embedding = nn.Conv3d(
|
||||
in_channels=num_channels,
|
||||
out_channels=config.hidden_size,
|
||||
kernel_size=(temporal_patch_size, patch_size, patch_size),
|
||||
stride=(temporal_patch_size, patch_size, patch_size),
|
||||
bias=False,
|
||||
)
|
||||
|
||||
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
|
||||
# pixel_values: (N, C * temporal_patch_size * patch_size²)
|
||||
if self.patch_embedding.weight.dtype != pixel_values.dtype:
|
||||
self.patch_embedding = self.patch_embedding.to(pixel_values.dtype)
|
||||
x = pixel_values.reshape(
|
||||
pixel_values.shape[0],
|
||||
self.num_channels,
|
||||
self.temporal_patch_size,
|
||||
self.patch_size,
|
||||
self.patch_size,
|
||||
)
|
||||
return self.patch_embedding(x).reshape(x.shape[0], -1)
|
||||
|
||||
|
||||
class MiniMaxVLAttention(nn.Module):
|
||||
"""Multi-head attention with MiniMax's partial 3D RoPE.
|
||||
|
||||
Partial means only the first ``rot_dim`` (< head_dim) dimensions of
|
||||
Q and K are rotated; the remaining dims are passed through unchanged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
embed_dim: int,
|
||||
num_heads: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
use_data_parallel = is_vit_use_data_parallel()
|
||||
self.tp_size = (
|
||||
1
|
||||
if use_data_parallel
|
||||
else parallel_state.get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.head_dim = embed_dim // num_heads
|
||||
self.num_heads_per_partition = dist_utils.divide(num_heads, self.tp_size)
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=embed_dim,
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=num_heads,
|
||||
total_num_kv_heads=num_heads,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.out_proj = RowParallelLinear(
|
||||
input_size=embed_dim,
|
||||
output_size=embed_dim,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.attn = MMEncoderAttention(
|
||||
num_heads=self.num_heads_per_partition,
|
||||
head_size=self.head_dim,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
# ApplyRotaryEmb handles the internal cos/sin repeat and partial
|
||||
# rotation (ro_dim = half_rot_dim * 2 < head_dim for MiniMax).
|
||||
# enable_fp32_compute=True runs the rotation in fp32 (q/k upcast,
|
||||
# fp32 cos/sin), matching the reference ``_minimax_rope_applier``.
|
||||
self.apply_rotary_emb = ApplyRotaryEmb(
|
||||
enforce_enable=True, enable_fp32_compute=True
|
||||
)
|
||||
|
||||
def _apply_rotary_emb(
|
||||
self,
|
||||
qk_reshaped: torch.Tensor,
|
||||
rotary_cos: torch.Tensor,
|
||||
rotary_sin: torch.Tensor,
|
||||
seq_len: int,
|
||||
rotary_segment_lengths: list[int] | None,
|
||||
) -> torch.Tensor:
|
||||
# Default fast path (all NVIDIA inputs, and ROCm short clips/images):
|
||||
# a single rotary kernel launch. ``rotary_segment_lengths`` is only
|
||||
# populated on ROCm (see ``MiniMaxVLVisionTransformer.forward``), so
|
||||
# the per-segment path below is ROCm-only and never touches the
|
||||
# NVIDIA/CUDA code path.
|
||||
if not current_platform.is_rocm() or rotary_segment_lengths is None:
|
||||
return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin)
|
||||
|
||||
# ROCm only: the HIP flash-attn Triton rotary kernel fails with
|
||||
# hipErrorInvalidValue once grid.y = cdiv(seqlen, BLOCK_M) exceeds
|
||||
# _HIP_MAX_GRID_DIM_Y (65536). BLOCK_M is 8 for rotary_dim <= 128
|
||||
# (MiniMax-M3 vision: rotary_dim=78), giving a hard limit of
|
||||
# 65536 * BLOCK_M tokens — measured exactly as 524288 OK / 524289 fail.
|
||||
# Only long videos cross it; since vision_segment_max_frames caps each
|
||||
# segment at a few frames (<< limit), applying RoPE per segment keeps
|
||||
# every sub-call in range. Splitting on segment boundaries is
|
||||
# mathematically exact because rotary_cos/sin are precomputed per token.
|
||||
# Images and short clips stay on the single-kernel fast path above.
|
||||
rotary_dim = rotary_cos.shape[-1] * 2
|
||||
block_m = 8 if rotary_dim <= 128 else 4
|
||||
hip_rotary_max_seqlen = _HIP_MAX_GRID_DIM_Y * block_m
|
||||
if seq_len <= hip_rotary_max_seqlen or len(rotary_segment_lengths) <= 1:
|
||||
return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin)
|
||||
|
||||
qk_segments = qk_reshaped.split(rotary_segment_lengths, dim=1)
|
||||
cos_segments = rotary_cos.split(rotary_segment_lengths, dim=0)
|
||||
sin_segments = rotary_sin.split(rotary_segment_lengths, dim=0)
|
||||
return torch.cat(
|
||||
[
|
||||
self.apply_rotary_emb(qk_s, cos_s, sin_s)
|
||||
for qk_s, cos_s, sin_s in zip(qk_segments, cos_segments, sin_segments)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
rotary_cos: torch.Tensor,
|
||||
rotary_sin: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
rotary_segment_lengths: list[int] | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# x: (N, 1, embed_dim) [seq=N, batch=1, chan=embed_dim]
|
||||
x_qkv, _ = self.qkv_proj(x) # (N, 1, 3 * heads_per_part * head_dim)
|
||||
seq_len, batch_size, _ = x_qkv.shape
|
||||
|
||||
# Rearrange to (b=1, N, 3, heads, head_dim) — same as Qwen2_5_VisionAttention
|
||||
qkv = rearrange(
|
||||
x_qkv,
|
||||
"s b (three head d) -> b s three head d",
|
||||
three=3,
|
||||
head=self.num_heads_per_partition,
|
||||
)
|
||||
qk, v = qkv[:, :, :2], qkv[:, :, 2] # (b,N,2,h,d) and (b,N,h,d)
|
||||
|
||||
# Stack q/k → (2*b, N, heads, head_dim) for joint RoPE application.
|
||||
# rotary_cos/sin: (N, half_rot_dim) — ApplyRotaryEmb expands internally
|
||||
# and rotates only the first 2*half_rot_dim dims, passing the rest through.
|
||||
qk_reshaped = rearrange(qk, "b s two h d -> (two b) s h d", two=2).contiguous()
|
||||
qk_rotated = self._apply_rotary_emb(
|
||||
qk_reshaped, rotary_cos, rotary_sin, seq_len, rotary_segment_lengths
|
||||
)
|
||||
qk_rotated = qk_rotated.view(
|
||||
2, batch_size, seq_len, self.num_heads_per_partition, self.head_dim
|
||||
)
|
||||
q, k = qk_rotated.unbind(dim=0) # each (b=1, N, heads, head_dim)
|
||||
|
||||
# Flash attention → (b, N, heads, head_dim)
|
||||
context = self.attn(
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
|
||||
# Back to (N, 1, embed_dim)
|
||||
context = rearrange(context, "b s h d -> s b (h d)", b=batch_size)
|
||||
output, _ = self.out_proj(context)
|
||||
return output
|
||||
|
||||
|
||||
class MiniMaxVLEncoderLayer(nn.Module):
|
||||
"""Single CLIP-style transformer block."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
embed_dim = config.hidden_size
|
||||
self.layer_norm1 = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
||||
self.self_attn = MiniMaxVLAttention(
|
||||
embed_dim=embed_dim,
|
||||
num_heads=config.num_attention_heads,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
self.layer_norm2 = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
||||
use_data_parallel = is_vit_use_data_parallel()
|
||||
self.fc1 = ColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
config.intermediate_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fc1",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.act = get_act_fn(getattr(config, "hidden_act", "gelu"))
|
||||
self.fc2 = RowParallelLinear(
|
||||
config.intermediate_size,
|
||||
config.hidden_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fc2",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
rotary_cos: torch.Tensor,
|
||||
rotary_sin: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
rotary_segment_lengths: list[int] | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# x: (N, 1, hidden_size)
|
||||
x = x + self.self_attn(
|
||||
self.layer_norm1(x),
|
||||
cu_seqlens,
|
||||
rotary_cos,
|
||||
rotary_sin,
|
||||
max_seqlen,
|
||||
rotary_segment_lengths,
|
||||
sequence_lengths,
|
||||
)
|
||||
residual = x
|
||||
x, _ = self.fc1(self.layer_norm2(x))
|
||||
x = self.act(x)
|
||||
x, _ = self.fc2(x)
|
||||
return residual + x
|
||||
|
||||
|
||||
class MiniMaxVLEncoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
num_hidden_layers_override: int | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
n = (
|
||||
config.num_hidden_layers
|
||||
if num_hidden_layers_override is None
|
||||
else num_hidden_layers_override
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
MiniMaxVLEncoderLayer(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.layers.{i}",
|
||||
)
|
||||
for i in range(n)
|
||||
]
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
rotary_cos: torch.Tensor,
|
||||
rotary_sin: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
rotary_segment_lengths: list[int] | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
for layer in self.layers:
|
||||
x = layer(
|
||||
x,
|
||||
cu_seqlens,
|
||||
rotary_cos,
|
||||
rotary_sin,
|
||||
max_seqlen,
|
||||
rotary_segment_lengths,
|
||||
sequence_lengths,
|
||||
)
|
||||
return x
|
||||
|
||||
|
||||
class MiniMaxVLVisionTransformer(nn.Module):
|
||||
"""CLIP-based ViT with 3D RoPE (t/h/w decomposed).
|
||||
|
||||
Faithfully mirrors the reference ``MiniMaxVLVisionTransformer``.
|
||||
FLASHINFER backend is not supported; standard flash-attn is used.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
num_hidden_layers_override: int | None = None,
|
||||
require_post_norm: bool | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
compression = config.img_token_compression_config
|
||||
self.spatial_merge_size: int = compression.get("spatial_merge_size", 2)
|
||||
self.temporal_patch_size: int = compression.get("temporal_patch_size", 2)
|
||||
self.vision_segment_max_frames: int | None = getattr(
|
||||
config, "vision_segment_max_frames", None
|
||||
)
|
||||
self.use_data_parallel = is_vit_use_data_parallel()
|
||||
|
||||
embed_dim = config.hidden_size
|
||||
head_dim = embed_dim // config.num_attention_heads
|
||||
# Backend selection + sharding info for building encoder metadata.
|
||||
# Defaults to FLASH_ATTN on SM80+; --mm-encoder-attn-backend FLASHINFER
|
||||
# selects the cuDNN ViT prefill path.
|
||||
self.hidden_size = embed_dim
|
||||
self.tp_size = (
|
||||
1
|
||||
if self.use_data_parallel
|
||||
else parallel_state.get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.attn_backend = get_vit_attn_backend(
|
||||
head_size=head_dim, dtype=torch.get_default_dtype()
|
||||
)
|
||||
rope_dims = 2 * (head_dim // 2)
|
||||
|
||||
# Split rope dims evenly across t/h/w (same formula as the reference)
|
||||
self.t_dim = int(2 * ((rope_dims // 3) // 2))
|
||||
self.h_dim = int(2 * ((rope_dims // 3) // 2))
|
||||
self.w_dim = int(2 * ((rope_dims // 3) // 2))
|
||||
# rot_dim = t_dim + h_dim + w_dim (may be < head_dim)
|
||||
|
||||
rope_theta: float = getattr(config, "rope_theta", 10000.0)
|
||||
inv_freq_t = 1.0 / (
|
||||
rope_theta
|
||||
** (torch.arange(0, self.t_dim, 2, dtype=torch.float32) / self.t_dim)
|
||||
)
|
||||
inv_freq_h = 1.0 / (
|
||||
rope_theta
|
||||
** (torch.arange(0, self.h_dim, 2, dtype=torch.float32) / self.h_dim)
|
||||
)
|
||||
inv_freq_w = 1.0 / (
|
||||
rope_theta
|
||||
** (torch.arange(0, self.w_dim, 2, dtype=torch.float32) / self.w_dim)
|
||||
)
|
||||
self.register_buffer("inv_freq_t", inv_freq_t, persistent=False)
|
||||
self.register_buffer("inv_freq_h", inv_freq_h, persistent=False)
|
||||
self.register_buffer("inv_freq_w", inv_freq_w, persistent=False)
|
||||
|
||||
self.embeddings = MiniMaxVLPatchEmbed(config)
|
||||
self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
||||
|
||||
n_layers = config.num_hidden_layers
|
||||
if num_hidden_layers_override is None:
|
||||
num_hidden_layers_override = n_layers
|
||||
self.encoder = MiniMaxVLEncoder(
|
||||
config=config,
|
||||
num_hidden_layers_override=num_hidden_layers_override,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.encoder",
|
||||
)
|
||||
|
||||
if require_post_norm is None:
|
||||
require_post_norm = num_hidden_layers_override == n_layers
|
||||
self.post_layernorm = (
|
||||
nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
||||
if require_post_norm
|
||||
else None
|
||||
)
|
||||
|
||||
# out_hidden_size needed by run_dp_sharded_mrope_vision_model
|
||||
self.out_hidden_size = embed_dim
|
||||
|
||||
# ── RoPE helpers ─────────────────────────────────────────────────────
|
||||
|
||||
def _get_3d_rope_embed(
|
||||
self, grid_t: int, grid_h: int, grid_w: int, spatial_merge_size: int
|
||||
) -> torch.Tensor:
|
||||
"""Compute 3D RoPE frequencies for a single (T, H, W) grid.
|
||||
|
||||
Returns (T*H*W, half_rot_dim) on the same device as inv_freq buffers.
|
||||
Mirrors the reference ``_get_3d_rope_embed`` exactly.
|
||||
"""
|
||||
tokens_per_frame = grid_h * grid_w
|
||||
|
||||
tpos_ids = (
|
||||
torch.arange(grid_t, device=self.inv_freq_t.device)
|
||||
.unsqueeze(1)
|
||||
.expand(-1, tokens_per_frame)
|
||||
.flatten()
|
||||
)
|
||||
|
||||
hpos_ids = (
|
||||
torch.arange(grid_h, device=self.inv_freq_h.device)
|
||||
.unsqueeze(1)
|
||||
.expand(-1, grid_w)
|
||||
.reshape(
|
||||
grid_h // spatial_merge_size,
|
||||
spatial_merge_size,
|
||||
grid_w // spatial_merge_size,
|
||||
spatial_merge_size,
|
||||
)
|
||||
.permute(0, 2, 1, 3)
|
||||
.unsqueeze(0)
|
||||
.expand(grid_t, -1, -1, -1, -1)
|
||||
.flatten()
|
||||
)
|
||||
wpos_ids = (
|
||||
torch.arange(grid_w, device=self.inv_freq_w.device)
|
||||
.unsqueeze(0)
|
||||
.expand(grid_h, -1)
|
||||
.reshape(
|
||||
grid_h // spatial_merge_size,
|
||||
spatial_merge_size,
|
||||
grid_w // spatial_merge_size,
|
||||
spatial_merge_size,
|
||||
)
|
||||
.permute(0, 2, 1, 3)
|
||||
.unsqueeze(0)
|
||||
.expand(grid_t, -1, -1, -1, -1)
|
||||
.flatten()
|
||||
)
|
||||
|
||||
max_t = max(grid_t, 1)
|
||||
max_hw = max(grid_h, grid_w)
|
||||
|
||||
seq_t = torch.arange(
|
||||
max_t, device=self.inv_freq_t.device, dtype=self.inv_freq_t.dtype
|
||||
)
|
||||
seq_hw = torch.arange(
|
||||
max_hw, device=self.inv_freq_h.device, dtype=self.inv_freq_h.dtype
|
||||
)
|
||||
|
||||
freqs_t = torch.outer(seq_t, self.inv_freq_t) # (max_t, t_dim/2)
|
||||
freqs_h = torch.outer(seq_hw, self.inv_freq_h) # (max_hw, h_dim/2)
|
||||
freqs_w = torch.outer(seq_hw, self.inv_freq_w) # (max_hw, w_dim/2)
|
||||
|
||||
return torch.cat(
|
||||
[freqs_t[tpos_ids], freqs_h[hpos_ids], freqs_w[wpos_ids]], dim=-1
|
||||
) # (T*H*W, half_rot_dim)
|
||||
|
||||
def _get_rope_embed_3d(
|
||||
self, grid_thw: list[list[int]], spatial_merge_size: int
|
||||
) -> torch.Tensor:
|
||||
embeds = [
|
||||
self._get_3d_rope_embed(t, h, w, spatial_merge_size) for t, h, w in grid_thw
|
||||
]
|
||||
return torch.cat(embeds, dim=0) # (total_N, half_rot_dim)
|
||||
|
||||
# ── Frame-limit helper (mirrors the reference) ───────────────────────
|
||||
|
||||
def _apply_max_frames_limit(self, grid_thw: list[list[int]]) -> list[list[int]]:
|
||||
if self.vision_segment_max_frames is None:
|
||||
return grid_thw
|
||||
max_f = self.vision_segment_max_frames
|
||||
out: list[list[int]] = []
|
||||
for t, h, w in grid_thw:
|
||||
if t <= max_f:
|
||||
out.append([t, h, w])
|
||||
else:
|
||||
for i in range(0, t, max_f):
|
||||
out.append([min(max_f, t - i), h, w])
|
||||
return out
|
||||
|
||||
# ── Forward ──────────────────────────────────────────────────────────
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thw: list[list[int]],
|
||||
) -> torch.Tensor:
|
||||
# pixel_values: (total_N, C * temporal_patch_size * patch_size²)
|
||||
# Output: (total_N, hidden_size)
|
||||
|
||||
hidden = self.embeddings(pixel_values) # (total_N, hidden_size)
|
||||
hidden = self.pre_layrnorm(hidden)
|
||||
|
||||
limited = self._apply_max_frames_limit(grid_thw)
|
||||
|
||||
# Token-level cumulative sequence lengths (one segment per limited grid).
|
||||
lens = [t * h * w for t, h, w in limited]
|
||||
cu_seqlens_np = np.zeros(len(lens) + 1, dtype=np.int32)
|
||||
np.cumsum(np.array(lens, dtype=np.int32), out=cu_seqlens_np[1:])
|
||||
|
||||
# Backend-specific encoder metadata. For FLASH_ATTN this returns the raw
|
||||
# token cu_seqlens, the max segment length, and sequence_lengths=None;
|
||||
# for FLASHINFER (cuDNN) it repacks cu_seqlens into element-offset
|
||||
# indptrs, buckets max_seqlen, and builds padded per-sequence lengths.
|
||||
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
|
||||
self.attn_backend, cu_seqlens_np, hidden.device
|
||||
)
|
||||
max_seqlen = torch.tensor(
|
||||
MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens_np),
|
||||
dtype=torch.int32,
|
||||
)
|
||||
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
||||
self.attn_backend,
|
||||
cu_seqlens_np,
|
||||
self.hidden_size,
|
||||
self.tp_size,
|
||||
hidden.device,
|
||||
)
|
||||
|
||||
# 3D RoPE: (total_N, half_rot_dim); ApplyRotaryEmb expands internally
|
||||
freqs = self._get_rope_embed_3d(limited, self.spatial_merge_size)
|
||||
freqs = freqs.to(device=hidden.device)
|
||||
# Keep cos/sin in fp32; ApplyRotaryEmb(enable_fp32_compute=True) runs the
|
||||
# rotation in fp32 to match the reference precision.
|
||||
rotary_cos, rotary_sin = freqs.cos(), freqs.sin()
|
||||
|
||||
# Encoder expects (N, 1, hidden_size) — add batch dim
|
||||
hidden = hidden.unsqueeze(1)
|
||||
# On ROCm, the flash_attn Triton rotary kernel can fail with
|
||||
# hipErrorInvalidValue when seqlen is very large, e.g. 192k video
|
||||
# tokens; pass per-segment lengths so RoPE can be applied in chunks.
|
||||
# On other platforms leave it None -> single-kernel fast path, so the
|
||||
# NVIDIA/CUDA code path is unchanged.
|
||||
rotary_segment_lengths = lens if current_platform.is_rocm() else None
|
||||
|
||||
hidden = self.encoder(
|
||||
hidden,
|
||||
cu_seqlens,
|
||||
rotary_cos,
|
||||
rotary_sin,
|
||||
max_seqlen,
|
||||
rotary_segment_lengths,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
hidden = hidden.squeeze(1) # back to (total_N, hidden_size)
|
||||
|
||||
if self.post_layernorm is not None:
|
||||
hidden = self.post_layernorm(hidden)
|
||||
|
||||
return hidden
|
||||
|
||||
|
||||
class MiniMaxVLMultiModalProjector(nn.Module):
|
||||
"""Two-layer MLP projector: vision_hidden → text_hidden."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_hidden_size: int,
|
||||
text_hidden_size: int,
|
||||
projector_hidden_size: int | None,
|
||||
multimodal_projector_bias: bool,
|
||||
projector_hidden_act: str = "gelu",
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
mid = projector_hidden_size if projector_hidden_size else text_hidden_size
|
||||
use_dp = is_vit_use_data_parallel()
|
||||
self.linear_1 = ColumnParallelLinear(
|
||||
vision_hidden_size,
|
||||
mid,
|
||||
bias=multimodal_projector_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_1",
|
||||
disable_tp=use_dp,
|
||||
)
|
||||
self.act = get_act_fn(projector_hidden_act)
|
||||
self.linear_2 = RowParallelLinear(
|
||||
mid,
|
||||
text_hidden_size,
|
||||
bias=multimodal_projector_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
disable_tp=use_dp,
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x, _ = self.linear_1(x)
|
||||
x = self.act(x)
|
||||
x, _ = self.linear_2(x)
|
||||
return x
|
||||
|
||||
|
||||
class MiniMaxVLPatchMerger(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
spatial_merge_size: int,
|
||||
text_hidden_size: int,
|
||||
projector_hidden_size: int | None,
|
||||
patch_merge_bias: bool,
|
||||
projector_hidden_act: str = "gelu",
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.spatial_merge_size = spatial_merge_size
|
||||
mid = projector_hidden_size if projector_hidden_size else text_hidden_size
|
||||
merge_in = text_hidden_size * spatial_merge_size**2
|
||||
use_dp = is_vit_use_data_parallel()
|
||||
self.linear_1 = ColumnParallelLinear(
|
||||
merge_in,
|
||||
mid,
|
||||
bias=patch_merge_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_1",
|
||||
disable_tp=use_dp,
|
||||
)
|
||||
self.act = get_act_fn(projector_hidden_act)
|
||||
self.linear_2 = RowParallelLinear(
|
||||
mid,
|
||||
text_hidden_size,
|
||||
bias=patch_merge_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
disable_tp=use_dp,
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# x: (N, text_hidden_size) → (N // merge_size², text_hidden_size)
|
||||
x = x.reshape(x.shape[0] // (self.spatial_merge_size**2), -1)
|
||||
x, _ = self.linear_1(x)
|
||||
x = self.act(x)
|
||||
x, _ = self.linear_2(x)
|
||||
return x
|
||||
|
||||
|
||||
class MiniMaxVLVisionModel(nn.Module):
|
||||
"""Full vision model: ViT → projector → patch merger."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
text_hidden_size: int,
|
||||
projector_hidden_size: int | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
compression = config.img_token_compression_config
|
||||
spatial_merge_size: int = compression.get("spatial_merge_size", 2)
|
||||
self.spatial_merge_size = spatial_merge_size
|
||||
self.use_data_parallel = is_vit_use_data_parallel()
|
||||
|
||||
# The released checkpoint ships no ``post_layernorm`` weights and
|
||||
# uses ``vision_feature_layer=-1`` with ``vision_feature_select_strategy
|
||||
# ="full"``, i.e. the raw last encoder hidden state (CLIP's
|
||||
# ``last_hidden_state`` is taken before the post layernorm). Applying an
|
||||
# untrained post layernorm here would corrupt the visual features.
|
||||
self.vision_model = MiniMaxVLVisionTransformer(
|
||||
config=config,
|
||||
require_post_norm=False,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "vision_model"),
|
||||
)
|
||||
self.multi_modal_projector = MiniMaxVLMultiModalProjector(
|
||||
vision_hidden_size=config.hidden_size,
|
||||
text_hidden_size=text_hidden_size,
|
||||
projector_hidden_size=projector_hidden_size,
|
||||
multimodal_projector_bias=getattr(
|
||||
config, "multimodal_projector_bias", True
|
||||
),
|
||||
projector_hidden_act=getattr(config, "projector_hidden_act", "gelu"),
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "multi_modal_projector"),
|
||||
)
|
||||
self.patch_merge_mlp = MiniMaxVLPatchMerger(
|
||||
spatial_merge_size=spatial_merge_size,
|
||||
text_hidden_size=text_hidden_size,
|
||||
projector_hidden_size=projector_hidden_size,
|
||||
patch_merge_bias=getattr(config, "patch_merge_bias", True),
|
||||
projector_hidden_act=getattr(config, "projector_hidden_act", "gelu"),
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "patch_merge_mlp"),
|
||||
)
|
||||
|
||||
self.dtype = self.vision_model.embeddings.patch_embedding.weight.dtype
|
||||
self.out_hidden_size = text_hidden_size
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thw: list[list[int]],
|
||||
) -> torch.Tensor:
|
||||
hidden = self.vision_model(pixel_values=pixel_values, grid_thw=grid_thw)
|
||||
if hidden.dim() == 3:
|
||||
hidden = hidden.squeeze(0)
|
||||
hidden = self.multi_modal_projector(hidden)
|
||||
hidden = self.patch_merge_mlp(hidden)
|
||||
return hidden
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj.", "q_proj.", "q"),
|
||||
("qkv_proj.", "k_proj.", "k"),
|
||||
("qkv_proj.", "v_proj.", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,312 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .model import (
|
||||
MiniMAXGemmaRMSNorm,
|
||||
MiniMaxM3DecoderLayer,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.enorm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = ReplicatedLinear(
|
||||
config.hidden_size * 2,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.eh_proj",
|
||||
)
|
||||
self.transformer_layer = MiniMaxM3DecoderLayer(
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
force_sparse_attn=True,
|
||||
force_moe=True,
|
||||
is_mtp_block=True,
|
||||
)
|
||||
self.final_layernorm = MiniMAXGemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
# Mask out inputs at position 0, as not needed by MTP.
|
||||
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
|
||||
|
||||
# Combine the normalized token embeddings with the normalized
|
||||
# previous hidden states.
|
||||
inputs_embeds = self.enorm(inputs_embeds)
|
||||
previous_hidden_states = self.hnorm(previous_hidden_states)
|
||||
hidden_states, _ = self.eh_proj(
|
||||
torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
|
||||
)
|
||||
|
||||
# Apply transformer layer.
|
||||
hidden_states, residual = self.transformer_layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
|
||||
hidden_states += residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class MiniMaxM3MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
# Use the draft (MTP) config, not the target model's. This is flat for a
|
||||
# standalone checkpoint, and the promoted text_config for a bundled one.
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.num_mtp_layers = config.num_mtp_modules
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): MiniMaxM3MultiTokenPredictorLayer(
|
||||
vllm_config, f"{prefix}.layers.{idx}"
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = MiniMaxM3MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.vocab_size,
|
||||
self.config.hidden_size,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
current_step_idx = spec_step_idx % self.model.num_mtp_layers
|
||||
mtp_layer = self.model.layers[str(current_step_idx)]
|
||||
return self.logits_processor(
|
||||
self.lm_head, mtp_layer.final_layernorm(hidden_states)
|
||||
)
|
||||
|
||||
def _get_mtp_layer_idx_from_weight_name(self, name: str) -> int | None:
|
||||
"""Return the MTP layer index in *.mtp.layers.{idx}.*, else None."""
|
||||
match = re.search(r"\.mtp\.layers\.(\d+)\.", name)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
def _map_checkpoint_name(self, name: str) -> str | None:
|
||||
"""Map a full checkpoint key to this MTP module's parameter name.
|
||||
|
||||
The MTP module only owns the *.mtp.layers.* weights plus the token
|
||||
embedding and LM head, which the checkpoint shares with the main model.
|
||||
Everything else belongs to other modules and is ignored here by returning
|
||||
None.
|
||||
"""
|
||||
# In the bundled checkpoint, the MTP weights are prefixed with
|
||||
# "language_model". The standalone MTP checkpoint has no such prefix.
|
||||
# Strip it if present.
|
||||
name = name.removeprefix("language_model.")
|
||||
|
||||
if name == "model.embed_tokens.weight":
|
||||
return "model.embed_tokens.weight"
|
||||
if name == "lm_head.weight":
|
||||
return "lm_head.weight"
|
||||
if "model.mtp.layers" in name:
|
||||
if "weight_scale_inv" in name:
|
||||
# The checkpoint stores block scales as "weight_scale_inv".
|
||||
# The ModelOpt MXFP8 layers expose them as "weight_scale".
|
||||
name = name.replace("weight_scale_inv", "weight_scale")
|
||||
# Strip "mtp" from prefix.
|
||||
return name.replace(".mtp.", ".")
|
||||
return None
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Map q/k/v projections to qkv_proj, and gate/up projections to gate_up_proj.
|
||||
stacked_params_mapping: list[tuple[str, str, int | str]] = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".qkv_proj", ".index_q_proj", "index_q"),
|
||||
(".qkv_proj", ".index_k_proj", "index_k"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
|
||||
# Map expert weights w1/w2/w3 to gate/down/up.
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_local_experts,
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
loaded_mtp_layers: set[int] = set()
|
||||
for name, loaded_weight in weights:
|
||||
mtp_layer = self._get_mtp_layer_idx_from_weight_name(name)
|
||||
mapped_name = self._map_checkpoint_name(name)
|
||||
if mapped_name is None:
|
||||
# This weight does not belong to the MTP module, so skip it.
|
||||
continue
|
||||
name = mapped_name
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
# Routed experts (w1/w2/w3) are handled below. Don't let the
|
||||
# stacked mapping rewrite them.
|
||||
if ("block_sparse_moe.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
expert_id,
|
||||
expert_shard_id,
|
||||
) in expert_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None or remapped_name not in params_dict:
|
||||
continue
|
||||
name = remapped_name
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
loaded_params.add(name)
|
||||
if mtp_layer is not None:
|
||||
loaded_mtp_layers.add(mtp_layer)
|
||||
|
||||
# Validate that weights were loaded for each MTP layer.
|
||||
for layer_idx in range(self.model.num_mtp_layers):
|
||||
if layer_idx not in loaded_mtp_layers:
|
||||
raise ValueError(
|
||||
f"Failed to load MTP layer {layer_idx} weights from checkpoint."
|
||||
)
|
||||
|
||||
return loaded_params
|
||||
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MSA (SM100/Blackwell) block-sparse attend for MiniMax M3.
|
||||
|
||||
Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``);
|
||||
decode falls back to the Triton split-K kernel (no MSA decode yet). ``fmha_sm100``
|
||||
imports are function-local, so this module is import-safe on AMD/non-SM100.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
SPARSE_BLOCK_SIZE,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseImpl,
|
||||
MiniMaxM3SparseMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionLayer
|
||||
|
||||
|
||||
class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
"""MSA block-sparse attend (``fmha_sm100``); Triton split-K decode."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
topk_idx: tuple[torch.Tensor | None, torch.Tensor | None],
|
||||
output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return output # profiling run; caches unbound
|
||||
main_md = attn_metadata[layer.layer_name] # type: ignore[attr-defined]
|
||||
assert isinstance(main_md, MiniMaxM3SparseMetadata)
|
||||
decode_topk, prefill_topk = topk_idx
|
||||
|
||||
nd = main_md.num_decode_tokens
|
||||
num_tokens = main_md.num_actual_tokens
|
||||
hd = self.head_size
|
||||
q = query[:num_tokens].view(-1, self.num_heads, hd)
|
||||
out = output[:num_tokens].view(-1, self.num_heads, hd)
|
||||
kv_cache = (
|
||||
kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache
|
||||
)
|
||||
|
||||
# Decode [:nd]: Triton split-K placeholder (no MSA decode yet).
|
||||
if main_md.num_decodes > 0:
|
||||
d = main_md.decode
|
||||
assert d is not None and decode_topk is not None
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q[:nd],
|
||||
kv_cache,
|
||||
decode_topk,
|
||||
d.block_table,
|
||||
d.seq_lens,
|
||||
self.num_kv_heads,
|
||||
self.scale,
|
||||
out[:nd],
|
||||
d.decode_query_len,
|
||||
)
|
||||
|
||||
# Prefill [nd:]: MSA sparse FMHA over the selected blocks.
|
||||
if main_md.num_prefills > 0:
|
||||
from vllm.third_party.fmha_sm100.sparse import (
|
||||
build_k2q_csr,
|
||||
sparse_atten_func,
|
||||
)
|
||||
|
||||
p = main_md.prefill
|
||||
assert p is not None and prefill_topk is not None
|
||||
qp = q[nd:]
|
||||
k_cache = kv_cache[:, 0].transpose(1, 2)
|
||||
v_cache = kv_cache[:, 1].transpose(1, 2)
|
||||
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
|
||||
prefill_topk,
|
||||
p.cu_seqlens_q,
|
||||
p.cu_seqlens_k,
|
||||
SPARSE_BLOCK_SIZE,
|
||||
total_k=0,
|
||||
max_seqlen_k=p.max_seq_len,
|
||||
max_seqlen_q=p.max_query_len,
|
||||
total_rows=p.total_kv_blocks,
|
||||
qhead_per_kv=qp.shape[1] // self.num_kv_heads,
|
||||
return_schedule=True,
|
||||
)
|
||||
sparse_atten_func(
|
||||
qp,
|
||||
k_cache,
|
||||
v_cache,
|
||||
k2q_row_ptr,
|
||||
k2q_q_indices,
|
||||
topK=self.topk_blocks,
|
||||
blk_kv=SPARSE_BLOCK_SIZE,
|
||||
causal=True,
|
||||
softmax_scale=self.scale,
|
||||
cu_seqlens_q=p.cu_seqlens_q,
|
||||
cu_seqlens_k=p.cu_seqlens_k,
|
||||
max_seqlen_q=p.max_query_len,
|
||||
max_seqlen_k=p.max_seq_len,
|
||||
page_table=p.block_table,
|
||||
seqused_k=p.seq_lens,
|
||||
schedule=schedule,
|
||||
out=out[nd:],
|
||||
)
|
||||
return output
|
||||
Reference in New Issue
Block a user